1 /*
2  * Copyright 2016 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice shall be included in
12  * all copies or substantial portions of the Software.
13  *
14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20  * OTHER DEALINGS IN THE SOFTWARE.
21  *
22  * Authors: Christian König
23  */
24 
25 #include <linux/dma-mapping.h>
26 #include <drm/ttm/ttm_range_manager.h>
27 
28 #include "amdgpu.h"
29 #include "amdgpu_vm.h"
30 #include "amdgpu_res_cursor.h"
31 #include "amdgpu_atomfirmware.h"
32 #include "atom.h"
33 
34 struct amdgpu_vram_reservation {
35 	u64 start;
36 	u64 size;
37 	struct list_head allocated;
38 	struct list_head blocks;
39 };
40 
41 static inline struct amdgpu_vram_mgr *
42 to_vram_mgr(struct ttm_resource_manager *man)
43 {
44 	return container_of(man, struct amdgpu_vram_mgr, manager);
45 }
46 
47 static inline struct amdgpu_device *
48 to_amdgpu_device(struct amdgpu_vram_mgr *mgr)
49 {
50 	return container_of(mgr, struct amdgpu_device, mman.vram_mgr);
51 }
52 
53 static inline struct drm_buddy_block *
54 amdgpu_vram_mgr_first_block(struct list_head *list)
55 {
56 	return list_first_entry_or_null(list, struct drm_buddy_block, link);
57 }
58 
59 static inline bool amdgpu_is_vram_mgr_blocks_contiguous(struct list_head *head)
60 {
61 	struct drm_buddy_block *block;
62 	u64 start, size;
63 
64 	block = amdgpu_vram_mgr_first_block(head);
65 	if (!block)
66 		return false;
67 
68 	while (head != block->link.next) {
69 		start = amdgpu_vram_mgr_block_start(block);
70 		size = amdgpu_vram_mgr_block_size(block);
71 
72 		block = list_entry(block->link.next, struct drm_buddy_block, link);
73 		if (start + size != amdgpu_vram_mgr_block_start(block))
74 			return false;
75 	}
76 
77 	return true;
78 }
79 
80 
81 
82 /**
83  * DOC: mem_info_vram_total
84  *
85  * The amdgpu driver provides a sysfs API for reporting current total VRAM
86  * available on the device
87  * The file mem_info_vram_total is used for this and returns the total
88  * amount of VRAM in bytes
89  */
90 static ssize_t amdgpu_mem_info_vram_total_show(struct device *dev,
91 		struct device_attribute *attr, char *buf)
92 {
93 	struct drm_device *ddev = dev_get_drvdata(dev);
94 	struct amdgpu_device *adev = drm_to_adev(ddev);
95 
96 	return sysfs_emit(buf, "%llu\n", adev->gmc.real_vram_size);
97 }
98 
99 /**
100  * DOC: mem_info_vis_vram_total
101  *
102  * The amdgpu driver provides a sysfs API for reporting current total
103  * visible VRAM available on the device
104  * The file mem_info_vis_vram_total is used for this and returns the total
105  * amount of visible VRAM in bytes
106  */
107 static ssize_t amdgpu_mem_info_vis_vram_total_show(struct device *dev,
108 		struct device_attribute *attr, char *buf)
109 {
110 	struct drm_device *ddev = dev_get_drvdata(dev);
111 	struct amdgpu_device *adev = drm_to_adev(ddev);
112 
113 	return sysfs_emit(buf, "%llu\n", adev->gmc.visible_vram_size);
114 }
115 
116 /**
117  * DOC: mem_info_vram_used
118  *
119  * The amdgpu driver provides a sysfs API for reporting current total VRAM
120  * available on the device
121  * The file mem_info_vram_used is used for this and returns the total
122  * amount of currently used VRAM in bytes
123  */
124 static ssize_t amdgpu_mem_info_vram_used_show(struct device *dev,
125 					      struct device_attribute *attr,
126 					      char *buf)
127 {
128 	struct drm_device *ddev = dev_get_drvdata(dev);
129 	struct amdgpu_device *adev = drm_to_adev(ddev);
130 	struct ttm_resource_manager *man = &adev->mman.vram_mgr.manager;
131 
132 	return sysfs_emit(buf, "%llu\n", ttm_resource_manager_usage(man));
133 }
134 
135 /**
136  * DOC: mem_info_vis_vram_used
137  *
138  * The amdgpu driver provides a sysfs API for reporting current total of
139  * used visible VRAM
140  * The file mem_info_vis_vram_used is used for this and returns the total
141  * amount of currently used visible VRAM in bytes
142  */
143 static ssize_t amdgpu_mem_info_vis_vram_used_show(struct device *dev,
144 						  struct device_attribute *attr,
145 						  char *buf)
146 {
147 	struct drm_device *ddev = dev_get_drvdata(dev);
148 	struct amdgpu_device *adev = drm_to_adev(ddev);
149 
150 	return sysfs_emit(buf, "%llu\n",
151 			  amdgpu_vram_mgr_vis_usage(&adev->mman.vram_mgr));
152 }
153 
154 /**
155  * DOC: mem_info_vram_vendor
156  *
157  * The amdgpu driver provides a sysfs API for reporting the vendor of the
158  * installed VRAM
159  * The file mem_info_vram_vendor is used for this and returns the name of the
160  * vendor.
161  */
162 static ssize_t amdgpu_mem_info_vram_vendor(struct device *dev,
163 					   struct device_attribute *attr,
164 					   char *buf)
165 {
166 	struct drm_device *ddev = dev_get_drvdata(dev);
167 	struct amdgpu_device *adev = drm_to_adev(ddev);
168 
169 	switch (adev->gmc.vram_vendor) {
170 	case SAMSUNG:
171 		return sysfs_emit(buf, "samsung\n");
172 	case INFINEON:
173 		return sysfs_emit(buf, "infineon\n");
174 	case ELPIDA:
175 		return sysfs_emit(buf, "elpida\n");
176 	case ETRON:
177 		return sysfs_emit(buf, "etron\n");
178 	case NANYA:
179 		return sysfs_emit(buf, "nanya\n");
180 	case HYNIX:
181 		return sysfs_emit(buf, "hynix\n");
182 	case MOSEL:
183 		return sysfs_emit(buf, "mosel\n");
184 	case WINBOND:
185 		return sysfs_emit(buf, "winbond\n");
186 	case ESMT:
187 		return sysfs_emit(buf, "esmt\n");
188 	case MICRON:
189 		return sysfs_emit(buf, "micron\n");
190 	default:
191 		return sysfs_emit(buf, "unknown\n");
192 	}
193 }
194 
195 static DEVICE_ATTR(mem_info_vram_total, S_IRUGO,
196 		   amdgpu_mem_info_vram_total_show, NULL);
197 static DEVICE_ATTR(mem_info_vis_vram_total, S_IRUGO,
198 		   amdgpu_mem_info_vis_vram_total_show,NULL);
199 static DEVICE_ATTR(mem_info_vram_used, S_IRUGO,
200 		   amdgpu_mem_info_vram_used_show, NULL);
201 static DEVICE_ATTR(mem_info_vis_vram_used, S_IRUGO,
202 		   amdgpu_mem_info_vis_vram_used_show, NULL);
203 static DEVICE_ATTR(mem_info_vram_vendor, S_IRUGO,
204 		   amdgpu_mem_info_vram_vendor, NULL);
205 
206 static struct attribute *amdgpu_vram_mgr_attributes[] = {
207 	&dev_attr_mem_info_vram_total.attr,
208 	&dev_attr_mem_info_vis_vram_total.attr,
209 	&dev_attr_mem_info_vram_used.attr,
210 	&dev_attr_mem_info_vis_vram_used.attr,
211 	&dev_attr_mem_info_vram_vendor.attr,
212 	NULL
213 };
214 
215 const struct attribute_group amdgpu_vram_mgr_attr_group = {
216 	.attrs = amdgpu_vram_mgr_attributes
217 };
218 
219 /**
220  * amdgpu_vram_mgr_vis_size - Calculate visible block size
221  *
222  * @adev: amdgpu_device pointer
223  * @block: DRM BUDDY block structure
224  *
225  * Calculate how many bytes of the DRM BUDDY block are inside visible VRAM
226  */
227 static u64 amdgpu_vram_mgr_vis_size(struct amdgpu_device *adev,
228 				    struct drm_buddy_block *block)
229 {
230 	u64 start = amdgpu_vram_mgr_block_start(block);
231 	u64 end = start + amdgpu_vram_mgr_block_size(block);
232 
233 	if (start >= adev->gmc.visible_vram_size)
234 		return 0;
235 
236 	return (end > adev->gmc.visible_vram_size ?
237 		adev->gmc.visible_vram_size : end) - start;
238 }
239 
240 /**
241  * amdgpu_vram_mgr_bo_visible_size - CPU visible BO size
242  *
243  * @bo: &amdgpu_bo buffer object (must be in VRAM)
244  *
245  * Returns:
246  * How much of the given &amdgpu_bo buffer object lies in CPU visible VRAM.
247  */
248 u64 amdgpu_vram_mgr_bo_visible_size(struct amdgpu_bo *bo)
249 {
250 	struct amdgpu_device *adev = amdgpu_ttm_adev(bo->tbo.bdev);
251 	struct ttm_resource *res = bo->tbo.resource;
252 	struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
253 	struct drm_buddy_block *block;
254 	u64 usage = 0;
255 
256 	if (amdgpu_gmc_vram_full_visible(&adev->gmc))
257 		return amdgpu_bo_size(bo);
258 
259 	if (res->start >= adev->gmc.visible_vram_size >> PAGE_SHIFT)
260 		return 0;
261 
262 	list_for_each_entry(block, &vres->blocks, link)
263 		usage += amdgpu_vram_mgr_vis_size(adev, block);
264 
265 	return usage;
266 }
267 
268 /* Commit the reservation of VRAM pages */
269 static void amdgpu_vram_mgr_do_reserve(struct ttm_resource_manager *man)
270 {
271 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
272 	struct amdgpu_device *adev = to_amdgpu_device(mgr);
273 	struct drm_buddy *mm = &mgr->mm;
274 	struct amdgpu_vram_reservation *rsv, *temp;
275 	struct drm_buddy_block *block;
276 	uint64_t vis_usage;
277 
278 	list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks) {
279 		if (drm_buddy_alloc_blocks(mm, rsv->start, rsv->start + rsv->size,
280 					   rsv->size, mm->chunk_size, &rsv->allocated,
281 					   DRM_BUDDY_RANGE_ALLOCATION))
282 			continue;
283 
284 		block = amdgpu_vram_mgr_first_block(&rsv->allocated);
285 		if (!block)
286 			continue;
287 
288 		dev_dbg(adev->dev, "Reservation 0x%llx - %lld, Succeeded\n",
289 			rsv->start, rsv->size);
290 
291 		vis_usage = amdgpu_vram_mgr_vis_size(adev, block);
292 		atomic64_add(vis_usage, &mgr->vis_usage);
293 		spin_lock(&man->bdev->lru_lock);
294 		man->usage += rsv->size;
295 		spin_unlock(&man->bdev->lru_lock);
296 		list_move(&rsv->blocks, &mgr->reserved_pages);
297 	}
298 }
299 
300 /**
301  * amdgpu_vram_mgr_reserve_range - Reserve a range from VRAM
302  *
303  * @mgr: amdgpu_vram_mgr pointer
304  * @start: start address of the range in VRAM
305  * @size: size of the range
306  *
307  * Reserve memory from start address with the specified size in VRAM
308  */
309 int amdgpu_vram_mgr_reserve_range(struct amdgpu_vram_mgr *mgr,
310 				  uint64_t start, uint64_t size)
311 {
312 	struct amdgpu_vram_reservation *rsv;
313 
314 	rsv = kzalloc(sizeof(*rsv), GFP_KERNEL);
315 	if (!rsv)
316 		return -ENOMEM;
317 
318 	INIT_LIST_HEAD(&rsv->allocated);
319 	INIT_LIST_HEAD(&rsv->blocks);
320 
321 	rsv->start = start;
322 	rsv->size = size;
323 
324 	mutex_lock(&mgr->lock);
325 	list_add_tail(&rsv->blocks, &mgr->reservations_pending);
326 	amdgpu_vram_mgr_do_reserve(&mgr->manager);
327 	mutex_unlock(&mgr->lock);
328 
329 	return 0;
330 }
331 
332 /**
333  * amdgpu_vram_mgr_query_page_status - query the reservation status
334  *
335  * @mgr: amdgpu_vram_mgr pointer
336  * @start: start address of a page in VRAM
337  *
338  * Returns:
339  *	-EBUSY: the page is still hold and in pending list
340  *	0: the page has been reserved
341  *	-ENOENT: the input page is not a reservation
342  */
343 int amdgpu_vram_mgr_query_page_status(struct amdgpu_vram_mgr *mgr,
344 				      uint64_t start)
345 {
346 	struct amdgpu_vram_reservation *rsv;
347 	int ret;
348 
349 	mutex_lock(&mgr->lock);
350 
351 	list_for_each_entry(rsv, &mgr->reservations_pending, blocks) {
352 		if (rsv->start <= start &&
353 		    (start < (rsv->start + rsv->size))) {
354 			ret = -EBUSY;
355 			goto out;
356 		}
357 	}
358 
359 	list_for_each_entry(rsv, &mgr->reserved_pages, blocks) {
360 		if (rsv->start <= start &&
361 		    (start < (rsv->start + rsv->size))) {
362 			ret = 0;
363 			goto out;
364 		}
365 	}
366 
367 	ret = -ENOENT;
368 out:
369 	mutex_unlock(&mgr->lock);
370 	return ret;
371 }
372 
373 /**
374  * amdgpu_vram_mgr_new - allocate new ranges
375  *
376  * @man: TTM memory type manager
377  * @tbo: TTM BO we need this range for
378  * @place: placement flags and restrictions
379  * @res: the resulting mem object
380  *
381  * Allocate VRAM for the given BO.
382  */
383 static int amdgpu_vram_mgr_new(struct ttm_resource_manager *man,
384 			       struct ttm_buffer_object *tbo,
385 			       const struct ttm_place *place,
386 			       struct ttm_resource **res)
387 {
388 	u64 vis_usage = 0, max_bytes, cur_size, min_block_size;
389 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
390 	struct amdgpu_device *adev = to_amdgpu_device(mgr);
391 	struct amdgpu_vram_mgr_resource *vres;
392 	u64 size, remaining_size, lpfn, fpfn;
393 	struct drm_buddy *mm = &mgr->mm;
394 	struct drm_buddy_block *block;
395 	unsigned long pages_per_block;
396 	int r;
397 
398 	lpfn = (u64)place->lpfn << PAGE_SHIFT;
399 	if (!lpfn)
400 		lpfn = man->size;
401 
402 	fpfn = (u64)place->fpfn << PAGE_SHIFT;
403 
404 	max_bytes = adev->gmc.mc_vram_size;
405 	if (tbo->type != ttm_bo_type_kernel)
406 		max_bytes -= AMDGPU_VM_RESERVED_VRAM;
407 
408 	if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
409 		pages_per_block = ~0ul;
410 	} else {
411 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
412 		pages_per_block = HPAGE_PMD_NR;
413 #else
414 		/* default to 2MB */
415 		pages_per_block = 2UL << (20UL - PAGE_SHIFT);
416 #endif
417 		pages_per_block = max_t(uint32_t, pages_per_block,
418 					tbo->page_alignment);
419 	}
420 
421 	vres = kzalloc(sizeof(*vres), GFP_KERNEL);
422 	if (!vres)
423 		return -ENOMEM;
424 
425 	ttm_resource_init(tbo, place, &vres->base);
426 
427 	/* bail out quickly if there's likely not enough VRAM for this BO */
428 	if (ttm_resource_manager_usage(man) > max_bytes) {
429 		r = -ENOSPC;
430 		goto error_fini;
431 	}
432 
433 	INIT_LIST_HEAD(&vres->blocks);
434 
435 	if (place->flags & TTM_PL_FLAG_TOPDOWN)
436 		vres->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION;
437 
438 	if (fpfn || lpfn != mgr->mm.size)
439 		/* Allocate blocks in desired range */
440 		vres->flags |= DRM_BUDDY_RANGE_ALLOCATION;
441 
442 	remaining_size = (u64)vres->base.num_pages << PAGE_SHIFT;
443 
444 	mutex_lock(&mgr->lock);
445 	while (remaining_size) {
446 		if (tbo->page_alignment)
447 			min_block_size = (u64)tbo->page_alignment << PAGE_SHIFT;
448 		else
449 			min_block_size = mgr->default_page_size;
450 
451 		BUG_ON(min_block_size < mm->chunk_size);
452 
453 		/* Limit maximum size to 2GiB due to SG table limitations */
454 		size = min(remaining_size, 2ULL << 30);
455 
456 		if (size >= (u64)pages_per_block << PAGE_SHIFT)
457 			min_block_size = (u64)pages_per_block << PAGE_SHIFT;
458 
459 		cur_size = size;
460 
461 		if (fpfn + size != (u64)place->lpfn << PAGE_SHIFT) {
462 			/*
463 			 * Except for actual range allocation, modify the size and
464 			 * min_block_size conforming to continuous flag enablement
465 			 */
466 			if (place->flags & TTM_PL_FLAG_CONTIGUOUS) {
467 				size = roundup_pow_of_two(size);
468 				min_block_size = size;
469 			/*
470 			 * Modify the size value if size is not
471 			 * aligned with min_block_size
472 			 */
473 			} else if (!IS_ALIGNED(size, min_block_size)) {
474 				size = round_up(size, min_block_size);
475 			}
476 		}
477 
478 		r = drm_buddy_alloc_blocks(mm, fpfn,
479 					   lpfn,
480 					   size,
481 					   min_block_size,
482 					   &vres->blocks,
483 					   vres->flags);
484 		if (unlikely(r))
485 			goto error_free_blocks;
486 
487 		if (size > remaining_size)
488 			remaining_size = 0;
489 		else
490 			remaining_size -= size;
491 	}
492 	mutex_unlock(&mgr->lock);
493 
494 	if (cur_size != size) {
495 		struct drm_buddy_block *block;
496 		struct list_head *trim_list;
497 		u64 original_size;
498 		LIST_HEAD(temp);
499 
500 		trim_list = &vres->blocks;
501 		original_size = (u64)vres->base.num_pages << PAGE_SHIFT;
502 
503 		/*
504 		 * If size value is rounded up to min_block_size, trim the last
505 		 * block to the required size
506 		 */
507 		if (!list_is_singular(&vres->blocks)) {
508 			block = list_last_entry(&vres->blocks, typeof(*block), link);
509 			list_move_tail(&block->link, &temp);
510 			trim_list = &temp;
511 			/*
512 			 * Compute the original_size value by subtracting the
513 			 * last block size with (aligned size - original size)
514 			 */
515 			original_size = amdgpu_vram_mgr_block_size(block) - (size - cur_size);
516 		}
517 
518 		mutex_lock(&mgr->lock);
519 		drm_buddy_block_trim(mm,
520 				     original_size,
521 				     trim_list);
522 		mutex_unlock(&mgr->lock);
523 
524 		if (!list_empty(&temp))
525 			list_splice_tail(trim_list, &vres->blocks);
526 	}
527 
528 	vres->base.start = 0;
529 	list_for_each_entry(block, &vres->blocks, link) {
530 		unsigned long start;
531 
532 		start = amdgpu_vram_mgr_block_start(block) +
533 			amdgpu_vram_mgr_block_size(block);
534 		start >>= PAGE_SHIFT;
535 
536 		if (start > vres->base.num_pages)
537 			start -= vres->base.num_pages;
538 		else
539 			start = 0;
540 		vres->base.start = max(vres->base.start, start);
541 
542 		vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
543 	}
544 
545 	if (amdgpu_is_vram_mgr_blocks_contiguous(&vres->blocks))
546 		vres->base.placement |= TTM_PL_FLAG_CONTIGUOUS;
547 
548 	if (adev->gmc.xgmi.connected_to_cpu)
549 		vres->base.bus.caching = ttm_cached;
550 	else
551 		vres->base.bus.caching = ttm_write_combined;
552 
553 	atomic64_add(vis_usage, &mgr->vis_usage);
554 	*res = &vres->base;
555 	return 0;
556 
557 error_free_blocks:
558 	drm_buddy_free_list(mm, &vres->blocks);
559 	mutex_unlock(&mgr->lock);
560 error_fini:
561 	ttm_resource_fini(man, &vres->base);
562 	kfree(vres);
563 
564 	return r;
565 }
566 
567 /**
568  * amdgpu_vram_mgr_del - free ranges
569  *
570  * @man: TTM memory type manager
571  * @res: TTM memory object
572  *
573  * Free the allocated VRAM again.
574  */
575 static void amdgpu_vram_mgr_del(struct ttm_resource_manager *man,
576 				struct ttm_resource *res)
577 {
578 	struct amdgpu_vram_mgr_resource *vres = to_amdgpu_vram_mgr_resource(res);
579 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
580 	struct amdgpu_device *adev = to_amdgpu_device(mgr);
581 	struct drm_buddy *mm = &mgr->mm;
582 	struct drm_buddy_block *block;
583 	uint64_t vis_usage = 0;
584 
585 	mutex_lock(&mgr->lock);
586 	list_for_each_entry(block, &vres->blocks, link)
587 		vis_usage += amdgpu_vram_mgr_vis_size(adev, block);
588 
589 	amdgpu_vram_mgr_do_reserve(man);
590 
591 	drm_buddy_free_list(mm, &vres->blocks);
592 	mutex_unlock(&mgr->lock);
593 
594 	atomic64_sub(vis_usage, &mgr->vis_usage);
595 
596 	ttm_resource_fini(man, res);
597 	kfree(vres);
598 }
599 
600 /**
601  * amdgpu_vram_mgr_alloc_sgt - allocate and fill a sg table
602  *
603  * @adev: amdgpu device pointer
604  * @res: TTM memory object
605  * @offset: byte offset from the base of VRAM BO
606  * @length: number of bytes to export in sg_table
607  * @dev: the other device
608  * @dir: dma direction
609  * @sgt: resulting sg table
610  *
611  * Allocate and fill a sg table from a VRAM allocation.
612  */
613 int amdgpu_vram_mgr_alloc_sgt(struct amdgpu_device *adev,
614 			      struct ttm_resource *res,
615 			      u64 offset, u64 length,
616 			      struct device *dev,
617 			      enum dma_data_direction dir,
618 			      struct sg_table **sgt)
619 {
620 	struct amdgpu_res_cursor cursor;
621 	struct scatterlist *sg;
622 	int num_entries = 0;
623 	int i, r;
624 
625 	*sgt = kmalloc(sizeof(**sgt), GFP_KERNEL);
626 	if (!*sgt)
627 		return -ENOMEM;
628 
629 	/* Determine the number of DRM_BUDDY blocks to export */
630 	amdgpu_res_first(res, offset, length, &cursor);
631 	while (cursor.remaining) {
632 		num_entries++;
633 		amdgpu_res_next(&cursor, cursor.size);
634 	}
635 
636 	r = sg_alloc_table(*sgt, num_entries, GFP_KERNEL);
637 	if (r)
638 		goto error_free;
639 
640 	/* Initialize scatterlist nodes of sg_table */
641 	for_each_sgtable_sg((*sgt), sg, i)
642 		sg->length = 0;
643 
644 	/*
645 	 * Walk down DRM_BUDDY blocks to populate scatterlist nodes
646 	 * @note: Use iterator api to get first the DRM_BUDDY block
647 	 * and the number of bytes from it. Access the following
648 	 * DRM_BUDDY block(s) if more buffer needs to exported
649 	 */
650 	amdgpu_res_first(res, offset, length, &cursor);
651 	for_each_sgtable_sg((*sgt), sg, i) {
652 		phys_addr_t phys = cursor.start + adev->gmc.aper_base;
653 		size_t size = cursor.size;
654 		dma_addr_t addr;
655 
656 		addr = dma_map_resource(dev, phys, size, dir,
657 					DMA_ATTR_SKIP_CPU_SYNC);
658 		r = dma_mapping_error(dev, addr);
659 		if (r)
660 			goto error_unmap;
661 
662 		sg_set_page(sg, NULL, size, 0);
663 		sg_dma_address(sg) = addr;
664 		sg_dma_len(sg) = size;
665 
666 		amdgpu_res_next(&cursor, cursor.size);
667 	}
668 
669 	return 0;
670 
671 error_unmap:
672 	for_each_sgtable_sg((*sgt), sg, i) {
673 		if (!sg->length)
674 			continue;
675 
676 		dma_unmap_resource(dev, sg->dma_address,
677 				   sg->length, dir,
678 				   DMA_ATTR_SKIP_CPU_SYNC);
679 	}
680 	sg_free_table(*sgt);
681 
682 error_free:
683 	kfree(*sgt);
684 	return r;
685 }
686 
687 /**
688  * amdgpu_vram_mgr_free_sgt - allocate and fill a sg table
689  *
690  * @dev: device pointer
691  * @dir: data direction of resource to unmap
692  * @sgt: sg table to free
693  *
694  * Free a previously allocate sg table.
695  */
696 void amdgpu_vram_mgr_free_sgt(struct device *dev,
697 			      enum dma_data_direction dir,
698 			      struct sg_table *sgt)
699 {
700 	struct scatterlist *sg;
701 	int i;
702 
703 	for_each_sgtable_sg(sgt, sg, i)
704 		dma_unmap_resource(dev, sg->dma_address,
705 				   sg->length, dir,
706 				   DMA_ATTR_SKIP_CPU_SYNC);
707 	sg_free_table(sgt);
708 	kfree(sgt);
709 }
710 
711 /**
712  * amdgpu_vram_mgr_vis_usage - how many bytes are used in the visible part
713  *
714  * @mgr: amdgpu_vram_mgr pointer
715  *
716  * Returns how many bytes are used in the visible part of VRAM
717  */
718 uint64_t amdgpu_vram_mgr_vis_usage(struct amdgpu_vram_mgr *mgr)
719 {
720 	return atomic64_read(&mgr->vis_usage);
721 }
722 
723 /**
724  * amdgpu_vram_mgr_intersects - test each drm buddy block for intersection
725  *
726  * @man: TTM memory type manager
727  * @res: The resource to test
728  * @place: The place to test against
729  * @size: Size of the new allocation
730  *
731  * Test each drm buddy block for intersection for eviction decision.
732  */
733 static bool amdgpu_vram_mgr_intersects(struct ttm_resource_manager *man,
734 				       struct ttm_resource *res,
735 				       const struct ttm_place *place,
736 				       size_t size)
737 {
738 	struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
739 	struct drm_buddy_block *block;
740 
741 	/* Check each drm buddy block individually */
742 	list_for_each_entry(block, &mgr->blocks, link) {
743 		unsigned long fpfn =
744 			amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
745 		unsigned long lpfn = fpfn +
746 			(amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
747 
748 		if (place->fpfn < lpfn &&
749 		    (!place->lpfn || place->lpfn > fpfn))
750 			return true;
751 	}
752 
753 	return false;
754 }
755 
756 /**
757  * amdgpu_vram_mgr_compatible - test each drm buddy block for compatibility
758  *
759  * @man: TTM memory type manager
760  * @res: The resource to test
761  * @place: The place to test against
762  * @size: Size of the new allocation
763  *
764  * Test each drm buddy block for placement compatibility.
765  */
766 static bool amdgpu_vram_mgr_compatible(struct ttm_resource_manager *man,
767 				       struct ttm_resource *res,
768 				       const struct ttm_place *place,
769 				       size_t size)
770 {
771 	struct amdgpu_vram_mgr_resource *mgr = to_amdgpu_vram_mgr_resource(res);
772 	struct drm_buddy_block *block;
773 
774 	/* Check each drm buddy block individually */
775 	list_for_each_entry(block, &mgr->blocks, link) {
776 		unsigned long fpfn =
777 			amdgpu_vram_mgr_block_start(block) >> PAGE_SHIFT;
778 		unsigned long lpfn = fpfn +
779 			(amdgpu_vram_mgr_block_size(block) >> PAGE_SHIFT);
780 
781 		if (fpfn < place->fpfn ||
782 		    (place->lpfn && lpfn > place->lpfn))
783 			return false;
784 	}
785 
786 	return true;
787 }
788 
789 /**
790  * amdgpu_vram_mgr_debug - dump VRAM table
791  *
792  * @man: TTM memory type manager
793  * @printer: DRM printer to use
794  *
795  * Dump the table content using printk.
796  */
797 static void amdgpu_vram_mgr_debug(struct ttm_resource_manager *man,
798 				  struct drm_printer *printer)
799 {
800 	struct amdgpu_vram_mgr *mgr = to_vram_mgr(man);
801 	struct drm_buddy *mm = &mgr->mm;
802 	struct drm_buddy_block *block;
803 
804 	drm_printf(printer, "  vis usage:%llu\n",
805 		   amdgpu_vram_mgr_vis_usage(mgr));
806 
807 	mutex_lock(&mgr->lock);
808 	drm_printf(printer, "default_page_size: %lluKiB\n",
809 		   mgr->default_page_size >> 10);
810 
811 	drm_buddy_print(mm, printer);
812 
813 	drm_printf(printer, "reserved:\n");
814 	list_for_each_entry(block, &mgr->reserved_pages, link)
815 		drm_buddy_block_print(mm, block, printer);
816 	mutex_unlock(&mgr->lock);
817 }
818 
819 static const struct ttm_resource_manager_func amdgpu_vram_mgr_func = {
820 	.alloc	= amdgpu_vram_mgr_new,
821 	.free	= amdgpu_vram_mgr_del,
822 	.intersects = amdgpu_vram_mgr_intersects,
823 	.compatible = amdgpu_vram_mgr_compatible,
824 	.debug	= amdgpu_vram_mgr_debug
825 };
826 
827 /**
828  * amdgpu_vram_mgr_init - init VRAM manager and DRM MM
829  *
830  * @adev: amdgpu_device pointer
831  *
832  * Allocate and initialize the VRAM manager.
833  */
834 int amdgpu_vram_mgr_init(struct amdgpu_device *adev)
835 {
836 	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
837 	struct ttm_resource_manager *man = &mgr->manager;
838 	int err;
839 
840 	ttm_resource_manager_init(man, &adev->mman.bdev,
841 				  adev->gmc.real_vram_size);
842 
843 	man->func = &amdgpu_vram_mgr_func;
844 
845 	err = drm_buddy_init(&mgr->mm, man->size, PAGE_SIZE);
846 	if (err)
847 		return err;
848 
849 	mutex_init(&mgr->lock);
850 	INIT_LIST_HEAD(&mgr->reservations_pending);
851 	INIT_LIST_HEAD(&mgr->reserved_pages);
852 	mgr->default_page_size = PAGE_SIZE;
853 
854 	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, &mgr->manager);
855 	ttm_resource_manager_set_used(man, true);
856 	return 0;
857 }
858 
859 /**
860  * amdgpu_vram_mgr_fini - free and destroy VRAM manager
861  *
862  * @adev: amdgpu_device pointer
863  *
864  * Destroy and free the VRAM manager, returns -EBUSY if ranges are still
865  * allocated inside it.
866  */
867 void amdgpu_vram_mgr_fini(struct amdgpu_device *adev)
868 {
869 	struct amdgpu_vram_mgr *mgr = &adev->mman.vram_mgr;
870 	struct ttm_resource_manager *man = &mgr->manager;
871 	int ret;
872 	struct amdgpu_vram_reservation *rsv, *temp;
873 
874 	ttm_resource_manager_set_used(man, false);
875 
876 	ret = ttm_resource_manager_evict_all(&adev->mman.bdev, man);
877 	if (ret)
878 		return;
879 
880 	mutex_lock(&mgr->lock);
881 	list_for_each_entry_safe(rsv, temp, &mgr->reservations_pending, blocks)
882 		kfree(rsv);
883 
884 	list_for_each_entry_safe(rsv, temp, &mgr->reserved_pages, blocks) {
885 		drm_buddy_free_list(&mgr->mm, &rsv->blocks);
886 		kfree(rsv);
887 	}
888 	drm_buddy_fini(&mgr->mm);
889 	mutex_unlock(&mgr->lock);
890 
891 	ttm_resource_manager_cleanup(man);
892 	ttm_set_driver_manager(&adev->mman.bdev, TTM_PL_VRAM, NULL);
893 }
894