1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Copyright (C) 2023 Loongson Technology Corporation Limited
4  */
5 
6 #include <drm/drm_drv.h>
7 #include <drm/drm_file.h>
8 #include <drm/drm_gem.h>
9 #include <drm/drm_managed.h>
10 #include <drm/drm_prime.h>
11 
12 #include "lsdc_drv.h"
13 #include "lsdc_ttm.h"
14 
15 const char *lsdc_mem_type_to_str(uint32_t mem_type)
16 {
17 	switch (mem_type) {
18 	case TTM_PL_VRAM:
19 		return "VRAM";
20 	case TTM_PL_TT:
21 		return "GTT";
22 	case TTM_PL_SYSTEM:
23 		return "SYSTEM";
24 	default:
25 		break;
26 	}
27 
28 	return "Unknown";
29 }
30 
31 const char *lsdc_domain_to_str(u32 domain)
32 {
33 	switch (domain) {
34 	case LSDC_GEM_DOMAIN_VRAM:
35 		return "VRAM";
36 	case LSDC_GEM_DOMAIN_GTT:
37 		return "GTT";
38 	case LSDC_GEM_DOMAIN_SYSTEM:
39 		return "SYSTEM";
40 	default:
41 		break;
42 	}
43 
44 	return "Unknown";
45 }
46 
47 static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain)
48 {
49 	u32 c = 0;
50 	u32 pflags = 0;
51 	u32 i;
52 
53 	if (lbo->tbo.base.size <= PAGE_SIZE)
54 		pflags |= TTM_PL_FLAG_TOPDOWN;
55 
56 	lbo->placement.placement = lbo->placements;
57 	lbo->placement.busy_placement = lbo->placements;
58 
59 	if (domain & LSDC_GEM_DOMAIN_VRAM) {
60 		lbo->placements[c].mem_type = TTM_PL_VRAM;
61 		lbo->placements[c++].flags = pflags;
62 	}
63 
64 	if (domain & LSDC_GEM_DOMAIN_GTT) {
65 		lbo->placements[c].mem_type = TTM_PL_TT;
66 		lbo->placements[c++].flags = pflags;
67 	}
68 
69 	if (domain & LSDC_GEM_DOMAIN_SYSTEM) {
70 		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
71 		lbo->placements[c++].flags = 0;
72 	}
73 
74 	if (!c) {
75 		lbo->placements[c].mem_type = TTM_PL_SYSTEM;
76 		lbo->placements[c++].flags = 0;
77 	}
78 
79 	lbo->placement.num_placement = c;
80 	lbo->placement.num_busy_placement = c;
81 
82 	for (i = 0; i < c; ++i) {
83 		lbo->placements[i].fpfn = 0;
84 		lbo->placements[i].lpfn = 0;
85 	}
86 }
87 
88 static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt)
89 {
90 	ttm_tt_fini(tt);
91 	kfree(tt);
92 }
93 
94 static struct ttm_tt *
95 lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags)
96 {
97 	struct ttm_tt *tt;
98 	int ret;
99 
100 	tt = kzalloc(sizeof(*tt), GFP_KERNEL);
101 	if (!tt)
102 		return NULL;
103 
104 	ret = ttm_sg_tt_init(tt, tbo, page_flags, ttm_cached);
105 	if (ret < 0) {
106 		kfree(tt);
107 		return NULL;
108 	}
109 
110 	return tt;
111 }
112 
113 static int lsdc_ttm_tt_populate(struct ttm_device *bdev,
114 				struct ttm_tt *ttm,
115 				struct ttm_operation_ctx *ctx)
116 {
117 	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
118 
119 	if (slave && ttm->sg) {
120 		drm_prime_sg_to_dma_addr_array(ttm->sg,
121 					       ttm->dma_address,
122 					       ttm->num_pages);
123 
124 		return 0;
125 	}
126 
127 	return ttm_pool_alloc(&bdev->pool, ttm, ctx);
128 }
129 
130 static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev,
131 				   struct ttm_tt *ttm)
132 {
133 	bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
134 
135 	if (slave)
136 		return;
137 
138 	return ttm_pool_free(&bdev->pool, ttm);
139 }
140 
141 static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo,
142 				struct ttm_placement *tplacement)
143 {
144 	struct ttm_resource *resource = tbo->resource;
145 	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
146 
147 	switch (resource->mem_type) {
148 	case TTM_PL_VRAM:
149 		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT);
150 		break;
151 	case TTM_PL_TT:
152 	default:
153 		lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM);
154 		break;
155 	}
156 
157 	*tplacement = lbo->placement;
158 }
159 
160 static int lsdc_bo_move(struct ttm_buffer_object *tbo,
161 			bool evict,
162 			struct ttm_operation_ctx *ctx,
163 			struct ttm_resource *new_mem,
164 			struct ttm_place *hop)
165 {
166 	struct drm_device *ddev = tbo->base.dev;
167 	struct ttm_resource *old_mem = tbo->resource;
168 	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
169 	int ret;
170 
171 	if (unlikely(tbo->pin_count > 0)) {
172 		drm_warn(ddev, "Can't move a pinned BO\n");
173 		return -EINVAL;
174 	}
175 
176 	ret = ttm_bo_wait_ctx(tbo, ctx);
177 	if (ret)
178 		return ret;
179 
180 	if (!old_mem) {
181 		drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n",
182 			lbo, lsdc_mem_type_to_str(new_mem->mem_type),
183 			lsdc_bo_size(lbo));
184 		ttm_bo_move_null(tbo, new_mem);
185 		return 0;
186 	}
187 
188 	if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) {
189 		ttm_bo_move_null(tbo, new_mem);
190 		drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n",
191 			lbo, lsdc_bo_size(lbo));
192 		return 0;
193 	}
194 
195 	if (old_mem->mem_type == TTM_PL_SYSTEM &&
196 	    new_mem->mem_type == TTM_PL_TT) {
197 		drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n",
198 			lbo, lsdc_bo_size(lbo));
199 		ttm_bo_move_null(tbo, new_mem);
200 		return 0;
201 	}
202 
203 	if (old_mem->mem_type == TTM_PL_TT &&
204 	    new_mem->mem_type == TTM_PL_SYSTEM) {
205 		drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n",
206 			lbo, lsdc_bo_size(lbo));
207 		ttm_resource_free(tbo, &tbo->resource);
208 		ttm_bo_assign_mem(tbo, new_mem);
209 		return 0;
210 	}
211 
212 	drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n",
213 		lbo,
214 		lsdc_mem_type_to_str(old_mem->mem_type),
215 		lsdc_mem_type_to_str(new_mem->mem_type),
216 		lsdc_bo_size(lbo));
217 
218 	return ttm_bo_move_memcpy(tbo, ctx, new_mem);
219 }
220 
221 static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev,
222 				  struct ttm_resource *mem)
223 {
224 	struct lsdc_device *ldev = tdev_to_ldev(bdev);
225 
226 	switch (mem->mem_type) {
227 	case TTM_PL_SYSTEM:
228 		break;
229 	case TTM_PL_TT:
230 		break;
231 	case TTM_PL_VRAM:
232 		mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base;
233 		mem->bus.is_iomem = true;
234 		mem->bus.caching = ttm_write_combined;
235 		break;
236 	default:
237 		return -EINVAL;
238 	}
239 
240 	return 0;
241 }
242 
243 static struct ttm_device_funcs lsdc_bo_driver = {
244 	.ttm_tt_create = lsdc_ttm_tt_create,
245 	.ttm_tt_populate = lsdc_ttm_tt_populate,
246 	.ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate,
247 	.ttm_tt_destroy = lsdc_ttm_tt_destroy,
248 	.eviction_valuable = ttm_bo_eviction_valuable,
249 	.evict_flags = lsdc_bo_evict_flags,
250 	.move = lsdc_bo_move,
251 	.io_mem_reserve = lsdc_bo_reserve_io_mem,
252 };
253 
254 u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo)
255 {
256 	struct ttm_buffer_object *tbo = &lbo->tbo;
257 	struct drm_device *ddev = tbo->base.dev;
258 	struct ttm_resource *resource = tbo->resource;
259 
260 	if (unlikely(!tbo->pin_count)) {
261 		drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n");
262 		return 0;
263 	}
264 
265 	if (unlikely(resource->mem_type == TTM_PL_SYSTEM))
266 		return 0;
267 
268 	return resource->start << PAGE_SHIFT;
269 }
270 
271 size_t lsdc_bo_size(struct lsdc_bo *lbo)
272 {
273 	struct ttm_buffer_object *tbo = &lbo->tbo;
274 
275 	return tbo->base.size;
276 }
277 
278 int lsdc_bo_reserve(struct lsdc_bo *lbo)
279 {
280 	return ttm_bo_reserve(&lbo->tbo, true, false, NULL);
281 }
282 
283 void lsdc_bo_unreserve(struct lsdc_bo *lbo)
284 {
285 	return ttm_bo_unreserve(&lbo->tbo);
286 }
287 
288 int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr)
289 {
290 	struct ttm_operation_ctx ctx = { false, false };
291 	struct ttm_buffer_object *tbo = &lbo->tbo;
292 	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
293 	int ret;
294 
295 	if (tbo->pin_count)
296 		goto bo_pinned;
297 
298 	if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM)
299 		return -EINVAL;
300 
301 	if (domain)
302 		lsdc_bo_set_placement(lbo, domain);
303 
304 	ret = ttm_bo_validate(tbo, &lbo->placement, &ctx);
305 	if (unlikely(ret)) {
306 		drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret);
307 		return ret;
308 	}
309 
310 	if (domain == LSDC_GEM_DOMAIN_VRAM)
311 		ldev->vram_pinned_size += lsdc_bo_size(lbo);
312 	else if (domain == LSDC_GEM_DOMAIN_GTT)
313 		ldev->gtt_pinned_size += lsdc_bo_size(lbo);
314 
315 bo_pinned:
316 	ttm_bo_pin(tbo);
317 
318 	if (gpu_addr)
319 		*gpu_addr = lsdc_bo_gpu_offset(lbo);
320 
321 	return 0;
322 }
323 
324 void lsdc_bo_unpin(struct lsdc_bo *lbo)
325 {
326 	struct ttm_buffer_object *tbo = &lbo->tbo;
327 	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
328 
329 	if (unlikely(!tbo->pin_count)) {
330 		drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo);
331 		return;
332 	}
333 
334 	ttm_bo_unpin(tbo);
335 
336 	if (!tbo->pin_count) {
337 		if (tbo->resource->mem_type == TTM_PL_VRAM)
338 			ldev->vram_pinned_size -= lsdc_bo_size(lbo);
339 		else if (tbo->resource->mem_type == TTM_PL_TT)
340 			ldev->gtt_pinned_size -= lsdc_bo_size(lbo);
341 	}
342 }
343 
344 void lsdc_bo_ref(struct lsdc_bo *lbo)
345 {
346 	struct ttm_buffer_object *tbo = &lbo->tbo;
347 
348 	ttm_bo_get(tbo);
349 }
350 
351 void lsdc_bo_unref(struct lsdc_bo *lbo)
352 {
353 	struct ttm_buffer_object *tbo = &lbo->tbo;
354 
355 	ttm_bo_put(tbo);
356 }
357 
358 int lsdc_bo_kmap(struct lsdc_bo *lbo)
359 {
360 	struct ttm_buffer_object *tbo = &lbo->tbo;
361 	struct drm_gem_object *gem = &tbo->base;
362 	struct drm_device *ddev = gem->dev;
363 	long ret;
364 	int err;
365 
366 	ret = dma_resv_wait_timeout(gem->resv, DMA_RESV_USAGE_KERNEL, false,
367 				    MAX_SCHEDULE_TIMEOUT);
368 	if (ret < 0) {
369 		drm_warn(ddev, "wait fence timeout\n");
370 		return ret;
371 	}
372 
373 	if (lbo->kptr)
374 		return 0;
375 
376 	err = ttm_bo_kmap(tbo, 0, PFN_UP(lsdc_bo_size(lbo)), &lbo->kmap);
377 	if (err) {
378 		drm_err(ddev, "kmap %p failed: %d\n", lbo, err);
379 		return err;
380 	}
381 
382 	lbo->kptr = ttm_kmap_obj_virtual(&lbo->kmap, &lbo->is_iomem);
383 
384 	return 0;
385 }
386 
387 void lsdc_bo_kunmap(struct lsdc_bo *lbo)
388 {
389 	if (!lbo->kptr)
390 		return;
391 
392 	lbo->kptr = NULL;
393 	ttm_bo_kunmap(&lbo->kmap);
394 }
395 
396 void lsdc_bo_clear(struct lsdc_bo *lbo)
397 {
398 	lsdc_bo_kmap(lbo);
399 
400 	if (lbo->is_iomem)
401 		memset_io((void __iomem *)lbo->kptr, 0, lbo->size);
402 	else
403 		memset(lbo->kptr, 0, lbo->size);
404 
405 	lsdc_bo_kunmap(lbo);
406 }
407 
408 int lsdc_bo_evict_vram(struct drm_device *ddev)
409 {
410 	struct lsdc_device *ldev = to_lsdc(ddev);
411 	struct ttm_device *bdev = &ldev->bdev;
412 	struct ttm_resource_manager *man;
413 
414 	man = ttm_manager_type(bdev, TTM_PL_VRAM);
415 	if (unlikely(!man))
416 		return 0;
417 
418 	return ttm_resource_manager_evict_all(bdev, man);
419 }
420 
421 static void lsdc_bo_destroy(struct ttm_buffer_object *tbo)
422 {
423 	struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
424 	struct lsdc_bo *lbo = to_lsdc_bo(tbo);
425 
426 	mutex_lock(&ldev->gem.mutex);
427 	list_del_init(&lbo->list);
428 	mutex_unlock(&ldev->gem.mutex);
429 
430 	drm_gem_object_release(&tbo->base);
431 
432 	kfree(lbo);
433 }
434 
435 struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev,
436 			       u32 domain,
437 			       size_t size,
438 			       bool kernel,
439 			       struct sg_table *sg,
440 			       struct dma_resv *resv)
441 {
442 	struct lsdc_device *ldev = to_lsdc(ddev);
443 	struct ttm_device *bdev = &ldev->bdev;
444 	struct ttm_buffer_object *tbo;
445 	struct lsdc_bo *lbo;
446 	enum ttm_bo_type bo_type;
447 	int ret;
448 
449 	lbo = kzalloc(sizeof(*lbo), GFP_KERNEL);
450 	if (!lbo)
451 		return ERR_PTR(-ENOMEM);
452 
453 	INIT_LIST_HEAD(&lbo->list);
454 
455 	lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM |
456 					LSDC_GEM_DOMAIN_GTT |
457 					LSDC_GEM_DOMAIN_SYSTEM);
458 
459 	tbo = &lbo->tbo;
460 
461 	size = ALIGN(size, PAGE_SIZE);
462 
463 	ret = drm_gem_object_init(ddev, &tbo->base, size);
464 	if (ret) {
465 		kfree(lbo);
466 		return ERR_PTR(ret);
467 	}
468 
469 	tbo->bdev = bdev;
470 
471 	if (kernel)
472 		bo_type = ttm_bo_type_kernel;
473 	else if (sg)
474 		bo_type = ttm_bo_type_sg;
475 	else
476 		bo_type = ttm_bo_type_device;
477 
478 	lsdc_bo_set_placement(lbo, domain);
479 	lbo->size = size;
480 
481 	ret = ttm_bo_init_validate(bdev, tbo, bo_type, &lbo->placement, 0,
482 				   false, sg, resv, lsdc_bo_destroy);
483 	if (ret) {
484 		kfree(lbo);
485 		return ERR_PTR(ret);
486 	}
487 
488 	return lbo;
489 }
490 
491 struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev,
492 					     u32 domain,
493 					     size_t size)
494 {
495 	struct lsdc_bo *lbo;
496 	int ret;
497 
498 	lbo = lsdc_bo_create(ddev, domain, size, true, NULL, NULL);
499 	if (IS_ERR(lbo))
500 		return ERR_CAST(lbo);
501 
502 	ret = lsdc_bo_reserve(lbo);
503 	if (unlikely(ret)) {
504 		lsdc_bo_unref(lbo);
505 		return ERR_PTR(ret);
506 	}
507 
508 	ret = lsdc_bo_pin(lbo, domain, NULL);
509 	lsdc_bo_unreserve(lbo);
510 	if (unlikely(ret)) {
511 		lsdc_bo_unref(lbo);
512 		return ERR_PTR(ret);
513 	}
514 
515 	return lbo;
516 }
517 
518 void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo)
519 {
520 	int ret;
521 
522 	ret = lsdc_bo_reserve(lbo);
523 	if (unlikely(ret))
524 		return;
525 
526 	lsdc_bo_unpin(lbo);
527 	lsdc_bo_unreserve(lbo);
528 
529 	lsdc_bo_unref(lbo);
530 }
531 
532 static void lsdc_ttm_fini(struct drm_device *ddev, void *data)
533 {
534 	struct lsdc_device *ldev = (struct lsdc_device *)data;
535 
536 	ttm_range_man_fini(&ldev->bdev, TTM_PL_VRAM);
537 	ttm_range_man_fini(&ldev->bdev, TTM_PL_TT);
538 
539 	ttm_device_fini(&ldev->bdev);
540 
541 	drm_dbg(ddev, "ttm finished\n");
542 }
543 
544 int lsdc_ttm_init(struct lsdc_device *ldev)
545 {
546 	struct drm_device *ddev = &ldev->base;
547 	unsigned long num_vram_pages;
548 	unsigned long num_gtt_pages;
549 	int ret;
550 
551 	ret = ttm_device_init(&ldev->bdev, &lsdc_bo_driver, ddev->dev,
552 			      ddev->anon_inode->i_mapping,
553 			      ddev->vma_offset_manager, false, true);
554 	if (ret)
555 		return ret;
556 
557 	num_vram_pages = ldev->vram_size >> PAGE_SHIFT;
558 
559 	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_VRAM, false, num_vram_pages);
560 	if (unlikely(ret))
561 		return ret;
562 
563 	drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages);
564 
565 	/* 512M is far enough for us now */
566 	ldev->gtt_size = 512 << 20;
567 
568 	num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT;
569 
570 	ret = ttm_range_man_init(&ldev->bdev, TTM_PL_TT, true, num_gtt_pages);
571 	if (unlikely(ret))
572 		return ret;
573 
574 	drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages);
575 
576 	return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev);
577 }
578 
579 void lsdc_ttm_debugfs_init(struct lsdc_device *ldev)
580 {
581 	struct ttm_device *bdev = &ldev->bdev;
582 	struct drm_device *ddev = &ldev->base;
583 	struct drm_minor *minor = ddev->primary;
584 	struct dentry *root = minor->debugfs_root;
585 	struct ttm_resource_manager *vram_man;
586 	struct ttm_resource_manager *gtt_man;
587 
588 	vram_man = ttm_manager_type(bdev, TTM_PL_VRAM);
589 	gtt_man = ttm_manager_type(bdev, TTM_PL_TT);
590 
591 	ttm_resource_manager_create_debugfs(vram_man, root, "vram_mm");
592 	ttm_resource_manager_create_debugfs(gtt_man, root, "gtt_mm");
593 }
594