1f39db26cSSui Jingfeng // SPDX-License-Identifier: GPL-2.0+
2f39db26cSSui Jingfeng /*
3f39db26cSSui Jingfeng * Copyright (C) 2023 Loongson Technology Corporation Limited
4f39db26cSSui Jingfeng */
5f39db26cSSui Jingfeng
6f39db26cSSui Jingfeng #include <drm/drm_drv.h>
7f39db26cSSui Jingfeng #include <drm/drm_file.h>
8f39db26cSSui Jingfeng #include <drm/drm_gem.h>
9f39db26cSSui Jingfeng #include <drm/drm_managed.h>
10f39db26cSSui Jingfeng #include <drm/drm_prime.h>
11f39db26cSSui Jingfeng
12f39db26cSSui Jingfeng #include "lsdc_drv.h"
13f39db26cSSui Jingfeng #include "lsdc_ttm.h"
14f39db26cSSui Jingfeng
lsdc_mem_type_to_str(uint32_t mem_type)15f39db26cSSui Jingfeng const char *lsdc_mem_type_to_str(uint32_t mem_type)
16f39db26cSSui Jingfeng {
17f39db26cSSui Jingfeng switch (mem_type) {
18f39db26cSSui Jingfeng case TTM_PL_VRAM:
19f39db26cSSui Jingfeng return "VRAM";
20f39db26cSSui Jingfeng case TTM_PL_TT:
21f39db26cSSui Jingfeng return "GTT";
22f39db26cSSui Jingfeng case TTM_PL_SYSTEM:
23f39db26cSSui Jingfeng return "SYSTEM";
24f39db26cSSui Jingfeng default:
25f39db26cSSui Jingfeng break;
26f39db26cSSui Jingfeng }
27f39db26cSSui Jingfeng
28f39db26cSSui Jingfeng return "Unknown";
29f39db26cSSui Jingfeng }
30f39db26cSSui Jingfeng
lsdc_domain_to_str(u32 domain)31f39db26cSSui Jingfeng const char *lsdc_domain_to_str(u32 domain)
32f39db26cSSui Jingfeng {
33f39db26cSSui Jingfeng switch (domain) {
34f39db26cSSui Jingfeng case LSDC_GEM_DOMAIN_VRAM:
35f39db26cSSui Jingfeng return "VRAM";
36f39db26cSSui Jingfeng case LSDC_GEM_DOMAIN_GTT:
37f39db26cSSui Jingfeng return "GTT";
38f39db26cSSui Jingfeng case LSDC_GEM_DOMAIN_SYSTEM:
39f39db26cSSui Jingfeng return "SYSTEM";
40f39db26cSSui Jingfeng default:
41f39db26cSSui Jingfeng break;
42f39db26cSSui Jingfeng }
43f39db26cSSui Jingfeng
44f39db26cSSui Jingfeng return "Unknown";
45f39db26cSSui Jingfeng }
46f39db26cSSui Jingfeng
lsdc_bo_set_placement(struct lsdc_bo * lbo,u32 domain)47f39db26cSSui Jingfeng static void lsdc_bo_set_placement(struct lsdc_bo *lbo, u32 domain)
48f39db26cSSui Jingfeng {
49f39db26cSSui Jingfeng u32 c = 0;
50f39db26cSSui Jingfeng u32 pflags = 0;
51f39db26cSSui Jingfeng u32 i;
52f39db26cSSui Jingfeng
53f39db26cSSui Jingfeng if (lbo->tbo.base.size <= PAGE_SIZE)
54f39db26cSSui Jingfeng pflags |= TTM_PL_FLAG_TOPDOWN;
55f39db26cSSui Jingfeng
56f39db26cSSui Jingfeng lbo->placement.placement = lbo->placements;
57f39db26cSSui Jingfeng lbo->placement.busy_placement = lbo->placements;
58f39db26cSSui Jingfeng
59f39db26cSSui Jingfeng if (domain & LSDC_GEM_DOMAIN_VRAM) {
60f39db26cSSui Jingfeng lbo->placements[c].mem_type = TTM_PL_VRAM;
61f39db26cSSui Jingfeng lbo->placements[c++].flags = pflags;
62f39db26cSSui Jingfeng }
63f39db26cSSui Jingfeng
64f39db26cSSui Jingfeng if (domain & LSDC_GEM_DOMAIN_GTT) {
65f39db26cSSui Jingfeng lbo->placements[c].mem_type = TTM_PL_TT;
66f39db26cSSui Jingfeng lbo->placements[c++].flags = pflags;
67f39db26cSSui Jingfeng }
68f39db26cSSui Jingfeng
69f39db26cSSui Jingfeng if (domain & LSDC_GEM_DOMAIN_SYSTEM) {
70f39db26cSSui Jingfeng lbo->placements[c].mem_type = TTM_PL_SYSTEM;
71f39db26cSSui Jingfeng lbo->placements[c++].flags = 0;
72f39db26cSSui Jingfeng }
73f39db26cSSui Jingfeng
74f39db26cSSui Jingfeng if (!c) {
75f39db26cSSui Jingfeng lbo->placements[c].mem_type = TTM_PL_SYSTEM;
76f39db26cSSui Jingfeng lbo->placements[c++].flags = 0;
77f39db26cSSui Jingfeng }
78f39db26cSSui Jingfeng
79f39db26cSSui Jingfeng lbo->placement.num_placement = c;
80f39db26cSSui Jingfeng lbo->placement.num_busy_placement = c;
81f39db26cSSui Jingfeng
82f39db26cSSui Jingfeng for (i = 0; i < c; ++i) {
83f39db26cSSui Jingfeng lbo->placements[i].fpfn = 0;
84f39db26cSSui Jingfeng lbo->placements[i].lpfn = 0;
85f39db26cSSui Jingfeng }
86f39db26cSSui Jingfeng }
87f39db26cSSui Jingfeng
lsdc_ttm_tt_destroy(struct ttm_device * bdev,struct ttm_tt * tt)88f39db26cSSui Jingfeng static void lsdc_ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *tt)
89f39db26cSSui Jingfeng {
90f39db26cSSui Jingfeng ttm_tt_fini(tt);
91f39db26cSSui Jingfeng kfree(tt);
92f39db26cSSui Jingfeng }
93f39db26cSSui Jingfeng
94f39db26cSSui Jingfeng static struct ttm_tt *
lsdc_ttm_tt_create(struct ttm_buffer_object * tbo,uint32_t page_flags)95f39db26cSSui Jingfeng lsdc_ttm_tt_create(struct ttm_buffer_object *tbo, uint32_t page_flags)
96f39db26cSSui Jingfeng {
97f39db26cSSui Jingfeng struct ttm_tt *tt;
98f39db26cSSui Jingfeng int ret;
99f39db26cSSui Jingfeng
100f39db26cSSui Jingfeng tt = kzalloc(sizeof(*tt), GFP_KERNEL);
101f39db26cSSui Jingfeng if (!tt)
102f39db26cSSui Jingfeng return NULL;
103f39db26cSSui Jingfeng
104f39db26cSSui Jingfeng ret = ttm_sg_tt_init(tt, tbo, page_flags, ttm_cached);
105f39db26cSSui Jingfeng if (ret < 0) {
106f39db26cSSui Jingfeng kfree(tt);
107f39db26cSSui Jingfeng return NULL;
108f39db26cSSui Jingfeng }
109f39db26cSSui Jingfeng
110f39db26cSSui Jingfeng return tt;
111f39db26cSSui Jingfeng }
112f39db26cSSui Jingfeng
lsdc_ttm_tt_populate(struct ttm_device * bdev,struct ttm_tt * ttm,struct ttm_operation_ctx * ctx)113f39db26cSSui Jingfeng static int lsdc_ttm_tt_populate(struct ttm_device *bdev,
114f39db26cSSui Jingfeng struct ttm_tt *ttm,
115f39db26cSSui Jingfeng struct ttm_operation_ctx *ctx)
116f39db26cSSui Jingfeng {
117f39db26cSSui Jingfeng bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
118f39db26cSSui Jingfeng
119f39db26cSSui Jingfeng if (slave && ttm->sg) {
120f39db26cSSui Jingfeng drm_prime_sg_to_dma_addr_array(ttm->sg,
121f39db26cSSui Jingfeng ttm->dma_address,
122f39db26cSSui Jingfeng ttm->num_pages);
123f39db26cSSui Jingfeng
124f39db26cSSui Jingfeng return 0;
125f39db26cSSui Jingfeng }
126f39db26cSSui Jingfeng
127f39db26cSSui Jingfeng return ttm_pool_alloc(&bdev->pool, ttm, ctx);
128f39db26cSSui Jingfeng }
129f39db26cSSui Jingfeng
lsdc_ttm_tt_unpopulate(struct ttm_device * bdev,struct ttm_tt * ttm)130f39db26cSSui Jingfeng static void lsdc_ttm_tt_unpopulate(struct ttm_device *bdev,
131f39db26cSSui Jingfeng struct ttm_tt *ttm)
132f39db26cSSui Jingfeng {
133f39db26cSSui Jingfeng bool slave = !!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL);
134f39db26cSSui Jingfeng
135f39db26cSSui Jingfeng if (slave)
136f39db26cSSui Jingfeng return;
137f39db26cSSui Jingfeng
138f39db26cSSui Jingfeng return ttm_pool_free(&bdev->pool, ttm);
139f39db26cSSui Jingfeng }
140f39db26cSSui Jingfeng
lsdc_bo_evict_flags(struct ttm_buffer_object * tbo,struct ttm_placement * tplacement)141f39db26cSSui Jingfeng static void lsdc_bo_evict_flags(struct ttm_buffer_object *tbo,
142f39db26cSSui Jingfeng struct ttm_placement *tplacement)
143f39db26cSSui Jingfeng {
144f39db26cSSui Jingfeng struct ttm_resource *resource = tbo->resource;
145f39db26cSSui Jingfeng struct lsdc_bo *lbo = to_lsdc_bo(tbo);
146f39db26cSSui Jingfeng
147f39db26cSSui Jingfeng switch (resource->mem_type) {
148f39db26cSSui Jingfeng case TTM_PL_VRAM:
149f39db26cSSui Jingfeng lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_GTT);
150f39db26cSSui Jingfeng break;
151f39db26cSSui Jingfeng case TTM_PL_TT:
152f39db26cSSui Jingfeng default:
153f39db26cSSui Jingfeng lsdc_bo_set_placement(lbo, LSDC_GEM_DOMAIN_SYSTEM);
154f39db26cSSui Jingfeng break;
155f39db26cSSui Jingfeng }
156f39db26cSSui Jingfeng
157f39db26cSSui Jingfeng *tplacement = lbo->placement;
158f39db26cSSui Jingfeng }
159f39db26cSSui Jingfeng
lsdc_bo_move(struct ttm_buffer_object * tbo,bool evict,struct ttm_operation_ctx * ctx,struct ttm_resource * new_mem,struct ttm_place * hop)160f39db26cSSui Jingfeng static int lsdc_bo_move(struct ttm_buffer_object *tbo,
161f39db26cSSui Jingfeng bool evict,
162f39db26cSSui Jingfeng struct ttm_operation_ctx *ctx,
163f39db26cSSui Jingfeng struct ttm_resource *new_mem,
164f39db26cSSui Jingfeng struct ttm_place *hop)
165f39db26cSSui Jingfeng {
166f39db26cSSui Jingfeng struct drm_device *ddev = tbo->base.dev;
167f39db26cSSui Jingfeng struct ttm_resource *old_mem = tbo->resource;
168f39db26cSSui Jingfeng struct lsdc_bo *lbo = to_lsdc_bo(tbo);
169f39db26cSSui Jingfeng int ret;
170f39db26cSSui Jingfeng
171f39db26cSSui Jingfeng if (unlikely(tbo->pin_count > 0)) {
172f39db26cSSui Jingfeng drm_warn(ddev, "Can't move a pinned BO\n");
173f39db26cSSui Jingfeng return -EINVAL;
174f39db26cSSui Jingfeng }
175f39db26cSSui Jingfeng
176f39db26cSSui Jingfeng ret = ttm_bo_wait_ctx(tbo, ctx);
177f39db26cSSui Jingfeng if (ret)
178f39db26cSSui Jingfeng return ret;
179f39db26cSSui Jingfeng
180f39db26cSSui Jingfeng if (!old_mem) {
181f39db26cSSui Jingfeng drm_dbg(ddev, "bo[%p] move: NULL to %s, size: %zu\n",
182f39db26cSSui Jingfeng lbo, lsdc_mem_type_to_str(new_mem->mem_type),
183f39db26cSSui Jingfeng lsdc_bo_size(lbo));
184f39db26cSSui Jingfeng ttm_bo_move_null(tbo, new_mem);
185f39db26cSSui Jingfeng return 0;
186f39db26cSSui Jingfeng }
187f39db26cSSui Jingfeng
188f39db26cSSui Jingfeng if (old_mem->mem_type == TTM_PL_SYSTEM && !tbo->ttm) {
189f39db26cSSui Jingfeng ttm_bo_move_null(tbo, new_mem);
190f39db26cSSui Jingfeng drm_dbg(ddev, "bo[%p] move: SYSTEM to NULL, size: %zu\n",
191f39db26cSSui Jingfeng lbo, lsdc_bo_size(lbo));
192f39db26cSSui Jingfeng return 0;
193f39db26cSSui Jingfeng }
194f39db26cSSui Jingfeng
195f39db26cSSui Jingfeng if (old_mem->mem_type == TTM_PL_SYSTEM &&
196f39db26cSSui Jingfeng new_mem->mem_type == TTM_PL_TT) {
197f39db26cSSui Jingfeng drm_dbg(ddev, "bo[%p] move: SYSTEM to GTT, size: %zu\n",
198f39db26cSSui Jingfeng lbo, lsdc_bo_size(lbo));
199f39db26cSSui Jingfeng ttm_bo_move_null(tbo, new_mem);
200f39db26cSSui Jingfeng return 0;
201f39db26cSSui Jingfeng }
202f39db26cSSui Jingfeng
203f39db26cSSui Jingfeng if (old_mem->mem_type == TTM_PL_TT &&
204f39db26cSSui Jingfeng new_mem->mem_type == TTM_PL_SYSTEM) {
205f39db26cSSui Jingfeng drm_dbg(ddev, "bo[%p] move: GTT to SYSTEM, size: %zu\n",
206f39db26cSSui Jingfeng lbo, lsdc_bo_size(lbo));
207f39db26cSSui Jingfeng ttm_resource_free(tbo, &tbo->resource);
208f39db26cSSui Jingfeng ttm_bo_assign_mem(tbo, new_mem);
209f39db26cSSui Jingfeng return 0;
210f39db26cSSui Jingfeng }
211f39db26cSSui Jingfeng
212f39db26cSSui Jingfeng drm_dbg(ddev, "bo[%p] move: %s to %s, size: %zu\n",
213f39db26cSSui Jingfeng lbo,
214f39db26cSSui Jingfeng lsdc_mem_type_to_str(old_mem->mem_type),
215f39db26cSSui Jingfeng lsdc_mem_type_to_str(new_mem->mem_type),
216f39db26cSSui Jingfeng lsdc_bo_size(lbo));
217f39db26cSSui Jingfeng
218f39db26cSSui Jingfeng return ttm_bo_move_memcpy(tbo, ctx, new_mem);
219f39db26cSSui Jingfeng }
220f39db26cSSui Jingfeng
lsdc_bo_reserve_io_mem(struct ttm_device * bdev,struct ttm_resource * mem)221f39db26cSSui Jingfeng static int lsdc_bo_reserve_io_mem(struct ttm_device *bdev,
222f39db26cSSui Jingfeng struct ttm_resource *mem)
223f39db26cSSui Jingfeng {
224f39db26cSSui Jingfeng struct lsdc_device *ldev = tdev_to_ldev(bdev);
225f39db26cSSui Jingfeng
226f39db26cSSui Jingfeng switch (mem->mem_type) {
227f39db26cSSui Jingfeng case TTM_PL_SYSTEM:
228f39db26cSSui Jingfeng break;
229f39db26cSSui Jingfeng case TTM_PL_TT:
230f39db26cSSui Jingfeng break;
231f39db26cSSui Jingfeng case TTM_PL_VRAM:
232f39db26cSSui Jingfeng mem->bus.offset = (mem->start << PAGE_SHIFT) + ldev->vram_base;
233f39db26cSSui Jingfeng mem->bus.is_iomem = true;
234f39db26cSSui Jingfeng mem->bus.caching = ttm_write_combined;
235f39db26cSSui Jingfeng break;
236f39db26cSSui Jingfeng default:
237f39db26cSSui Jingfeng return -EINVAL;
238f39db26cSSui Jingfeng }
239f39db26cSSui Jingfeng
240f39db26cSSui Jingfeng return 0;
241f39db26cSSui Jingfeng }
242f39db26cSSui Jingfeng
243f39db26cSSui Jingfeng static struct ttm_device_funcs lsdc_bo_driver = {
244f39db26cSSui Jingfeng .ttm_tt_create = lsdc_ttm_tt_create,
245f39db26cSSui Jingfeng .ttm_tt_populate = lsdc_ttm_tt_populate,
246f39db26cSSui Jingfeng .ttm_tt_unpopulate = lsdc_ttm_tt_unpopulate,
247f39db26cSSui Jingfeng .ttm_tt_destroy = lsdc_ttm_tt_destroy,
248f39db26cSSui Jingfeng .eviction_valuable = ttm_bo_eviction_valuable,
249f39db26cSSui Jingfeng .evict_flags = lsdc_bo_evict_flags,
250f39db26cSSui Jingfeng .move = lsdc_bo_move,
251f39db26cSSui Jingfeng .io_mem_reserve = lsdc_bo_reserve_io_mem,
252f39db26cSSui Jingfeng };
253f39db26cSSui Jingfeng
lsdc_bo_gpu_offset(struct lsdc_bo * lbo)254f39db26cSSui Jingfeng u64 lsdc_bo_gpu_offset(struct lsdc_bo *lbo)
255f39db26cSSui Jingfeng {
256f39db26cSSui Jingfeng struct ttm_buffer_object *tbo = &lbo->tbo;
257f39db26cSSui Jingfeng struct drm_device *ddev = tbo->base.dev;
258f39db26cSSui Jingfeng struct ttm_resource *resource = tbo->resource;
259f39db26cSSui Jingfeng
260f39db26cSSui Jingfeng if (unlikely(!tbo->pin_count)) {
261f39db26cSSui Jingfeng drm_err(ddev, "unpinned bo, gpu virtual address is invalid\n");
262f39db26cSSui Jingfeng return 0;
263f39db26cSSui Jingfeng }
264f39db26cSSui Jingfeng
265f39db26cSSui Jingfeng if (unlikely(resource->mem_type == TTM_PL_SYSTEM))
266f39db26cSSui Jingfeng return 0;
267f39db26cSSui Jingfeng
268f39db26cSSui Jingfeng return resource->start << PAGE_SHIFT;
269f39db26cSSui Jingfeng }
270f39db26cSSui Jingfeng
lsdc_bo_size(struct lsdc_bo * lbo)271f39db26cSSui Jingfeng size_t lsdc_bo_size(struct lsdc_bo *lbo)
272f39db26cSSui Jingfeng {
273f39db26cSSui Jingfeng struct ttm_buffer_object *tbo = &lbo->tbo;
274f39db26cSSui Jingfeng
275f39db26cSSui Jingfeng return tbo->base.size;
276f39db26cSSui Jingfeng }
277f39db26cSSui Jingfeng
lsdc_bo_reserve(struct lsdc_bo * lbo)278f39db26cSSui Jingfeng int lsdc_bo_reserve(struct lsdc_bo *lbo)
279f39db26cSSui Jingfeng {
280f39db26cSSui Jingfeng return ttm_bo_reserve(&lbo->tbo, true, false, NULL);
281f39db26cSSui Jingfeng }
282f39db26cSSui Jingfeng
lsdc_bo_unreserve(struct lsdc_bo * lbo)283f39db26cSSui Jingfeng void lsdc_bo_unreserve(struct lsdc_bo *lbo)
284f39db26cSSui Jingfeng {
285f39db26cSSui Jingfeng return ttm_bo_unreserve(&lbo->tbo);
286f39db26cSSui Jingfeng }
287f39db26cSSui Jingfeng
lsdc_bo_pin(struct lsdc_bo * lbo,u32 domain,u64 * gpu_addr)288f39db26cSSui Jingfeng int lsdc_bo_pin(struct lsdc_bo *lbo, u32 domain, u64 *gpu_addr)
289f39db26cSSui Jingfeng {
290f39db26cSSui Jingfeng struct ttm_operation_ctx ctx = { false, false };
291f39db26cSSui Jingfeng struct ttm_buffer_object *tbo = &lbo->tbo;
292f39db26cSSui Jingfeng struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
293f39db26cSSui Jingfeng int ret;
294f39db26cSSui Jingfeng
295f39db26cSSui Jingfeng if (tbo->pin_count)
296f39db26cSSui Jingfeng goto bo_pinned;
297f39db26cSSui Jingfeng
298f39db26cSSui Jingfeng if (lbo->sharing_count && domain == LSDC_GEM_DOMAIN_VRAM)
299f39db26cSSui Jingfeng return -EINVAL;
300f39db26cSSui Jingfeng
301f39db26cSSui Jingfeng if (domain)
302f39db26cSSui Jingfeng lsdc_bo_set_placement(lbo, domain);
303f39db26cSSui Jingfeng
304f39db26cSSui Jingfeng ret = ttm_bo_validate(tbo, &lbo->placement, &ctx);
305f39db26cSSui Jingfeng if (unlikely(ret)) {
306f39db26cSSui Jingfeng drm_err(&ldev->base, "%p validate failed: %d\n", lbo, ret);
307f39db26cSSui Jingfeng return ret;
308f39db26cSSui Jingfeng }
309f39db26cSSui Jingfeng
310f39db26cSSui Jingfeng if (domain == LSDC_GEM_DOMAIN_VRAM)
311f39db26cSSui Jingfeng ldev->vram_pinned_size += lsdc_bo_size(lbo);
312f39db26cSSui Jingfeng else if (domain == LSDC_GEM_DOMAIN_GTT)
313f39db26cSSui Jingfeng ldev->gtt_pinned_size += lsdc_bo_size(lbo);
314f39db26cSSui Jingfeng
315f39db26cSSui Jingfeng bo_pinned:
316f39db26cSSui Jingfeng ttm_bo_pin(tbo);
317f39db26cSSui Jingfeng
318f39db26cSSui Jingfeng if (gpu_addr)
319f39db26cSSui Jingfeng *gpu_addr = lsdc_bo_gpu_offset(lbo);
320f39db26cSSui Jingfeng
321f39db26cSSui Jingfeng return 0;
322f39db26cSSui Jingfeng }
323f39db26cSSui Jingfeng
lsdc_bo_unpin(struct lsdc_bo * lbo)324f39db26cSSui Jingfeng void lsdc_bo_unpin(struct lsdc_bo *lbo)
325f39db26cSSui Jingfeng {
326f39db26cSSui Jingfeng struct ttm_buffer_object *tbo = &lbo->tbo;
327f39db26cSSui Jingfeng struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
328f39db26cSSui Jingfeng
329f39db26cSSui Jingfeng if (unlikely(!tbo->pin_count)) {
330f39db26cSSui Jingfeng drm_dbg(&ldev->base, "%p unpin is not necessary\n", lbo);
331f39db26cSSui Jingfeng return;
332f39db26cSSui Jingfeng }
333f39db26cSSui Jingfeng
334f39db26cSSui Jingfeng ttm_bo_unpin(tbo);
335f39db26cSSui Jingfeng
336f39db26cSSui Jingfeng if (!tbo->pin_count) {
337f39db26cSSui Jingfeng if (tbo->resource->mem_type == TTM_PL_VRAM)
338f39db26cSSui Jingfeng ldev->vram_pinned_size -= lsdc_bo_size(lbo);
339f39db26cSSui Jingfeng else if (tbo->resource->mem_type == TTM_PL_TT)
340f39db26cSSui Jingfeng ldev->gtt_pinned_size -= lsdc_bo_size(lbo);
341f39db26cSSui Jingfeng }
342f39db26cSSui Jingfeng }
343f39db26cSSui Jingfeng
lsdc_bo_ref(struct lsdc_bo * lbo)344f39db26cSSui Jingfeng void lsdc_bo_ref(struct lsdc_bo *lbo)
345f39db26cSSui Jingfeng {
346f39db26cSSui Jingfeng struct ttm_buffer_object *tbo = &lbo->tbo;
347f39db26cSSui Jingfeng
348f39db26cSSui Jingfeng ttm_bo_get(tbo);
349f39db26cSSui Jingfeng }
350f39db26cSSui Jingfeng
lsdc_bo_unref(struct lsdc_bo * lbo)351f39db26cSSui Jingfeng void lsdc_bo_unref(struct lsdc_bo *lbo)
352f39db26cSSui Jingfeng {
353f39db26cSSui Jingfeng struct ttm_buffer_object *tbo = &lbo->tbo;
354f39db26cSSui Jingfeng
355f39db26cSSui Jingfeng ttm_bo_put(tbo);
356f39db26cSSui Jingfeng }
357f39db26cSSui Jingfeng
lsdc_bo_kmap(struct lsdc_bo * lbo)358f39db26cSSui Jingfeng int lsdc_bo_kmap(struct lsdc_bo *lbo)
359f39db26cSSui Jingfeng {
360f39db26cSSui Jingfeng struct ttm_buffer_object *tbo = &lbo->tbo;
361f39db26cSSui Jingfeng struct drm_gem_object *gem = &tbo->base;
362f39db26cSSui Jingfeng struct drm_device *ddev = gem->dev;
363f39db26cSSui Jingfeng long ret;
364f39db26cSSui Jingfeng int err;
365f39db26cSSui Jingfeng
366f39db26cSSui Jingfeng ret = dma_resv_wait_timeout(gem->resv, DMA_RESV_USAGE_KERNEL, false,
367f39db26cSSui Jingfeng MAX_SCHEDULE_TIMEOUT);
368f39db26cSSui Jingfeng if (ret < 0) {
369f39db26cSSui Jingfeng drm_warn(ddev, "wait fence timeout\n");
370f39db26cSSui Jingfeng return ret;
371f39db26cSSui Jingfeng }
372f39db26cSSui Jingfeng
373f39db26cSSui Jingfeng if (lbo->kptr)
374f39db26cSSui Jingfeng return 0;
375f39db26cSSui Jingfeng
376f39db26cSSui Jingfeng err = ttm_bo_kmap(tbo, 0, PFN_UP(lsdc_bo_size(lbo)), &lbo->kmap);
377f39db26cSSui Jingfeng if (err) {
378f39db26cSSui Jingfeng drm_err(ddev, "kmap %p failed: %d\n", lbo, err);
379f39db26cSSui Jingfeng return err;
380f39db26cSSui Jingfeng }
381f39db26cSSui Jingfeng
382f39db26cSSui Jingfeng lbo->kptr = ttm_kmap_obj_virtual(&lbo->kmap, &lbo->is_iomem);
383f39db26cSSui Jingfeng
384f39db26cSSui Jingfeng return 0;
385f39db26cSSui Jingfeng }
386f39db26cSSui Jingfeng
lsdc_bo_kunmap(struct lsdc_bo * lbo)387f39db26cSSui Jingfeng void lsdc_bo_kunmap(struct lsdc_bo *lbo)
388f39db26cSSui Jingfeng {
389f39db26cSSui Jingfeng if (!lbo->kptr)
390f39db26cSSui Jingfeng return;
391f39db26cSSui Jingfeng
392f39db26cSSui Jingfeng lbo->kptr = NULL;
393f39db26cSSui Jingfeng ttm_bo_kunmap(&lbo->kmap);
394f39db26cSSui Jingfeng }
395f39db26cSSui Jingfeng
lsdc_bo_clear(struct lsdc_bo * lbo)396f39db26cSSui Jingfeng void lsdc_bo_clear(struct lsdc_bo *lbo)
397f39db26cSSui Jingfeng {
398f39db26cSSui Jingfeng lsdc_bo_kmap(lbo);
399f39db26cSSui Jingfeng
400f39db26cSSui Jingfeng if (lbo->is_iomem)
401f39db26cSSui Jingfeng memset_io((void __iomem *)lbo->kptr, 0, lbo->size);
402f39db26cSSui Jingfeng else
403f39db26cSSui Jingfeng memset(lbo->kptr, 0, lbo->size);
404f39db26cSSui Jingfeng
405f39db26cSSui Jingfeng lsdc_bo_kunmap(lbo);
406f39db26cSSui Jingfeng }
407f39db26cSSui Jingfeng
lsdc_bo_evict_vram(struct drm_device * ddev)408f39db26cSSui Jingfeng int lsdc_bo_evict_vram(struct drm_device *ddev)
409f39db26cSSui Jingfeng {
410f39db26cSSui Jingfeng struct lsdc_device *ldev = to_lsdc(ddev);
411f39db26cSSui Jingfeng struct ttm_device *bdev = &ldev->bdev;
412f39db26cSSui Jingfeng struct ttm_resource_manager *man;
413f39db26cSSui Jingfeng
414f39db26cSSui Jingfeng man = ttm_manager_type(bdev, TTM_PL_VRAM);
415f39db26cSSui Jingfeng if (unlikely(!man))
416f39db26cSSui Jingfeng return 0;
417f39db26cSSui Jingfeng
418f39db26cSSui Jingfeng return ttm_resource_manager_evict_all(bdev, man);
419f39db26cSSui Jingfeng }
420f39db26cSSui Jingfeng
lsdc_bo_destroy(struct ttm_buffer_object * tbo)421f39db26cSSui Jingfeng static void lsdc_bo_destroy(struct ttm_buffer_object *tbo)
422f39db26cSSui Jingfeng {
423f39db26cSSui Jingfeng struct lsdc_device *ldev = tdev_to_ldev(tbo->bdev);
424f39db26cSSui Jingfeng struct lsdc_bo *lbo = to_lsdc_bo(tbo);
425f39db26cSSui Jingfeng
426f39db26cSSui Jingfeng mutex_lock(&ldev->gem.mutex);
427f39db26cSSui Jingfeng list_del_init(&lbo->list);
428f39db26cSSui Jingfeng mutex_unlock(&ldev->gem.mutex);
429f39db26cSSui Jingfeng
430f39db26cSSui Jingfeng drm_gem_object_release(&tbo->base);
431f39db26cSSui Jingfeng
432f39db26cSSui Jingfeng kfree(lbo);
433f39db26cSSui Jingfeng }
434f39db26cSSui Jingfeng
lsdc_bo_create(struct drm_device * ddev,u32 domain,size_t size,bool kernel,struct sg_table * sg,struct dma_resv * resv)435f39db26cSSui Jingfeng struct lsdc_bo *lsdc_bo_create(struct drm_device *ddev,
436f39db26cSSui Jingfeng u32 domain,
437f39db26cSSui Jingfeng size_t size,
438f39db26cSSui Jingfeng bool kernel,
439f39db26cSSui Jingfeng struct sg_table *sg,
440f39db26cSSui Jingfeng struct dma_resv *resv)
441f39db26cSSui Jingfeng {
442f39db26cSSui Jingfeng struct lsdc_device *ldev = to_lsdc(ddev);
443f39db26cSSui Jingfeng struct ttm_device *bdev = &ldev->bdev;
444f39db26cSSui Jingfeng struct ttm_buffer_object *tbo;
445f39db26cSSui Jingfeng struct lsdc_bo *lbo;
446f39db26cSSui Jingfeng enum ttm_bo_type bo_type;
447f39db26cSSui Jingfeng int ret;
448f39db26cSSui Jingfeng
449f39db26cSSui Jingfeng lbo = kzalloc(sizeof(*lbo), GFP_KERNEL);
450f39db26cSSui Jingfeng if (!lbo)
451f39db26cSSui Jingfeng return ERR_PTR(-ENOMEM);
452f39db26cSSui Jingfeng
453f39db26cSSui Jingfeng INIT_LIST_HEAD(&lbo->list);
454f39db26cSSui Jingfeng
455f39db26cSSui Jingfeng lbo->initial_domain = domain & (LSDC_GEM_DOMAIN_VRAM |
456f39db26cSSui Jingfeng LSDC_GEM_DOMAIN_GTT |
457f39db26cSSui Jingfeng LSDC_GEM_DOMAIN_SYSTEM);
458f39db26cSSui Jingfeng
459f39db26cSSui Jingfeng tbo = &lbo->tbo;
460f39db26cSSui Jingfeng
461f39db26cSSui Jingfeng size = ALIGN(size, PAGE_SIZE);
462f39db26cSSui Jingfeng
463f39db26cSSui Jingfeng ret = drm_gem_object_init(ddev, &tbo->base, size);
464f39db26cSSui Jingfeng if (ret) {
465f39db26cSSui Jingfeng kfree(lbo);
466f39db26cSSui Jingfeng return ERR_PTR(ret);
467f39db26cSSui Jingfeng }
468f39db26cSSui Jingfeng
469f39db26cSSui Jingfeng tbo->bdev = bdev;
470f39db26cSSui Jingfeng
471f39db26cSSui Jingfeng if (kernel)
472f39db26cSSui Jingfeng bo_type = ttm_bo_type_kernel;
473f39db26cSSui Jingfeng else if (sg)
474f39db26cSSui Jingfeng bo_type = ttm_bo_type_sg;
475f39db26cSSui Jingfeng else
476f39db26cSSui Jingfeng bo_type = ttm_bo_type_device;
477f39db26cSSui Jingfeng
478f39db26cSSui Jingfeng lsdc_bo_set_placement(lbo, domain);
479f39db26cSSui Jingfeng lbo->size = size;
480f39db26cSSui Jingfeng
481f39db26cSSui Jingfeng ret = ttm_bo_init_validate(bdev, tbo, bo_type, &lbo->placement, 0,
482f39db26cSSui Jingfeng false, sg, resv, lsdc_bo_destroy);
483f39db26cSSui Jingfeng if (ret) {
484f39db26cSSui Jingfeng kfree(lbo);
485f39db26cSSui Jingfeng return ERR_PTR(ret);
486f39db26cSSui Jingfeng }
487f39db26cSSui Jingfeng
488f39db26cSSui Jingfeng return lbo;
489f39db26cSSui Jingfeng }
490f39db26cSSui Jingfeng
lsdc_bo_create_kernel_pinned(struct drm_device * ddev,u32 domain,size_t size)491f39db26cSSui Jingfeng struct lsdc_bo *lsdc_bo_create_kernel_pinned(struct drm_device *ddev,
492f39db26cSSui Jingfeng u32 domain,
493f39db26cSSui Jingfeng size_t size)
494f39db26cSSui Jingfeng {
495f39db26cSSui Jingfeng struct lsdc_bo *lbo;
496f39db26cSSui Jingfeng int ret;
497f39db26cSSui Jingfeng
498f39db26cSSui Jingfeng lbo = lsdc_bo_create(ddev, domain, size, true, NULL, NULL);
499*b0e9267dSDan Carpenter if (IS_ERR(lbo))
500*b0e9267dSDan Carpenter return ERR_CAST(lbo);
501f39db26cSSui Jingfeng
502f39db26cSSui Jingfeng ret = lsdc_bo_reserve(lbo);
503f39db26cSSui Jingfeng if (unlikely(ret)) {
504f39db26cSSui Jingfeng lsdc_bo_unref(lbo);
505f39db26cSSui Jingfeng return ERR_PTR(ret);
506f39db26cSSui Jingfeng }
507f39db26cSSui Jingfeng
508f39db26cSSui Jingfeng ret = lsdc_bo_pin(lbo, domain, NULL);
509f39db26cSSui Jingfeng lsdc_bo_unreserve(lbo);
510f39db26cSSui Jingfeng if (unlikely(ret)) {
511f39db26cSSui Jingfeng lsdc_bo_unref(lbo);
512f39db26cSSui Jingfeng return ERR_PTR(ret);
513f39db26cSSui Jingfeng }
514f39db26cSSui Jingfeng
515f39db26cSSui Jingfeng return lbo;
516f39db26cSSui Jingfeng }
517f39db26cSSui Jingfeng
lsdc_bo_free_kernel_pinned(struct lsdc_bo * lbo)518f39db26cSSui Jingfeng void lsdc_bo_free_kernel_pinned(struct lsdc_bo *lbo)
519f39db26cSSui Jingfeng {
520f39db26cSSui Jingfeng int ret;
521f39db26cSSui Jingfeng
522f39db26cSSui Jingfeng ret = lsdc_bo_reserve(lbo);
523f39db26cSSui Jingfeng if (unlikely(ret))
524f39db26cSSui Jingfeng return;
525f39db26cSSui Jingfeng
526f39db26cSSui Jingfeng lsdc_bo_unpin(lbo);
527f39db26cSSui Jingfeng lsdc_bo_unreserve(lbo);
528f39db26cSSui Jingfeng
529f39db26cSSui Jingfeng lsdc_bo_unref(lbo);
530f39db26cSSui Jingfeng }
531f39db26cSSui Jingfeng
lsdc_ttm_fini(struct drm_device * ddev,void * data)532f39db26cSSui Jingfeng static void lsdc_ttm_fini(struct drm_device *ddev, void *data)
533f39db26cSSui Jingfeng {
534f39db26cSSui Jingfeng struct lsdc_device *ldev = (struct lsdc_device *)data;
535f39db26cSSui Jingfeng
536f39db26cSSui Jingfeng ttm_range_man_fini(&ldev->bdev, TTM_PL_VRAM);
537f39db26cSSui Jingfeng ttm_range_man_fini(&ldev->bdev, TTM_PL_TT);
538f39db26cSSui Jingfeng
539f39db26cSSui Jingfeng ttm_device_fini(&ldev->bdev);
540f39db26cSSui Jingfeng
541f39db26cSSui Jingfeng drm_dbg(ddev, "ttm finished\n");
542f39db26cSSui Jingfeng }
543f39db26cSSui Jingfeng
lsdc_ttm_init(struct lsdc_device * ldev)544f39db26cSSui Jingfeng int lsdc_ttm_init(struct lsdc_device *ldev)
545f39db26cSSui Jingfeng {
546f39db26cSSui Jingfeng struct drm_device *ddev = &ldev->base;
547f39db26cSSui Jingfeng unsigned long num_vram_pages;
548f39db26cSSui Jingfeng unsigned long num_gtt_pages;
549f39db26cSSui Jingfeng int ret;
550f39db26cSSui Jingfeng
551f39db26cSSui Jingfeng ret = ttm_device_init(&ldev->bdev, &lsdc_bo_driver, ddev->dev,
552f39db26cSSui Jingfeng ddev->anon_inode->i_mapping,
553f39db26cSSui Jingfeng ddev->vma_offset_manager, false, true);
554f39db26cSSui Jingfeng if (ret)
555f39db26cSSui Jingfeng return ret;
556f39db26cSSui Jingfeng
557f39db26cSSui Jingfeng num_vram_pages = ldev->vram_size >> PAGE_SHIFT;
558f39db26cSSui Jingfeng
559f39db26cSSui Jingfeng ret = ttm_range_man_init(&ldev->bdev, TTM_PL_VRAM, false, num_vram_pages);
560f39db26cSSui Jingfeng if (unlikely(ret))
561f39db26cSSui Jingfeng return ret;
562f39db26cSSui Jingfeng
563f39db26cSSui Jingfeng drm_info(ddev, "VRAM: %lu pages ready\n", num_vram_pages);
564f39db26cSSui Jingfeng
565f39db26cSSui Jingfeng /* 512M is far enough for us now */
566f39db26cSSui Jingfeng ldev->gtt_size = 512 << 20;
567f39db26cSSui Jingfeng
568f39db26cSSui Jingfeng num_gtt_pages = ldev->gtt_size >> PAGE_SHIFT;
569f39db26cSSui Jingfeng
570f39db26cSSui Jingfeng ret = ttm_range_man_init(&ldev->bdev, TTM_PL_TT, true, num_gtt_pages);
571f39db26cSSui Jingfeng if (unlikely(ret))
572f39db26cSSui Jingfeng return ret;
573f39db26cSSui Jingfeng
574f39db26cSSui Jingfeng drm_info(ddev, "GTT: %lu pages ready\n", num_gtt_pages);
575f39db26cSSui Jingfeng
576f39db26cSSui Jingfeng return drmm_add_action_or_reset(ddev, lsdc_ttm_fini, ldev);
577f39db26cSSui Jingfeng }
578f39db26cSSui Jingfeng
lsdc_ttm_debugfs_init(struct lsdc_device * ldev)579f39db26cSSui Jingfeng void lsdc_ttm_debugfs_init(struct lsdc_device *ldev)
580f39db26cSSui Jingfeng {
581f39db26cSSui Jingfeng struct ttm_device *bdev = &ldev->bdev;
582f39db26cSSui Jingfeng struct drm_device *ddev = &ldev->base;
583f39db26cSSui Jingfeng struct drm_minor *minor = ddev->primary;
584f39db26cSSui Jingfeng struct dentry *root = minor->debugfs_root;
585f39db26cSSui Jingfeng struct ttm_resource_manager *vram_man;
586f39db26cSSui Jingfeng struct ttm_resource_manager *gtt_man;
587f39db26cSSui Jingfeng
588f39db26cSSui Jingfeng vram_man = ttm_manager_type(bdev, TTM_PL_VRAM);
589f39db26cSSui Jingfeng gtt_man = ttm_manager_type(bdev, TTM_PL_TT);
590f39db26cSSui Jingfeng
591f39db26cSSui Jingfeng ttm_resource_manager_create_debugfs(vram_man, root, "vram_mm");
592f39db26cSSui Jingfeng ttm_resource_manager_create_debugfs(gtt_man, root, "gtt_mm");
593f39db26cSSui Jingfeng }
594