113 static void drm_gem_vram_cleanup(struct drm_gem_vram_object *gbo)  in drm_gem_vram_cleanup()  argument
120 	WARN_ON(gbo->vmap_use_count);  in drm_gem_vram_cleanup()
121 	WARN_ON(iosys_map_is_set(&gbo->map));  in drm_gem_vram_cleanup()
123 	drm_gem_object_release(&gbo->bo.base);  in drm_gem_vram_cleanup()
126 static void drm_gem_vram_destroy(struct drm_gem_vram_object *gbo)  in drm_gem_vram_destroy()  argument
128 	drm_gem_vram_cleanup(gbo);  in drm_gem_vram_destroy()
129 	kfree(gbo);  in drm_gem_vram_destroy()
134 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_bo(bo);  in ttm_buffer_object_destroy()  local
136 	drm_gem_vram_destroy(gbo);  in ttm_buffer_object_destroy()
139 static void drm_gem_vram_placement(struct drm_gem_vram_object *gbo,  in drm_gem_vram_placement()  argument
149 	gbo->placement.placement = gbo->placements;  in drm_gem_vram_placement()
150 	gbo->placement.busy_placement = gbo->placements;  in drm_gem_vram_placement()
153 		gbo->placements[c].mem_type = TTM_PL_VRAM;  in drm_gem_vram_placement()
154 		gbo->placements[c++].flags = invariant_flags;  in drm_gem_vram_placement()
158 		gbo->placements[c].mem_type = TTM_PL_SYSTEM;  in drm_gem_vram_placement()
159 		gbo->placements[c++].flags = invariant_flags;  in drm_gem_vram_placement()
162 	gbo->placement.num_placement = c;  in drm_gem_vram_placement()
163 	gbo->placement.num_busy_placement = c;  in drm_gem_vram_placement()
166 		gbo->placements[i].fpfn = 0;  in drm_gem_vram_placement()
167 		gbo->placements[i].lpfn = 0;  in drm_gem_vram_placement()
191 	struct drm_gem_vram_object *gbo;  in drm_gem_vram_create()  local
204 		gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_create()
206 		gbo = kzalloc(sizeof(*gbo), GFP_KERNEL);  in drm_gem_vram_create()
207 		if (!gbo)  in drm_gem_vram_create()
209 		gem = &gbo->bo.base;  in drm_gem_vram_create()
217 		kfree(gbo);  in drm_gem_vram_create()
223 	gbo->bo.bdev = bdev;  in drm_gem_vram_create()
224 	drm_gem_vram_placement(gbo, DRM_GEM_VRAM_PL_FLAG_SYSTEM);  in drm_gem_vram_create()
230 	ret = ttm_bo_init_validate(bdev, &gbo->bo, ttm_bo_type_device,  in drm_gem_vram_create()
231 				   &gbo->placement, pg_align, false, NULL, NULL,  in drm_gem_vram_create()
236 	return gbo;  in drm_gem_vram_create()
246 void drm_gem_vram_put(struct drm_gem_vram_object *gbo)  in drm_gem_vram_put()  argument
248 	ttm_bo_put(&gbo->bo);  in drm_gem_vram_put()
252 static u64 drm_gem_vram_pg_offset(struct drm_gem_vram_object *gbo)  in drm_gem_vram_pg_offset()  argument
255 	if (WARN_ON_ONCE(!gbo->bo.resource ||  in drm_gem_vram_pg_offset()
256 			 gbo->bo.resource->mem_type == TTM_PL_SYSTEM))  in drm_gem_vram_pg_offset()
259 	return gbo->bo.resource->start;  in drm_gem_vram_pg_offset()
274 s64 drm_gem_vram_offset(struct drm_gem_vram_object *gbo)  in drm_gem_vram_offset()  argument
276 	if (WARN_ON_ONCE(!gbo->bo.pin_count))  in drm_gem_vram_offset()
278 	return drm_gem_vram_pg_offset(gbo) << PAGE_SHIFT;  in drm_gem_vram_offset()
282 static int drm_gem_vram_pin_locked(struct drm_gem_vram_object *gbo,  in drm_gem_vram_pin_locked()  argument
288 	if (gbo->bo.pin_count)  in drm_gem_vram_pin_locked()
292 		drm_gem_vram_placement(gbo, pl_flag);  in drm_gem_vram_pin_locked()
294 	ret = ttm_bo_validate(&gbo->bo, &gbo->placement, &ctx);  in drm_gem_vram_pin_locked()
299 	ttm_bo_pin(&gbo->bo);  in drm_gem_vram_pin_locked()
327 int drm_gem_vram_pin(struct drm_gem_vram_object *gbo, unsigned long pl_flag)  in drm_gem_vram_pin()  argument
331 	ret = ttm_bo_reserve(&gbo->bo, true, false, NULL);  in drm_gem_vram_pin()
334 	ret = drm_gem_vram_pin_locked(gbo, pl_flag);  in drm_gem_vram_pin()
335 	ttm_bo_unreserve(&gbo->bo);  in drm_gem_vram_pin()
341 static void drm_gem_vram_unpin_locked(struct drm_gem_vram_object *gbo)  in drm_gem_vram_unpin_locked()  argument
343 	ttm_bo_unpin(&gbo->bo);  in drm_gem_vram_unpin_locked()
354 int drm_gem_vram_unpin(struct drm_gem_vram_object *gbo)  in drm_gem_vram_unpin()  argument
358 	ret = ttm_bo_reserve(&gbo->bo, true, false, NULL);  in drm_gem_vram_unpin()
362 	drm_gem_vram_unpin_locked(gbo);  in drm_gem_vram_unpin()
363 	ttm_bo_unreserve(&gbo->bo);  in drm_gem_vram_unpin()
369 static int drm_gem_vram_kmap_locked(struct drm_gem_vram_object *gbo,  in drm_gem_vram_kmap_locked()  argument
374 	if (gbo->vmap_use_count > 0)  in drm_gem_vram_kmap_locked()
382 	if (iosys_map_is_null(&gbo->map)) {  in drm_gem_vram_kmap_locked()
383 		ret = ttm_bo_vmap(&gbo->bo, &gbo->map);  in drm_gem_vram_kmap_locked()
389 	++gbo->vmap_use_count;  in drm_gem_vram_kmap_locked()
390 	*map = gbo->map;  in drm_gem_vram_kmap_locked()
395 static void drm_gem_vram_kunmap_locked(struct drm_gem_vram_object *gbo,  in drm_gem_vram_kunmap_locked()  argument
398 	struct drm_device *dev = gbo->bo.base.dev;  in drm_gem_vram_kunmap_locked()
400 	if (drm_WARN_ON_ONCE(dev, !gbo->vmap_use_count))  in drm_gem_vram_kunmap_locked()
403 	if (drm_WARN_ON_ONCE(dev, !iosys_map_is_equal(&gbo->map, map)))  in drm_gem_vram_kunmap_locked()
406 	if (--gbo->vmap_use_count > 0)  in drm_gem_vram_kunmap_locked()
433 int drm_gem_vram_vmap(struct drm_gem_vram_object *gbo, struct iosys_map *map)  in drm_gem_vram_vmap()  argument
437 	dma_resv_assert_held(gbo->bo.base.resv);  in drm_gem_vram_vmap()
439 	ret = drm_gem_vram_pin_locked(gbo, 0);  in drm_gem_vram_vmap()
442 	ret = drm_gem_vram_kmap_locked(gbo, map);  in drm_gem_vram_vmap()
449 	drm_gem_vram_unpin_locked(gbo);  in drm_gem_vram_vmap()
462 void drm_gem_vram_vunmap(struct drm_gem_vram_object *gbo,  in drm_gem_vram_vunmap()  argument
465 	dma_resv_assert_held(gbo->bo.base.resv);  in drm_gem_vram_vunmap()
467 	drm_gem_vram_kunmap_locked(gbo, map);  in drm_gem_vram_vunmap()
468 	drm_gem_vram_unpin_locked(gbo);  in drm_gem_vram_vunmap()
498 	struct drm_gem_vram_object *gbo;  in drm_gem_vram_fill_create_dumb()  local
514 	gbo = drm_gem_vram_create(dev, size, pg_align);  in drm_gem_vram_fill_create_dumb()
515 	if (IS_ERR(gbo))  in drm_gem_vram_fill_create_dumb()
516 		return PTR_ERR(gbo);  in drm_gem_vram_fill_create_dumb()
518 	ret = drm_gem_handle_create(file, &gbo->bo.base, &handle);  in drm_gem_vram_fill_create_dumb()
522 	drm_gem_object_put(&gbo->bo.base);  in drm_gem_vram_fill_create_dumb()
531 	drm_gem_object_put(&gbo->bo.base);  in drm_gem_vram_fill_create_dumb()
545 static void drm_gem_vram_bo_driver_evict_flags(struct drm_gem_vram_object *gbo,  in drm_gem_vram_bo_driver_evict_flags()  argument
548 	drm_gem_vram_placement(gbo, DRM_GEM_VRAM_PL_FLAG_SYSTEM);  in drm_gem_vram_bo_driver_evict_flags()
549 	*pl = gbo->placement;  in drm_gem_vram_bo_driver_evict_flags()
552 static void drm_gem_vram_bo_driver_move_notify(struct drm_gem_vram_object *gbo)  in drm_gem_vram_bo_driver_move_notify()  argument
554 	struct ttm_buffer_object *bo = &gbo->bo;  in drm_gem_vram_bo_driver_move_notify()
557 	if (drm_WARN_ON_ONCE(dev, gbo->vmap_use_count))  in drm_gem_vram_bo_driver_move_notify()
560 	ttm_bo_vunmap(bo, &gbo->map);  in drm_gem_vram_bo_driver_move_notify()
561 	iosys_map_clear(&gbo->map); /* explicitly clear mapping for next vmap call */  in drm_gem_vram_bo_driver_move_notify()
564 static int drm_gem_vram_bo_driver_move(struct drm_gem_vram_object *gbo,  in drm_gem_vram_bo_driver_move()  argument
569 	drm_gem_vram_bo_driver_move_notify(gbo);  in drm_gem_vram_bo_driver_move()
570 	return ttm_bo_move_memcpy(&gbo->bo, ctx, new_mem);  in drm_gem_vram_bo_driver_move()
584 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_free()  local
586 	drm_gem_vram_put(gbo);  in drm_gem_vram_object_free()
628 	struct drm_gem_vram_object *gbo;  in __drm_gem_vram_plane_helper_cleanup_fb()  local
636 		gbo = drm_gem_vram_of_gem(obj);  in __drm_gem_vram_plane_helper_cleanup_fb()
637 		drm_gem_vram_unpin(gbo);  in __drm_gem_vram_plane_helper_cleanup_fb()
660 	struct drm_gem_vram_object *gbo;  in drm_gem_vram_plane_helper_prepare_fb()  local
674 		gbo = drm_gem_vram_of_gem(obj);  in drm_gem_vram_plane_helper_prepare_fb()
675 		ret = drm_gem_vram_pin(gbo, DRM_GEM_VRAM_PL_FLAG_VRAM);  in drm_gem_vram_plane_helper_prepare_fb()
774 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_pin()  local
784 	return drm_gem_vram_pin(gbo, 0);  in drm_gem_vram_object_pin()
794 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_unpin()  local
796 	drm_gem_vram_unpin(gbo);  in drm_gem_vram_object_unpin()
812 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_vmap()  local
814 	return drm_gem_vram_vmap(gbo, map);  in drm_gem_vram_object_vmap()
826 	struct drm_gem_vram_object *gbo = drm_gem_vram_of_gem(gem);  in drm_gem_vram_object_vunmap()  local
828 	drm_gem_vram_vunmap(gbo, map);  in drm_gem_vram_object_vunmap()
887 	struct drm_gem_vram_object *gbo;  in bo_driver_evict_flags()  local
893 	gbo = drm_gem_vram_of_bo(bo);  in bo_driver_evict_flags()
895 	drm_gem_vram_bo_driver_evict_flags(gbo, placement);  in bo_driver_evict_flags()
900 	struct drm_gem_vram_object *gbo;  in bo_driver_delete_mem_notify()  local
906 	gbo = drm_gem_vram_of_bo(bo);  in bo_driver_delete_mem_notify()
908 	drm_gem_vram_bo_driver_move_notify(gbo);  in bo_driver_delete_mem_notify()
917 	struct drm_gem_vram_object *gbo;  in bo_driver_move()  local
930 	gbo = drm_gem_vram_of_bo(bo);  in bo_driver_move()
932 	return drm_gem_vram_bo_driver_move(gbo, evict, ctx, new_mem);  in bo_driver_move()