1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2021 Intel Corporation
4  */
5 #include <drm/ttm/ttm_bo_driver.h>
6 #include <drm/ttm/ttm_device.h>
7 #include <drm/ttm/ttm_range_manager.h>
8 
9 #include "i915_drv.h"
10 #include "i915_scatterlist.h"
11 #include "i915_ttm_buddy_manager.h"
12 
13 #include "intel_region_ttm.h"
14 
15 #include "gem/i915_gem_region.h"
16 #include "gem/i915_gem_ttm.h" /* For the funcs/ops export only */
17 /**
18  * DOC: TTM support structure
19  *
20  * The code in this file deals with setting up memory managers for TTM
21  * LMEM and MOCK regions and converting the output from
22  * the managers to struct sg_table, Basically providing the mapping from
23  * i915 GEM regions to TTM memory types and resource managers.
24  */
25 
26 /**
27  * intel_region_ttm_device_init - Initialize a TTM device
28  * @dev_priv: Pointer to an i915 device private structure.
29  *
30  * Return: 0 on success, negative error code on failure.
31  */
32 int intel_region_ttm_device_init(struct drm_i915_private *dev_priv)
33 {
34 	struct drm_device *drm = &dev_priv->drm;
35 
36 	return ttm_device_init(&dev_priv->bdev, i915_ttm_driver(),
37 			       drm->dev, drm->anon_inode->i_mapping,
38 			       drm->vma_offset_manager, false, false);
39 }
40 
41 /**
42  * intel_region_ttm_device_fini - Finalize a TTM device
43  * @dev_priv: Pointer to an i915 device private structure.
44  */
45 void intel_region_ttm_device_fini(struct drm_i915_private *dev_priv)
46 {
47 	ttm_device_fini(&dev_priv->bdev);
48 }
49 
50 /*
51  * Map the i915 memory regions to TTM memory types. We use the
52  * driver-private types for now, reserving TTM_PL_VRAM for stolen
53  * memory and TTM_PL_TT for GGTT use if decided to implement this.
54  */
55 int intel_region_to_ttm_type(const struct intel_memory_region *mem)
56 {
57 	int type;
58 
59 	GEM_BUG_ON(mem->type != INTEL_MEMORY_LOCAL &&
60 		   mem->type != INTEL_MEMORY_MOCK &&
61 		   mem->type != INTEL_MEMORY_SYSTEM);
62 
63 	if (mem->type == INTEL_MEMORY_SYSTEM)
64 		return TTM_PL_SYSTEM;
65 
66 	type = mem->instance + TTM_PL_PRIV;
67 	GEM_BUG_ON(type >= TTM_NUM_MEM_TYPES);
68 
69 	return type;
70 }
71 
72 /**
73  * intel_region_ttm_init - Initialize a memory region for TTM.
74  * @mem: The region to initialize.
75  *
76  * This function initializes a suitable TTM resource manager for the
77  * region, and if it's a LMEM region type, attaches it to the TTM
78  * device. MOCK regions are NOT attached to the TTM device, since we don't
79  * have one for the mock selftests.
80  *
81  * Return: 0 on success, negative error code on failure.
82  */
83 int intel_region_ttm_init(struct intel_memory_region *mem)
84 {
85 	struct ttm_device *bdev = &mem->i915->bdev;
86 	int mem_type = intel_region_to_ttm_type(mem);
87 	int ret;
88 
89 	ret = i915_ttm_buddy_man_init(bdev, mem_type, false,
90 				      resource_size(&mem->region),
91 				      mem->io_size,
92 				      mem->min_page_size, PAGE_SIZE);
93 	if (ret)
94 		return ret;
95 
96 	mem->region_private = ttm_manager_type(bdev, mem_type);
97 
98 	return 0;
99 }
100 
101 /**
102  * intel_region_ttm_fini - Finalize a TTM region.
103  * @mem: The memory region
104  *
105  * This functions takes down the TTM resource manager associated with the
106  * memory region, and if it was registered with the TTM device,
107  * removes that registration.
108  */
109 int intel_region_ttm_fini(struct intel_memory_region *mem)
110 {
111 	struct ttm_resource_manager *man = mem->region_private;
112 	int ret = -EBUSY;
113 	int count;
114 
115 	/*
116 	 * Put the region's move fences. This releases requests that
117 	 * may hold on to contexts and vms that may hold on to buffer
118 	 * objects placed in this region.
119 	 */
120 	if (man)
121 		ttm_resource_manager_cleanup(man);
122 
123 	/* Flush objects from region. */
124 	for (count = 0; count < 10; ++count) {
125 		i915_gem_flush_free_objects(mem->i915);
126 
127 		mutex_lock(&mem->objects.lock);
128 		if (list_empty(&mem->objects.list))
129 			ret = 0;
130 		mutex_unlock(&mem->objects.lock);
131 		if (!ret)
132 			break;
133 
134 		msleep(20);
135 		flush_delayed_work(&mem->i915->bdev.wq);
136 	}
137 
138 	/* If we leaked objects, Don't free the region causing use after free */
139 	if (ret || !man)
140 		return ret;
141 
142 	ret = i915_ttm_buddy_man_fini(&mem->i915->bdev,
143 				      intel_region_to_ttm_type(mem));
144 	GEM_WARN_ON(ret);
145 	mem->region_private = NULL;
146 
147 	return ret;
148 }
149 
150 /**
151  * intel_region_ttm_resource_to_rsgt -
152  * Convert an opaque TTM resource manager resource to a refcounted sg_table.
153  * @mem: The memory region.
154  * @res: The resource manager resource obtained from the TTM resource manager.
155  * @page_alignment: Required page alignment for each sg entry. Power of two.
156  *
157  * The gem backends typically use sg-tables for operations on the underlying
158  * io_memory. So provide a way for the backends to translate the
159  * nodes they are handed from TTM to sg-tables.
160  *
161  * Return: A malloced sg_table on success, an error pointer on failure.
162  */
163 struct i915_refct_sgt *
164 intel_region_ttm_resource_to_rsgt(struct intel_memory_region *mem,
165 				  struct ttm_resource *res,
166 				  u32 page_alignment)
167 {
168 	if (mem->is_range_manager) {
169 		struct ttm_range_mgr_node *range_node =
170 			to_ttm_range_mgr_node(res);
171 
172 		return i915_rsgt_from_mm_node(&range_node->mm_nodes[0],
173 					      mem->region.start,
174 					      page_alignment);
175 	} else {
176 		return i915_rsgt_from_buddy_resource(res, mem->region.start,
177 						     page_alignment);
178 	}
179 }
180 
181 #ifdef CONFIG_DRM_I915_SELFTEST
182 /**
183  * intel_region_ttm_resource_alloc - Allocate memory resources from a region
184  * @mem: The memory region,
185  * @size: The requested size in bytes
186  * @flags: Allocation flags
187  *
188  * This functionality is provided only for callers that need to allocate
189  * memory from standalone TTM range managers, without the TTM eviction
190  * functionality. Don't use if you are not completely sure that's the
191  * case. The returned opaque node can be converted to an sg_table using
192  * intel_region_ttm_resource_to_st(), and can be freed using
193  * intel_region_ttm_resource_free().
194  *
195  * Return: A valid pointer on success, an error pointer on failure.
196  */
197 struct ttm_resource *
198 intel_region_ttm_resource_alloc(struct intel_memory_region *mem,
199 				resource_size_t offset,
200 				resource_size_t size,
201 				unsigned int flags)
202 {
203 	struct ttm_resource_manager *man = mem->region_private;
204 	struct ttm_place place = {};
205 	struct ttm_buffer_object mock_bo = {};
206 	struct ttm_resource *res;
207 	int ret;
208 
209 	if (flags & I915_BO_ALLOC_CONTIGUOUS)
210 		place.flags |= TTM_PL_FLAG_CONTIGUOUS;
211 	if (offset != I915_BO_INVALID_OFFSET) {
212 		place.fpfn = offset >> PAGE_SHIFT;
213 		place.lpfn = place.fpfn + (size >> PAGE_SHIFT);
214 	} else if (mem->io_size && mem->io_size < mem->total) {
215 		if (flags & I915_BO_ALLOC_GPU_ONLY) {
216 			place.flags |= TTM_PL_FLAG_TOPDOWN;
217 		} else {
218 			place.fpfn = 0;
219 			place.lpfn = mem->io_size >> PAGE_SHIFT;
220 		}
221 	}
222 
223 	mock_bo.base.size = size;
224 	mock_bo.bdev = &mem->i915->bdev;
225 
226 	ret = man->func->alloc(man, &mock_bo, &place, &res);
227 	if (ret == -ENOSPC)
228 		ret = -ENXIO;
229 	if (!ret)
230 		res->bo = NULL; /* Rather blow up, then some uaf */
231 	return ret ? ERR_PTR(ret) : res;
232 }
233 
234 #endif
235 
236 /**
237  * intel_region_ttm_resource_free - Free a resource allocated from a resource manager
238  * @mem: The region the resource was allocated from.
239  * @res: The opaque resource representing an allocation.
240  */
241 void intel_region_ttm_resource_free(struct intel_memory_region *mem,
242 				    struct ttm_resource *res)
243 {
244 	struct ttm_resource_manager *man = mem->region_private;
245 	struct ttm_buffer_object mock_bo = {};
246 
247 	mock_bo.base.size = res->num_pages << PAGE_SHIFT;
248 	mock_bo.bdev = &mem->i915->bdev;
249 	res->bo = &mock_bo;
250 
251 	man->func->free(man, res);
252 }
253