1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include "intel_memory_region.h"
7 #include "i915_drv.h"
8 
9 static const struct {
10 	u16 class;
11 	u16 instance;
12 } intel_region_map[] = {
13 	[INTEL_REGION_SMEM] = {
14 		.class = INTEL_MEMORY_SYSTEM,
15 		.instance = 0,
16 	},
17 	[INTEL_REGION_LMEM] = {
18 		.class = INTEL_MEMORY_LOCAL,
19 		.instance = 0,
20 	},
21 	[INTEL_REGION_STOLEN_SMEM] = {
22 		.class = INTEL_MEMORY_STOLEN_SYSTEM,
23 		.instance = 0,
24 	},
25 };
26 
27 struct intel_memory_region *
28 intel_memory_region_by_type(struct drm_i915_private *i915,
29 			    enum intel_memory_type mem_type)
30 {
31 	struct intel_memory_region *mr;
32 	int id;
33 
34 	for_each_memory_region(mr, i915, id)
35 		if (mr->type == mem_type)
36 			return mr;
37 
38 	return NULL;
39 }
40 
41 static u64
42 intel_memory_region_free_pages(struct intel_memory_region *mem,
43 			       struct list_head *blocks)
44 {
45 	struct i915_buddy_block *block, *on;
46 	u64 size = 0;
47 
48 	list_for_each_entry_safe(block, on, blocks, link) {
49 		size += i915_buddy_block_size(&mem->mm, block);
50 		i915_buddy_free(&mem->mm, block);
51 	}
52 	INIT_LIST_HEAD(blocks);
53 
54 	return size;
55 }
56 
57 void
58 __intel_memory_region_put_pages_buddy(struct intel_memory_region *mem,
59 				      struct list_head *blocks)
60 {
61 	mutex_lock(&mem->mm_lock);
62 	mem->avail += intel_memory_region_free_pages(mem, blocks);
63 	mutex_unlock(&mem->mm_lock);
64 }
65 
66 void
67 __intel_memory_region_put_block_buddy(struct i915_buddy_block *block)
68 {
69 	struct list_head blocks;
70 
71 	INIT_LIST_HEAD(&blocks);
72 	list_add(&block->link, &blocks);
73 	__intel_memory_region_put_pages_buddy(block->private, &blocks);
74 }
75 
76 int
77 __intel_memory_region_get_pages_buddy(struct intel_memory_region *mem,
78 				      resource_size_t size,
79 				      unsigned int flags,
80 				      struct list_head *blocks)
81 {
82 	unsigned int min_order = 0;
83 	unsigned long n_pages;
84 
85 	GEM_BUG_ON(!IS_ALIGNED(size, mem->mm.chunk_size));
86 	GEM_BUG_ON(!list_empty(blocks));
87 
88 	if (flags & I915_ALLOC_MIN_PAGE_SIZE) {
89 		min_order = ilog2(mem->min_page_size) -
90 			    ilog2(mem->mm.chunk_size);
91 	}
92 
93 	if (flags & I915_ALLOC_CONTIGUOUS) {
94 		size = roundup_pow_of_two(size);
95 		min_order = ilog2(size) - ilog2(mem->mm.chunk_size);
96 	}
97 
98 	if (size > mem->mm.size)
99 		return -E2BIG;
100 
101 	n_pages = size >> ilog2(mem->mm.chunk_size);
102 
103 	mutex_lock(&mem->mm_lock);
104 
105 	do {
106 		struct i915_buddy_block *block;
107 		unsigned int order;
108 
109 		order = fls(n_pages) - 1;
110 		GEM_BUG_ON(order > mem->mm.max_order);
111 		GEM_BUG_ON(order < min_order);
112 
113 		do {
114 			block = i915_buddy_alloc(&mem->mm, order);
115 			if (!IS_ERR(block))
116 				break;
117 
118 			if (order-- == min_order)
119 				goto err_free_blocks;
120 		} while (1);
121 
122 		n_pages -= BIT(order);
123 
124 		block->private = mem;
125 		list_add_tail(&block->link, blocks);
126 
127 		if (!n_pages)
128 			break;
129 	} while (1);
130 
131 	mem->avail -= size;
132 	mutex_unlock(&mem->mm_lock);
133 	return 0;
134 
135 err_free_blocks:
136 	intel_memory_region_free_pages(mem, blocks);
137 	mutex_unlock(&mem->mm_lock);
138 	return -ENXIO;
139 }
140 
141 struct i915_buddy_block *
142 __intel_memory_region_get_block_buddy(struct intel_memory_region *mem,
143 				      resource_size_t size,
144 				      unsigned int flags)
145 {
146 	struct i915_buddy_block *block;
147 	LIST_HEAD(blocks);
148 	int ret;
149 
150 	ret = __intel_memory_region_get_pages_buddy(mem, size, flags, &blocks);
151 	if (ret)
152 		return ERR_PTR(ret);
153 
154 	block = list_first_entry(&blocks, typeof(*block), link);
155 	list_del_init(&block->link);
156 	return block;
157 }
158 
159 int intel_memory_region_init_buddy(struct intel_memory_region *mem)
160 {
161 	return i915_buddy_init(&mem->mm, resource_size(&mem->region),
162 			       PAGE_SIZE);
163 }
164 
165 void intel_memory_region_release_buddy(struct intel_memory_region *mem)
166 {
167 	i915_buddy_free_list(&mem->mm, &mem->reserved);
168 	i915_buddy_fini(&mem->mm);
169 }
170 
171 int intel_memory_region_reserve(struct intel_memory_region *mem,
172 				u64 offset, u64 size)
173 {
174 	int ret;
175 
176 	mutex_lock(&mem->mm_lock);
177 	ret = i915_buddy_alloc_range(&mem->mm, &mem->reserved, offset, size);
178 	mutex_unlock(&mem->mm_lock);
179 
180 	return ret;
181 }
182 
183 struct intel_memory_region *
184 intel_memory_region_create(struct drm_i915_private *i915,
185 			   resource_size_t start,
186 			   resource_size_t size,
187 			   resource_size_t min_page_size,
188 			   resource_size_t io_start,
189 			   const struct intel_memory_region_ops *ops)
190 {
191 	struct intel_memory_region *mem;
192 	int err;
193 
194 	mem = kzalloc(sizeof(*mem), GFP_KERNEL);
195 	if (!mem)
196 		return ERR_PTR(-ENOMEM);
197 
198 	mem->i915 = i915;
199 	mem->region = (struct resource)DEFINE_RES_MEM(start, size);
200 	mem->io_start = io_start;
201 	mem->min_page_size = min_page_size;
202 	mem->ops = ops;
203 	mem->total = size;
204 	mem->avail = mem->total;
205 
206 	mutex_init(&mem->objects.lock);
207 	INIT_LIST_HEAD(&mem->objects.list);
208 	INIT_LIST_HEAD(&mem->objects.purgeable);
209 	INIT_LIST_HEAD(&mem->reserved);
210 
211 	mutex_init(&mem->mm_lock);
212 
213 	if (ops->init) {
214 		err = ops->init(mem);
215 		if (err)
216 			goto err_free;
217 	}
218 
219 	kref_init(&mem->kref);
220 	return mem;
221 
222 err_free:
223 	kfree(mem);
224 	return ERR_PTR(err);
225 }
226 
227 void intel_memory_region_set_name(struct intel_memory_region *mem,
228 				  const char *fmt, ...)
229 {
230 	va_list ap;
231 
232 	va_start(ap, fmt);
233 	vsnprintf(mem->name, sizeof(mem->name), fmt, ap);
234 	va_end(ap);
235 }
236 
237 static void __intel_memory_region_destroy(struct kref *kref)
238 {
239 	struct intel_memory_region *mem =
240 		container_of(kref, typeof(*mem), kref);
241 
242 	if (mem->ops->release)
243 		mem->ops->release(mem);
244 
245 	mutex_destroy(&mem->mm_lock);
246 	mutex_destroy(&mem->objects.lock);
247 	kfree(mem);
248 }
249 
250 struct intel_memory_region *
251 intel_memory_region_get(struct intel_memory_region *mem)
252 {
253 	kref_get(&mem->kref);
254 	return mem;
255 }
256 
257 void intel_memory_region_put(struct intel_memory_region *mem)
258 {
259 	kref_put(&mem->kref, __intel_memory_region_destroy);
260 }
261 
262 /* Global memory region registration -- only slight layer inversions! */
263 
264 int intel_memory_regions_hw_probe(struct drm_i915_private *i915)
265 {
266 	int err, i;
267 
268 	for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) {
269 		struct intel_memory_region *mem = ERR_PTR(-ENODEV);
270 		u16 type, instance;
271 
272 		if (!HAS_REGION(i915, BIT(i)))
273 			continue;
274 
275 		type = intel_region_map[i].class;
276 		instance = intel_region_map[i].instance;
277 		switch (type) {
278 		case INTEL_MEMORY_SYSTEM:
279 			mem = i915_gem_shmem_setup(i915);
280 			break;
281 		case INTEL_MEMORY_STOLEN_SYSTEM:
282 			mem = i915_gem_stolen_setup(i915);
283 			break;
284 		default:
285 			continue;
286 		}
287 
288 		if (IS_ERR(mem)) {
289 			err = PTR_ERR(mem);
290 			drm_err(&i915->drm,
291 				"Failed to setup region(%d) type=%d\n",
292 				err, type);
293 			goto out_cleanup;
294 		}
295 
296 		mem->id = i;
297 		mem->type = type;
298 		mem->instance = instance;
299 
300 		i915->mm.regions[i] = mem;
301 	}
302 
303 	return 0;
304 
305 out_cleanup:
306 	intel_memory_regions_driver_release(i915);
307 	return err;
308 }
309 
310 void intel_memory_regions_driver_release(struct drm_i915_private *i915)
311 {
312 	int i;
313 
314 	for (i = 0; i < ARRAY_SIZE(i915->mm.regions); i++) {
315 		struct intel_memory_region *region =
316 			fetch_and_zero(&i915->mm.regions[i]);
317 
318 		if (region)
319 			intel_memory_region_put(region);
320 	}
321 }
322 
323 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
324 #include "selftests/intel_memory_region.c"
325 #include "selftests/mock_region.c"
326 #endif
327