1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <linux/slab.h> 7 8 #include <drm/ttm/ttm_bo_driver.h> 9 #include <drm/ttm/ttm_placement.h> 10 11 #include <drm/drm_buddy.h> 12 13 #include "i915_ttm_buddy_manager.h" 14 15 #include "i915_gem.h" 16 17 struct i915_ttm_buddy_manager { 18 struct ttm_resource_manager manager; 19 struct drm_buddy mm; 20 struct list_head reserved; 21 struct mutex lock; 22 unsigned long visible_size; 23 unsigned long visible_avail; 24 unsigned long visible_reserved; 25 u64 default_page_size; 26 }; 27 28 static struct i915_ttm_buddy_manager * 29 to_buddy_manager(struct ttm_resource_manager *man) 30 { 31 return container_of(man, struct i915_ttm_buddy_manager, manager); 32 } 33 34 static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, 35 struct ttm_buffer_object *bo, 36 const struct ttm_place *place, 37 struct ttm_resource **res) 38 { 39 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 40 struct i915_ttm_buddy_resource *bman_res; 41 struct drm_buddy *mm = &bman->mm; 42 unsigned long n_pages, lpfn; 43 u64 min_page_size; 44 u64 size; 45 int err; 46 47 lpfn = place->lpfn; 48 if (!lpfn) 49 lpfn = man->size; 50 51 bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL); 52 if (!bman_res) 53 return -ENOMEM; 54 55 ttm_resource_init(bo, place, &bman_res->base); 56 INIT_LIST_HEAD(&bman_res->blocks); 57 bman_res->mm = mm; 58 59 if (place->flags & TTM_PL_FLAG_TOPDOWN) 60 bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; 61 62 if (place->fpfn || lpfn != man->size) 63 bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION; 64 65 GEM_BUG_ON(!bman_res->base.num_pages); 66 size = bman_res->base.num_pages << PAGE_SHIFT; 67 68 min_page_size = bman->default_page_size; 69 if (bo->page_alignment) 70 min_page_size = bo->page_alignment << PAGE_SHIFT; 71 72 GEM_BUG_ON(min_page_size < mm->chunk_size); 73 GEM_BUG_ON(!IS_ALIGNED(size, min_page_size)); 74 75 if (place->fpfn + bman_res->base.num_pages != place->lpfn && 76 place->flags & TTM_PL_FLAG_CONTIGUOUS) { 77 unsigned long pages; 78 79 size = roundup_pow_of_two(size); 80 min_page_size = size; 81 82 pages = size >> ilog2(mm->chunk_size); 83 if (pages > lpfn) 84 lpfn = pages; 85 } 86 87 if (size > lpfn << PAGE_SHIFT) { 88 err = -E2BIG; 89 goto err_free_res; 90 } 91 92 n_pages = size >> ilog2(mm->chunk_size); 93 94 mutex_lock(&bman->lock); 95 if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) { 96 mutex_unlock(&bman->lock); 97 err = -ENOSPC; 98 goto err_free_res; 99 } 100 101 err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, 102 (u64)lpfn << PAGE_SHIFT, 103 (u64)n_pages << PAGE_SHIFT, 104 min_page_size, 105 &bman_res->blocks, 106 bman_res->flags); 107 if (unlikely(err)) 108 goto err_free_blocks; 109 110 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 111 u64 original_size = (u64)bman_res->base.num_pages << PAGE_SHIFT; 112 113 drm_buddy_block_trim(mm, 114 original_size, 115 &bman_res->blocks); 116 } 117 118 if (lpfn <= bman->visible_size) { 119 bman_res->used_visible_size = bman_res->base.num_pages; 120 } else { 121 struct drm_buddy_block *block; 122 123 list_for_each_entry(block, &bman_res->blocks, link) { 124 unsigned long start = 125 drm_buddy_block_offset(block) >> PAGE_SHIFT; 126 127 if (start < bman->visible_size) { 128 unsigned long end = start + 129 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 130 131 bman_res->used_visible_size += 132 min(end, bman->visible_size) - start; 133 } 134 } 135 } 136 137 if (bman_res->used_visible_size) 138 bman->visible_avail -= bman_res->used_visible_size; 139 140 mutex_unlock(&bman->lock); 141 142 if (place->lpfn - place->fpfn == n_pages) 143 bman_res->base.start = place->fpfn; 144 else if (lpfn <= bman->visible_size) 145 bman_res->base.start = 0; 146 else 147 bman_res->base.start = bman->visible_size; 148 149 *res = &bman_res->base; 150 return 0; 151 152 err_free_blocks: 153 drm_buddy_free_list(mm, &bman_res->blocks); 154 mutex_unlock(&bman->lock); 155 err_free_res: 156 ttm_resource_fini(man, &bman_res->base); 157 kfree(bman_res); 158 return err; 159 } 160 161 static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man, 162 struct ttm_resource *res) 163 { 164 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); 165 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 166 167 mutex_lock(&bman->lock); 168 drm_buddy_free_list(&bman->mm, &bman_res->blocks); 169 bman->visible_avail += bman_res->used_visible_size; 170 mutex_unlock(&bman->lock); 171 172 ttm_resource_fini(man, res); 173 kfree(bman_res); 174 } 175 176 static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man, 177 struct drm_printer *printer) 178 { 179 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 180 struct drm_buddy_block *block; 181 182 mutex_lock(&bman->lock); 183 drm_printf(printer, "default_page_size: %lluKiB\n", 184 bman->default_page_size >> 10); 185 drm_printf(printer, "visible_avail: %lluMiB\n", 186 (u64)bman->visible_avail << PAGE_SHIFT >> 20); 187 drm_printf(printer, "visible_size: %lluMiB\n", 188 (u64)bman->visible_size << PAGE_SHIFT >> 20); 189 drm_printf(printer, "visible_reserved: %lluMiB\n", 190 (u64)bman->visible_reserved << PAGE_SHIFT >> 20); 191 192 drm_buddy_print(&bman->mm, printer); 193 194 drm_printf(printer, "reserved:\n"); 195 list_for_each_entry(block, &bman->reserved, link) 196 drm_buddy_block_print(&bman->mm, block, printer); 197 mutex_unlock(&bman->lock); 198 } 199 200 static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = { 201 .alloc = i915_ttm_buddy_man_alloc, 202 .free = i915_ttm_buddy_man_free, 203 .debug = i915_ttm_buddy_man_debug, 204 }; 205 206 /** 207 * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager 208 * @bdev: The ttm device 209 * @type: Memory type we want to manage 210 * @use_tt: Set use_tt for the manager 211 * @size: The size in bytes to manage 212 * @visible_size: The CPU visible size in bytes to manage 213 * @default_page_size: The default minimum page size in bytes for allocations, 214 * this must be at least as large as @chunk_size, and can be overridden by 215 * setting the BO page_alignment, to be larger or smaller as needed. 216 * @chunk_size: The minimum page size in bytes for our allocations i.e 217 * order-zero 218 * 219 * Note that the starting address is assumed to be zero here, since this 220 * simplifies keeping the property where allocated blocks having natural 221 * power-of-two alignment. So long as the real starting address is some large 222 * power-of-two, or naturally start from zero, then this should be fine. Also 223 * the &i915_ttm_buddy_man_reserve interface can be used to preserve alignment 224 * if say there is some unusable range from the start of the region. We can 225 * revisit this in the future and make the interface accept an actual starting 226 * offset and let it take care of the rest. 227 * 228 * Note that if the @size is not aligned to the @chunk_size then we perform the 229 * required rounding to get the usable size. The final size in pages can be 230 * taken from &ttm_resource_manager.size. 231 * 232 * Return: 0 on success, negative error code on failure. 233 */ 234 int i915_ttm_buddy_man_init(struct ttm_device *bdev, 235 unsigned int type, bool use_tt, 236 u64 size, u64 visible_size, u64 default_page_size, 237 u64 chunk_size) 238 { 239 struct ttm_resource_manager *man; 240 struct i915_ttm_buddy_manager *bman; 241 int err; 242 243 bman = kzalloc(sizeof(*bman), GFP_KERNEL); 244 if (!bman) 245 return -ENOMEM; 246 247 err = drm_buddy_init(&bman->mm, size, chunk_size); 248 if (err) 249 goto err_free_bman; 250 251 mutex_init(&bman->lock); 252 INIT_LIST_HEAD(&bman->reserved); 253 GEM_BUG_ON(default_page_size < chunk_size); 254 bman->default_page_size = default_page_size; 255 bman->visible_size = visible_size >> PAGE_SHIFT; 256 bman->visible_avail = bman->visible_size; 257 258 man = &bman->manager; 259 man->use_tt = use_tt; 260 man->func = &i915_ttm_buddy_manager_func; 261 ttm_resource_manager_init(man, bdev, bman->mm.size >> PAGE_SHIFT); 262 263 ttm_resource_manager_set_used(man, true); 264 ttm_set_driver_manager(bdev, type, man); 265 266 return 0; 267 268 err_free_bman: 269 kfree(bman); 270 return err; 271 } 272 273 /** 274 * i915_ttm_buddy_man_fini - Destroy the buddy allocator ttm manager 275 * @bdev: The ttm device 276 * @type: Memory type we want to manage 277 * 278 * Note that if we reserved anything with &i915_ttm_buddy_man_reserve, this will 279 * also be freed for us here. 280 * 281 * Return: 0 on success, negative error code on failure. 282 */ 283 int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type) 284 { 285 struct ttm_resource_manager *man = ttm_manager_type(bdev, type); 286 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 287 struct drm_buddy *mm = &bman->mm; 288 int ret; 289 290 ttm_resource_manager_set_used(man, false); 291 292 ret = ttm_resource_manager_evict_all(bdev, man); 293 if (ret) 294 return ret; 295 296 ttm_set_driver_manager(bdev, type, NULL); 297 298 mutex_lock(&bman->lock); 299 drm_buddy_free_list(mm, &bman->reserved); 300 drm_buddy_fini(mm); 301 bman->visible_avail += bman->visible_reserved; 302 WARN_ON_ONCE(bman->visible_avail != bman->visible_size); 303 mutex_unlock(&bman->lock); 304 305 ttm_resource_manager_cleanup(man); 306 kfree(bman); 307 308 return 0; 309 } 310 311 /** 312 * i915_ttm_buddy_man_reserve - Reserve address range 313 * @man: The buddy allocator ttm manager 314 * @start: The offset in bytes, where the region start is assumed to be zero 315 * @size: The size in bytes 316 * 317 * Note that the starting address for the region is always assumed to be zero. 318 * 319 * Return: 0 on success, negative error code on failure. 320 */ 321 int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, 322 u64 start, u64 size) 323 { 324 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 325 struct drm_buddy *mm = &bman->mm; 326 unsigned long fpfn = start >> PAGE_SHIFT; 327 unsigned long flags = 0; 328 int ret; 329 330 flags |= DRM_BUDDY_RANGE_ALLOCATION; 331 332 mutex_lock(&bman->lock); 333 ret = drm_buddy_alloc_blocks(mm, start, 334 start + size, 335 size, mm->chunk_size, 336 &bman->reserved, 337 flags); 338 339 if (fpfn < bman->visible_size) { 340 unsigned long lpfn = fpfn + (size >> PAGE_SHIFT); 341 unsigned long visible = min(lpfn, bman->visible_size) - fpfn; 342 343 bman->visible_reserved += visible; 344 bman->visible_avail -= visible; 345 } 346 mutex_unlock(&bman->lock); 347 348 return ret; 349 } 350 351 /** 352 * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion 353 * in pages. 354 * @man: The buddy allocator ttm manager 355 */ 356 u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man) 357 { 358 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 359 360 return bman->visible_size; 361 } 362 363 /** 364 * i915_ttm_buddy_man_avail - Query the avail tracking for the manager. 365 * 366 * @man: The buddy allocator ttm manager 367 * @avail: The total available memory in pages for the entire manager. 368 * @visible_avail: The total available memory in pages for the CPU visible 369 * portion. Note that this will always give the same value as @avail on 370 * configurations that don't have a small BAR. 371 */ 372 void i915_ttm_buddy_man_avail(struct ttm_resource_manager *man, 373 u64 *avail, u64 *visible_avail) 374 { 375 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 376 377 mutex_lock(&bman->lock); 378 *avail = bman->mm.avail >> PAGE_SHIFT; 379 *visible_avail = bman->visible_avail; 380 mutex_unlock(&bman->lock); 381 } 382 383 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 384 void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man, 385 u64 size) 386 { 387 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 388 389 bman->visible_size = size; 390 } 391 #endif 392