1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include <linux/slab.h> 7 8 #include <drm/ttm/ttm_placement.h> 9 #include <drm/ttm/ttm_bo.h> 10 11 #include <drm/drm_buddy.h> 12 13 #include "i915_ttm_buddy_manager.h" 14 15 #include "i915_gem.h" 16 17 struct i915_ttm_buddy_manager { 18 struct ttm_resource_manager manager; 19 struct drm_buddy mm; 20 struct list_head reserved; 21 struct mutex lock; 22 unsigned long visible_size; 23 unsigned long visible_avail; 24 unsigned long visible_reserved; 25 u64 default_page_size; 26 }; 27 28 static struct i915_ttm_buddy_manager * 29 to_buddy_manager(struct ttm_resource_manager *man) 30 { 31 return container_of(man, struct i915_ttm_buddy_manager, manager); 32 } 33 34 static int i915_ttm_buddy_man_alloc(struct ttm_resource_manager *man, 35 struct ttm_buffer_object *bo, 36 const struct ttm_place *place, 37 struct ttm_resource **res) 38 { 39 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 40 struct i915_ttm_buddy_resource *bman_res; 41 struct drm_buddy *mm = &bman->mm; 42 unsigned long n_pages, lpfn; 43 u64 min_page_size; 44 u64 size; 45 int err; 46 47 lpfn = place->lpfn; 48 if (!lpfn) 49 lpfn = man->size; 50 51 bman_res = kzalloc(sizeof(*bman_res), GFP_KERNEL); 52 if (!bman_res) 53 return -ENOMEM; 54 55 ttm_resource_init(bo, place, &bman_res->base); 56 INIT_LIST_HEAD(&bman_res->blocks); 57 bman_res->mm = mm; 58 59 if (place->flags & TTM_PL_FLAG_TOPDOWN) 60 bman_res->flags |= DRM_BUDDY_TOPDOWN_ALLOCATION; 61 62 if (place->fpfn || lpfn != man->size) 63 bman_res->flags |= DRM_BUDDY_RANGE_ALLOCATION; 64 65 GEM_BUG_ON(!bman_res->base.size); 66 size = bman_res->base.size; 67 68 min_page_size = bman->default_page_size; 69 if (bo->page_alignment) 70 min_page_size = bo->page_alignment << PAGE_SHIFT; 71 72 GEM_BUG_ON(min_page_size < mm->chunk_size); 73 GEM_BUG_ON(!IS_ALIGNED(size, min_page_size)); 74 75 if (place->fpfn + PFN_UP(bman_res->base.size) != place->lpfn && 76 place->flags & TTM_PL_FLAG_CONTIGUOUS) { 77 unsigned long pages; 78 79 size = roundup_pow_of_two(size); 80 min_page_size = size; 81 82 pages = size >> ilog2(mm->chunk_size); 83 if (pages > lpfn) 84 lpfn = pages; 85 } 86 87 if (size > lpfn << PAGE_SHIFT) { 88 err = -E2BIG; 89 goto err_free_res; 90 } 91 92 n_pages = size >> ilog2(mm->chunk_size); 93 94 mutex_lock(&bman->lock); 95 if (lpfn <= bman->visible_size && n_pages > bman->visible_avail) { 96 mutex_unlock(&bman->lock); 97 err = -ENOSPC; 98 goto err_free_res; 99 } 100 101 err = drm_buddy_alloc_blocks(mm, (u64)place->fpfn << PAGE_SHIFT, 102 (u64)lpfn << PAGE_SHIFT, 103 (u64)n_pages << PAGE_SHIFT, 104 min_page_size, 105 &bman_res->blocks, 106 bman_res->flags); 107 if (unlikely(err)) 108 goto err_free_blocks; 109 110 if (place->flags & TTM_PL_FLAG_CONTIGUOUS) { 111 u64 original_size = (u64)bman_res->base.size; 112 113 drm_buddy_block_trim(mm, 114 original_size, 115 &bman_res->blocks); 116 } 117 118 if (lpfn <= bman->visible_size) { 119 bman_res->used_visible_size = PFN_UP(bman_res->base.size); 120 } else { 121 struct drm_buddy_block *block; 122 123 list_for_each_entry(block, &bman_res->blocks, link) { 124 unsigned long start = 125 drm_buddy_block_offset(block) >> PAGE_SHIFT; 126 127 if (start < bman->visible_size) { 128 unsigned long end = start + 129 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 130 131 bman_res->used_visible_size += 132 min(end, bman->visible_size) - start; 133 } 134 } 135 } 136 137 if (bman_res->used_visible_size) 138 bman->visible_avail -= bman_res->used_visible_size; 139 140 mutex_unlock(&bman->lock); 141 142 *res = &bman_res->base; 143 return 0; 144 145 err_free_blocks: 146 drm_buddy_free_list(mm, &bman_res->blocks); 147 mutex_unlock(&bman->lock); 148 err_free_res: 149 ttm_resource_fini(man, &bman_res->base); 150 kfree(bman_res); 151 return err; 152 } 153 154 static void i915_ttm_buddy_man_free(struct ttm_resource_manager *man, 155 struct ttm_resource *res) 156 { 157 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); 158 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 159 160 mutex_lock(&bman->lock); 161 drm_buddy_free_list(&bman->mm, &bman_res->blocks); 162 bman->visible_avail += bman_res->used_visible_size; 163 mutex_unlock(&bman->lock); 164 165 ttm_resource_fini(man, res); 166 kfree(bman_res); 167 } 168 169 static bool i915_ttm_buddy_man_intersects(struct ttm_resource_manager *man, 170 struct ttm_resource *res, 171 const struct ttm_place *place, 172 size_t size) 173 { 174 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); 175 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 176 struct drm_buddy *mm = &bman->mm; 177 struct drm_buddy_block *block; 178 179 if (!place->fpfn && !place->lpfn) 180 return true; 181 182 GEM_BUG_ON(!place->lpfn); 183 184 /* 185 * If we just want something mappable then we can quickly check 186 * if the current victim resource is using any of the CPU 187 * visible portion. 188 */ 189 if (!place->fpfn && 190 place->lpfn == i915_ttm_buddy_man_visible_size(man)) 191 return bman_res->used_visible_size > 0; 192 193 /* Check each drm buddy block individually */ 194 list_for_each_entry(block, &bman_res->blocks, link) { 195 unsigned long fpfn = 196 drm_buddy_block_offset(block) >> PAGE_SHIFT; 197 unsigned long lpfn = fpfn + 198 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 199 200 if (place->fpfn < lpfn && place->lpfn > fpfn) 201 return true; 202 } 203 204 return false; 205 } 206 207 static bool i915_ttm_buddy_man_compatible(struct ttm_resource_manager *man, 208 struct ttm_resource *res, 209 const struct ttm_place *place, 210 size_t size) 211 { 212 struct i915_ttm_buddy_resource *bman_res = to_ttm_buddy_resource(res); 213 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 214 struct drm_buddy *mm = &bman->mm; 215 struct drm_buddy_block *block; 216 217 if (!place->fpfn && !place->lpfn) 218 return true; 219 220 GEM_BUG_ON(!place->lpfn); 221 222 if (!place->fpfn && 223 place->lpfn == i915_ttm_buddy_man_visible_size(man)) 224 return bman_res->used_visible_size == PFN_UP(res->size); 225 226 /* Check each drm buddy block individually */ 227 list_for_each_entry(block, &bman_res->blocks, link) { 228 unsigned long fpfn = 229 drm_buddy_block_offset(block) >> PAGE_SHIFT; 230 unsigned long lpfn = fpfn + 231 (drm_buddy_block_size(mm, block) >> PAGE_SHIFT); 232 233 if (fpfn < place->fpfn || lpfn > place->lpfn) 234 return false; 235 } 236 237 return true; 238 } 239 240 static void i915_ttm_buddy_man_debug(struct ttm_resource_manager *man, 241 struct drm_printer *printer) 242 { 243 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 244 struct drm_buddy_block *block; 245 246 mutex_lock(&bman->lock); 247 drm_printf(printer, "default_page_size: %lluKiB\n", 248 bman->default_page_size >> 10); 249 drm_printf(printer, "visible_avail: %lluMiB\n", 250 (u64)bman->visible_avail << PAGE_SHIFT >> 20); 251 drm_printf(printer, "visible_size: %lluMiB\n", 252 (u64)bman->visible_size << PAGE_SHIFT >> 20); 253 drm_printf(printer, "visible_reserved: %lluMiB\n", 254 (u64)bman->visible_reserved << PAGE_SHIFT >> 20); 255 256 drm_buddy_print(&bman->mm, printer); 257 258 drm_printf(printer, "reserved:\n"); 259 list_for_each_entry(block, &bman->reserved, link) 260 drm_buddy_block_print(&bman->mm, block, printer); 261 mutex_unlock(&bman->lock); 262 } 263 264 static const struct ttm_resource_manager_func i915_ttm_buddy_manager_func = { 265 .alloc = i915_ttm_buddy_man_alloc, 266 .free = i915_ttm_buddy_man_free, 267 .intersects = i915_ttm_buddy_man_intersects, 268 .compatible = i915_ttm_buddy_man_compatible, 269 .debug = i915_ttm_buddy_man_debug, 270 }; 271 272 /** 273 * i915_ttm_buddy_man_init - Setup buddy allocator based ttm manager 274 * @bdev: The ttm device 275 * @type: Memory type we want to manage 276 * @use_tt: Set use_tt for the manager 277 * @size: The size in bytes to manage 278 * @visible_size: The CPU visible size in bytes to manage 279 * @default_page_size: The default minimum page size in bytes for allocations, 280 * this must be at least as large as @chunk_size, and can be overridden by 281 * setting the BO page_alignment, to be larger or smaller as needed. 282 * @chunk_size: The minimum page size in bytes for our allocations i.e 283 * order-zero 284 * 285 * Note that the starting address is assumed to be zero here, since this 286 * simplifies keeping the property where allocated blocks having natural 287 * power-of-two alignment. So long as the real starting address is some large 288 * power-of-two, or naturally start from zero, then this should be fine. Also 289 * the &i915_ttm_buddy_man_reserve interface can be used to preserve alignment 290 * if say there is some unusable range from the start of the region. We can 291 * revisit this in the future and make the interface accept an actual starting 292 * offset and let it take care of the rest. 293 * 294 * Note that if the @size is not aligned to the @chunk_size then we perform the 295 * required rounding to get the usable size. The final size in pages can be 296 * taken from &ttm_resource_manager.size. 297 * 298 * Return: 0 on success, negative error code on failure. 299 */ 300 int i915_ttm_buddy_man_init(struct ttm_device *bdev, 301 unsigned int type, bool use_tt, 302 u64 size, u64 visible_size, u64 default_page_size, 303 u64 chunk_size) 304 { 305 struct ttm_resource_manager *man; 306 struct i915_ttm_buddy_manager *bman; 307 int err; 308 309 bman = kzalloc(sizeof(*bman), GFP_KERNEL); 310 if (!bman) 311 return -ENOMEM; 312 313 err = drm_buddy_init(&bman->mm, size, chunk_size); 314 if (err) 315 goto err_free_bman; 316 317 mutex_init(&bman->lock); 318 INIT_LIST_HEAD(&bman->reserved); 319 GEM_BUG_ON(default_page_size < chunk_size); 320 bman->default_page_size = default_page_size; 321 bman->visible_size = visible_size >> PAGE_SHIFT; 322 bman->visible_avail = bman->visible_size; 323 324 man = &bman->manager; 325 man->use_tt = use_tt; 326 man->func = &i915_ttm_buddy_manager_func; 327 ttm_resource_manager_init(man, bdev, bman->mm.size >> PAGE_SHIFT); 328 329 ttm_resource_manager_set_used(man, true); 330 ttm_set_driver_manager(bdev, type, man); 331 332 return 0; 333 334 err_free_bman: 335 kfree(bman); 336 return err; 337 } 338 339 /** 340 * i915_ttm_buddy_man_fini - Destroy the buddy allocator ttm manager 341 * @bdev: The ttm device 342 * @type: Memory type we want to manage 343 * 344 * Note that if we reserved anything with &i915_ttm_buddy_man_reserve, this will 345 * also be freed for us here. 346 * 347 * Return: 0 on success, negative error code on failure. 348 */ 349 int i915_ttm_buddy_man_fini(struct ttm_device *bdev, unsigned int type) 350 { 351 struct ttm_resource_manager *man = ttm_manager_type(bdev, type); 352 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 353 struct drm_buddy *mm = &bman->mm; 354 int ret; 355 356 ttm_resource_manager_set_used(man, false); 357 358 ret = ttm_resource_manager_evict_all(bdev, man); 359 if (ret) 360 return ret; 361 362 ttm_set_driver_manager(bdev, type, NULL); 363 364 mutex_lock(&bman->lock); 365 drm_buddy_free_list(mm, &bman->reserved); 366 drm_buddy_fini(mm); 367 bman->visible_avail += bman->visible_reserved; 368 WARN_ON_ONCE(bman->visible_avail != bman->visible_size); 369 mutex_unlock(&bman->lock); 370 371 ttm_resource_manager_cleanup(man); 372 kfree(bman); 373 374 return 0; 375 } 376 377 /** 378 * i915_ttm_buddy_man_reserve - Reserve address range 379 * @man: The buddy allocator ttm manager 380 * @start: The offset in bytes, where the region start is assumed to be zero 381 * @size: The size in bytes 382 * 383 * Note that the starting address for the region is always assumed to be zero. 384 * 385 * Return: 0 on success, negative error code on failure. 386 */ 387 int i915_ttm_buddy_man_reserve(struct ttm_resource_manager *man, 388 u64 start, u64 size) 389 { 390 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 391 struct drm_buddy *mm = &bman->mm; 392 unsigned long fpfn = start >> PAGE_SHIFT; 393 unsigned long flags = 0; 394 int ret; 395 396 flags |= DRM_BUDDY_RANGE_ALLOCATION; 397 398 mutex_lock(&bman->lock); 399 ret = drm_buddy_alloc_blocks(mm, start, 400 start + size, 401 size, mm->chunk_size, 402 &bman->reserved, 403 flags); 404 405 if (fpfn < bman->visible_size) { 406 unsigned long lpfn = fpfn + (size >> PAGE_SHIFT); 407 unsigned long visible = min(lpfn, bman->visible_size) - fpfn; 408 409 bman->visible_reserved += visible; 410 bman->visible_avail -= visible; 411 } 412 mutex_unlock(&bman->lock); 413 414 return ret; 415 } 416 417 /** 418 * i915_ttm_buddy_man_visible_size - Return the size of the CPU visible portion 419 * in pages. 420 * @man: The buddy allocator ttm manager 421 */ 422 u64 i915_ttm_buddy_man_visible_size(struct ttm_resource_manager *man) 423 { 424 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 425 426 return bman->visible_size; 427 } 428 429 /** 430 * i915_ttm_buddy_man_avail - Query the avail tracking for the manager. 431 * 432 * @man: The buddy allocator ttm manager 433 * @avail: The total available memory in pages for the entire manager. 434 * @visible_avail: The total available memory in pages for the CPU visible 435 * portion. Note that this will always give the same value as @avail on 436 * configurations that don't have a small BAR. 437 */ 438 void i915_ttm_buddy_man_avail(struct ttm_resource_manager *man, 439 u64 *avail, u64 *visible_avail) 440 { 441 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 442 443 mutex_lock(&bman->lock); 444 *avail = bman->mm.avail >> PAGE_SHIFT; 445 *visible_avail = bman->visible_avail; 446 mutex_unlock(&bman->lock); 447 } 448 449 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST) 450 void i915_ttm_buddy_man_force_visible_size(struct ttm_resource_manager *man, 451 u64 size) 452 { 453 struct i915_ttm_buddy_manager *bman = to_buddy_manager(man); 454 455 bman->visible_size = size; 456 } 457 #endif 458