1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */ 2 /************************************************************************** 3 * 4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA 5 * All Rights Reserved. 6 * 7 * Permission is hereby granted, free of charge, to any person obtaining a 8 * copy of this software and associated documentation files (the 9 * "Software"), to deal in the Software without restriction, including 10 * without limitation the rights to use, copy, modify, merge, publish, 11 * distribute, sub license, and/or sell copies of the Software, and to 12 * permit persons to whom the Software is furnished to do so, subject to 13 * the following conditions: 14 * 15 * The above copyright notice and this permission notice (including the 16 * next paragraph) shall be included in all copies or substantial portions 17 * of the Software. 18 * 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25 * USE OR OTHER DEALINGS IN THE SOFTWARE. 26 * 27 **************************************************************************/ 28 /* 29 * Authors: Thomas Hellstrom <thellstrom-at-vmware-dot-com> 30 */ 31 32 #define pr_fmt(fmt) "[TTM] " fmt 33 34 #include <linux/sched.h> 35 #include <linux/shmem_fs.h> 36 #include <linux/file.h> 37 #include <drm/drm_cache.h> 38 #include <drm/ttm/ttm_bo_driver.h> 39 40 #include "ttm_module.h" 41 42 static unsigned long ttm_pages_limit; 43 44 MODULE_PARM_DESC(pages_limit, "Limit for the allocated pages"); 45 module_param_named(pages_limit, ttm_pages_limit, ulong, 0644); 46 47 static unsigned long ttm_dma32_pages_limit; 48 49 MODULE_PARM_DESC(dma32_pages_limit, "Limit for the allocated DMA32 pages"); 50 module_param_named(dma32_pages_limit, ttm_dma32_pages_limit, ulong, 0644); 51 52 static atomic_long_t ttm_pages_allocated; 53 static atomic_long_t ttm_dma32_pages_allocated; 54 55 /* 56 * Allocates a ttm structure for the given BO. 57 */ 58 int ttm_tt_create(struct ttm_buffer_object *bo, bool zero_alloc) 59 { 60 struct ttm_device *bdev = bo->bdev; 61 uint32_t page_flags = 0; 62 63 dma_resv_assert_held(bo->base.resv); 64 65 if (bo->ttm) 66 return 0; 67 68 switch (bo->type) { 69 case ttm_bo_type_device: 70 if (zero_alloc) 71 page_flags |= TTM_TT_FLAG_ZERO_ALLOC; 72 break; 73 case ttm_bo_type_kernel: 74 break; 75 case ttm_bo_type_sg: 76 page_flags |= TTM_TT_FLAG_EXTERNAL; 77 break; 78 default: 79 pr_err("Illegal buffer object type\n"); 80 return -EINVAL; 81 } 82 83 bo->ttm = bdev->funcs->ttm_tt_create(bo, page_flags); 84 if (unlikely(bo->ttm == NULL)) 85 return -ENOMEM; 86 87 WARN_ON(bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL_MAPPABLE && 88 !(bo->ttm->page_flags & TTM_TT_FLAG_EXTERNAL)); 89 90 return 0; 91 } 92 93 /* 94 * Allocates storage for pointers to the pages that back the ttm. 95 */ 96 static int ttm_tt_alloc_page_directory(struct ttm_tt *ttm) 97 { 98 ttm->pages = kvmalloc_array(ttm->num_pages, sizeof(void*), 99 GFP_KERNEL | __GFP_ZERO); 100 if (!ttm->pages) 101 return -ENOMEM; 102 return 0; 103 } 104 105 static int ttm_dma_tt_alloc_page_directory(struct ttm_tt *ttm) 106 { 107 ttm->pages = kvmalloc_array(ttm->num_pages, 108 sizeof(*ttm->pages) + 109 sizeof(*ttm->dma_address), 110 GFP_KERNEL | __GFP_ZERO); 111 if (!ttm->pages) 112 return -ENOMEM; 113 114 ttm->dma_address = (void *)(ttm->pages + ttm->num_pages); 115 return 0; 116 } 117 118 static int ttm_sg_tt_alloc_page_directory(struct ttm_tt *ttm) 119 { 120 ttm->dma_address = kvmalloc_array(ttm->num_pages, 121 sizeof(*ttm->dma_address), 122 GFP_KERNEL | __GFP_ZERO); 123 if (!ttm->dma_address) 124 return -ENOMEM; 125 return 0; 126 } 127 128 void ttm_tt_destroy(struct ttm_device *bdev, struct ttm_tt *ttm) 129 { 130 bdev->funcs->ttm_tt_destroy(bdev, ttm); 131 } 132 133 static void ttm_tt_init_fields(struct ttm_tt *ttm, 134 struct ttm_buffer_object *bo, 135 uint32_t page_flags, 136 enum ttm_caching caching) 137 { 138 ttm->num_pages = PAGE_ALIGN(bo->base.size) >> PAGE_SHIFT; 139 ttm->caching = ttm_cached; 140 ttm->page_flags = page_flags; 141 ttm->dma_address = NULL; 142 ttm->swap_storage = NULL; 143 ttm->sg = bo->sg; 144 ttm->caching = caching; 145 } 146 147 int ttm_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, 148 uint32_t page_flags, enum ttm_caching caching) 149 { 150 ttm_tt_init_fields(ttm, bo, page_flags, caching); 151 152 if (ttm_tt_alloc_page_directory(ttm)) { 153 pr_err("Failed allocating page table\n"); 154 return -ENOMEM; 155 } 156 return 0; 157 } 158 EXPORT_SYMBOL(ttm_tt_init); 159 160 void ttm_tt_fini(struct ttm_tt *ttm) 161 { 162 WARN_ON(ttm->page_flags & TTM_TT_FLAG_PRIV_POPULATED); 163 164 if (ttm->swap_storage) 165 fput(ttm->swap_storage); 166 ttm->swap_storage = NULL; 167 168 if (ttm->pages) 169 kvfree(ttm->pages); 170 else 171 kvfree(ttm->dma_address); 172 ttm->pages = NULL; 173 ttm->dma_address = NULL; 174 } 175 EXPORT_SYMBOL(ttm_tt_fini); 176 177 int ttm_sg_tt_init(struct ttm_tt *ttm, struct ttm_buffer_object *bo, 178 uint32_t page_flags, enum ttm_caching caching) 179 { 180 int ret; 181 182 ttm_tt_init_fields(ttm, bo, page_flags, caching); 183 184 if (page_flags & TTM_TT_FLAG_EXTERNAL) 185 ret = ttm_sg_tt_alloc_page_directory(ttm); 186 else 187 ret = ttm_dma_tt_alloc_page_directory(ttm); 188 if (ret) { 189 pr_err("Failed allocating page table\n"); 190 return -ENOMEM; 191 } 192 return 0; 193 } 194 EXPORT_SYMBOL(ttm_sg_tt_init); 195 196 int ttm_tt_swapin(struct ttm_tt *ttm) 197 { 198 struct address_space *swap_space; 199 struct file *swap_storage; 200 struct page *from_page; 201 struct page *to_page; 202 gfp_t gfp_mask; 203 int i, ret; 204 205 swap_storage = ttm->swap_storage; 206 BUG_ON(swap_storage == NULL); 207 208 swap_space = swap_storage->f_mapping; 209 gfp_mask = mapping_gfp_mask(swap_space); 210 211 for (i = 0; i < ttm->num_pages; ++i) { 212 from_page = shmem_read_mapping_page_gfp(swap_space, i, 213 gfp_mask); 214 if (IS_ERR(from_page)) { 215 ret = PTR_ERR(from_page); 216 goto out_err; 217 } 218 to_page = ttm->pages[i]; 219 if (unlikely(to_page == NULL)) { 220 ret = -ENOMEM; 221 goto out_err; 222 } 223 224 copy_highpage(to_page, from_page); 225 put_page(from_page); 226 } 227 228 fput(swap_storage); 229 ttm->swap_storage = NULL; 230 ttm->page_flags &= ~TTM_TT_FLAG_SWAPPED; 231 232 return 0; 233 234 out_err: 235 return ret; 236 } 237 238 /** 239 * ttm_tt_swapout - swap out tt object 240 * 241 * @bdev: TTM device structure. 242 * @ttm: The struct ttm_tt. 243 * @gfp_flags: Flags to use for memory allocation. 244 * 245 * Swapout a TT object to a shmem_file, return number of pages swapped out or 246 * negative error code. 247 */ 248 int ttm_tt_swapout(struct ttm_device *bdev, struct ttm_tt *ttm, 249 gfp_t gfp_flags) 250 { 251 loff_t size = (loff_t)ttm->num_pages << PAGE_SHIFT; 252 struct address_space *swap_space; 253 struct file *swap_storage; 254 struct page *from_page; 255 struct page *to_page; 256 int i, ret; 257 258 swap_storage = shmem_file_setup("ttm swap", size, 0); 259 if (IS_ERR(swap_storage)) { 260 pr_err("Failed allocating swap storage\n"); 261 return PTR_ERR(swap_storage); 262 } 263 264 swap_space = swap_storage->f_mapping; 265 gfp_flags &= mapping_gfp_mask(swap_space); 266 267 for (i = 0; i < ttm->num_pages; ++i) { 268 from_page = ttm->pages[i]; 269 if (unlikely(from_page == NULL)) 270 continue; 271 272 to_page = shmem_read_mapping_page_gfp(swap_space, i, gfp_flags); 273 if (IS_ERR(to_page)) { 274 ret = PTR_ERR(to_page); 275 goto out_err; 276 } 277 copy_highpage(to_page, from_page); 278 set_page_dirty(to_page); 279 mark_page_accessed(to_page); 280 put_page(to_page); 281 } 282 283 ttm_tt_unpopulate(bdev, ttm); 284 ttm->swap_storage = swap_storage; 285 ttm->page_flags |= TTM_TT_FLAG_SWAPPED; 286 287 return ttm->num_pages; 288 289 out_err: 290 fput(swap_storage); 291 292 return ret; 293 } 294 295 int ttm_tt_populate(struct ttm_device *bdev, 296 struct ttm_tt *ttm, struct ttm_operation_ctx *ctx) 297 { 298 int ret; 299 300 if (!ttm) 301 return -EINVAL; 302 303 if (ttm_tt_is_populated(ttm)) 304 return 0; 305 306 if (!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { 307 atomic_long_add(ttm->num_pages, &ttm_pages_allocated); 308 if (bdev->pool.use_dma32) 309 atomic_long_add(ttm->num_pages, 310 &ttm_dma32_pages_allocated); 311 } 312 313 while (atomic_long_read(&ttm_pages_allocated) > ttm_pages_limit || 314 atomic_long_read(&ttm_dma32_pages_allocated) > 315 ttm_dma32_pages_limit) { 316 317 ret = ttm_global_swapout(ctx, GFP_KERNEL); 318 if (ret == 0) 319 break; 320 if (ret < 0) 321 goto error; 322 } 323 324 if (bdev->funcs->ttm_tt_populate) 325 ret = bdev->funcs->ttm_tt_populate(bdev, ttm, ctx); 326 else 327 ret = ttm_pool_alloc(&bdev->pool, ttm, ctx); 328 if (ret) 329 goto error; 330 331 ttm->page_flags |= TTM_TT_FLAG_PRIV_POPULATED; 332 if (unlikely(ttm->page_flags & TTM_TT_FLAG_SWAPPED)) { 333 ret = ttm_tt_swapin(ttm); 334 if (unlikely(ret != 0)) { 335 ttm_tt_unpopulate(bdev, ttm); 336 return ret; 337 } 338 } 339 340 return 0; 341 342 error: 343 if (!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { 344 atomic_long_sub(ttm->num_pages, &ttm_pages_allocated); 345 if (bdev->pool.use_dma32) 346 atomic_long_sub(ttm->num_pages, 347 &ttm_dma32_pages_allocated); 348 } 349 return ret; 350 } 351 EXPORT_SYMBOL(ttm_tt_populate); 352 353 void ttm_tt_unpopulate(struct ttm_device *bdev, struct ttm_tt *ttm) 354 { 355 if (!ttm_tt_is_populated(ttm)) 356 return; 357 358 if (bdev->funcs->ttm_tt_unpopulate) 359 bdev->funcs->ttm_tt_unpopulate(bdev, ttm); 360 else 361 ttm_pool_free(&bdev->pool, ttm); 362 363 if (!(ttm->page_flags & TTM_TT_FLAG_EXTERNAL)) { 364 atomic_long_sub(ttm->num_pages, &ttm_pages_allocated); 365 if (bdev->pool.use_dma32) 366 atomic_long_sub(ttm->num_pages, 367 &ttm_dma32_pages_allocated); 368 } 369 370 ttm->page_flags &= ~TTM_TT_FLAG_PRIV_POPULATED; 371 } 372 373 #ifdef CONFIG_DEBUG_FS 374 375 /* Test the shrinker functions and dump the result */ 376 static int ttm_tt_debugfs_shrink_show(struct seq_file *m, void *data) 377 { 378 struct ttm_operation_ctx ctx = { false, false }; 379 380 seq_printf(m, "%d\n", ttm_global_swapout(&ctx, GFP_KERNEL)); 381 return 0; 382 } 383 DEFINE_SHOW_ATTRIBUTE(ttm_tt_debugfs_shrink); 384 385 #endif 386 387 388 /* 389 * ttm_tt_mgr_init - register with the MM shrinker 390 * 391 * Register with the MM shrinker for swapping out BOs. 392 */ 393 void ttm_tt_mgr_init(unsigned long num_pages, unsigned long num_dma32_pages) 394 { 395 #ifdef CONFIG_DEBUG_FS 396 debugfs_create_file("tt_shrink", 0400, ttm_debugfs_root, NULL, 397 &ttm_tt_debugfs_shrink_fops); 398 #endif 399 400 if (!ttm_pages_limit) 401 ttm_pages_limit = num_pages; 402 403 if (!ttm_dma32_pages_limit) 404 ttm_dma32_pages_limit = num_dma32_pages; 405 } 406 407 static void ttm_kmap_iter_tt_map_local(struct ttm_kmap_iter *iter, 408 struct dma_buf_map *dmap, 409 pgoff_t i) 410 { 411 struct ttm_kmap_iter_tt *iter_tt = 412 container_of(iter, typeof(*iter_tt), base); 413 414 dma_buf_map_set_vaddr(dmap, kmap_local_page_prot(iter_tt->tt->pages[i], 415 iter_tt->prot)); 416 } 417 418 static void ttm_kmap_iter_tt_unmap_local(struct ttm_kmap_iter *iter, 419 struct dma_buf_map *map) 420 { 421 kunmap_local(map->vaddr); 422 } 423 424 static const struct ttm_kmap_iter_ops ttm_kmap_iter_tt_ops = { 425 .map_local = ttm_kmap_iter_tt_map_local, 426 .unmap_local = ttm_kmap_iter_tt_unmap_local, 427 .maps_tt = true, 428 }; 429 430 /** 431 * ttm_kmap_iter_tt_init - Initialize a struct ttm_kmap_iter_tt 432 * @iter_tt: The struct ttm_kmap_iter_tt to initialize. 433 * @tt: Struct ttm_tt holding page pointers of the struct ttm_resource. 434 * 435 * Return: Pointer to the embedded struct ttm_kmap_iter. 436 */ 437 struct ttm_kmap_iter * 438 ttm_kmap_iter_tt_init(struct ttm_kmap_iter_tt *iter_tt, 439 struct ttm_tt *tt) 440 { 441 iter_tt->base.ops = &ttm_kmap_iter_tt_ops; 442 iter_tt->tt = tt; 443 if (tt) 444 iter_tt->prot = ttm_prot_from_caching(tt->caching, PAGE_KERNEL); 445 else 446 iter_tt->prot = PAGE_KERNEL; 447 448 return &iter_tt->base; 449 } 450 EXPORT_SYMBOL(ttm_kmap_iter_tt_init); 451