1 /* 2 * Copyright 2020 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #ifndef _TTM_RESOURCE_H_ 26 #define _TTM_RESOURCE_H_ 27 28 #include <linux/types.h> 29 #include <linux/list.h> 30 #include <linux/mutex.h> 31 #include <linux/iosys-map.h> 32 #include <linux/dma-fence.h> 33 34 #include <drm/drm_print.h> 35 #include <drm/ttm/ttm_caching.h> 36 #include <drm/ttm/ttm_kmap_iter.h> 37 38 #define TTM_MAX_BO_PRIORITY 4U 39 #define TTM_NUM_MEM_TYPES 8 40 41 struct ttm_device; 42 struct ttm_resource_manager; 43 struct ttm_resource; 44 struct ttm_place; 45 struct ttm_buffer_object; 46 struct ttm_placement; 47 struct iosys_map; 48 struct io_mapping; 49 struct sg_table; 50 struct scatterlist; 51 52 struct ttm_resource_manager_func { 53 /** 54 * struct ttm_resource_manager_func member alloc 55 * 56 * @man: Pointer to a memory type manager. 57 * @bo: Pointer to the buffer object we're allocating space for. 58 * @place: Placement details. 59 * @res: Resulting pointer to the ttm_resource. 60 * 61 * This function should allocate space in the memory type managed 62 * by @man. Placement details if applicable are given by @place. If 63 * successful, a filled in ttm_resource object should be returned in 64 * @res. @res::start should be set to a value identifying the beginning 65 * of the range allocated, and the function should return zero. 66 * If the manager can't fulfill the request -ENOSPC should be returned. 67 * If a system error occurred, preventing the request to be fulfilled, 68 * the function should return a negative error code. 69 * 70 * This function may not be called from within atomic context and needs 71 * to take care of its own locking to protect any data structures 72 * managing the space. 73 */ 74 int (*alloc)(struct ttm_resource_manager *man, 75 struct ttm_buffer_object *bo, 76 const struct ttm_place *place, 77 struct ttm_resource **res); 78 79 /** 80 * struct ttm_resource_manager_func member free 81 * 82 * @man: Pointer to a memory type manager. 83 * @res: Pointer to a struct ttm_resource to be freed. 84 * 85 * This function frees memory type resources previously allocated. 86 * May not be called from within atomic context. 87 */ 88 void (*free)(struct ttm_resource_manager *man, 89 struct ttm_resource *res); 90 91 /** 92 * struct ttm_resource_manager_func member intersects 93 * 94 * @man: Pointer to a memory type manager. 95 * @res: Pointer to a struct ttm_resource to be checked. 96 * @place: Placement to check against. 97 * @size: Size of the check. 98 * 99 * Test if @res intersects with @place + @size. Used to judge if 100 * evictions are valueable or not. 101 */ 102 bool (*intersects)(struct ttm_resource_manager *man, 103 struct ttm_resource *res, 104 const struct ttm_place *place, 105 size_t size); 106 107 /** 108 * struct ttm_resource_manager_func member compatible 109 * 110 * @man: Pointer to a memory type manager. 111 * @res: Pointer to a struct ttm_resource to be checked. 112 * @place: Placement to check against. 113 * @size: Size of the check. 114 * 115 * Test if @res compatible with @place + @size. Used to check of 116 * the need to move the backing store or not. 117 */ 118 bool (*compatible)(struct ttm_resource_manager *man, 119 struct ttm_resource *res, 120 const struct ttm_place *place, 121 size_t size); 122 123 /** 124 * struct ttm_resource_manager_func member debug 125 * 126 * @man: Pointer to a memory type manager. 127 * @printer: Prefix to be used in printout to identify the caller. 128 * 129 * This function is called to print out the state of the memory 130 * type manager to aid debugging of out-of-memory conditions. 131 * It may not be called from within atomic context. 132 */ 133 void (*debug)(struct ttm_resource_manager *man, 134 struct drm_printer *printer); 135 }; 136 137 /** 138 * struct ttm_resource_manager 139 * 140 * @use_type: The memory type is enabled. 141 * @use_tt: If a TT object should be used for the backing store. 142 * @size: Size of the managed region. 143 * @bdev: ttm device this manager belongs to 144 * @func: structure pointer implementing the range manager. See above 145 * @move_lock: lock for move fence 146 * @move: The fence of the last pipelined move operation. 147 * @lru: The lru list for this memory type. 148 * 149 * This structure is used to identify and manage memory types for a device. 150 */ 151 struct ttm_resource_manager { 152 /* 153 * No protection. Constant from start. 154 */ 155 bool use_type; 156 bool use_tt; 157 struct ttm_device *bdev; 158 uint64_t size; 159 const struct ttm_resource_manager_func *func; 160 spinlock_t move_lock; 161 162 /* 163 * Protected by @move_lock. 164 */ 165 struct dma_fence *move; 166 167 /* 168 * Protected by the bdev->lru_lock. 169 */ 170 struct list_head lru[TTM_MAX_BO_PRIORITY]; 171 172 /** 173 * @usage: How much of the resources are used, protected by the 174 * bdev->lru_lock. 175 */ 176 uint64_t usage; 177 }; 178 179 /** 180 * struct ttm_bus_placement 181 * 182 * @addr: mapped virtual address 183 * @offset: physical addr 184 * @is_iomem: is this io memory ? 185 * @caching: See enum ttm_caching 186 * 187 * Structure indicating the bus placement of an object. 188 */ 189 struct ttm_bus_placement { 190 void *addr; 191 phys_addr_t offset; 192 bool is_iomem; 193 enum ttm_caching caching; 194 }; 195 196 /** 197 * struct ttm_resource 198 * 199 * @start: Start of the allocation. 200 * @size: Actual size of resource in bytes. 201 * @mem_type: Resource type of the allocation. 202 * @placement: Placement flags. 203 * @bus: Placement on io bus accessible to the CPU 204 * @bo: weak reference to the BO, protected by ttm_device::lru_lock 205 * 206 * Structure indicating the placement and space resources used by a 207 * buffer object. 208 */ 209 struct ttm_resource { 210 unsigned long start; 211 size_t size; 212 uint32_t mem_type; 213 uint32_t placement; 214 struct ttm_bus_placement bus; 215 struct ttm_buffer_object *bo; 216 217 /** 218 * @lru: Least recently used list, see &ttm_resource_manager.lru 219 */ 220 struct list_head lru; 221 }; 222 223 /** 224 * struct ttm_resource_cursor 225 * 226 * @priority: the current priority 227 * 228 * Cursor to iterate over the resources in a manager. 229 */ 230 struct ttm_resource_cursor { 231 unsigned int priority; 232 }; 233 234 /** 235 * struct ttm_lru_bulk_move_pos 236 * 237 * @first: first res in the bulk move range 238 * @last: last res in the bulk move range 239 * 240 * Range of resources for a lru bulk move. 241 */ 242 struct ttm_lru_bulk_move_pos { 243 struct ttm_resource *first; 244 struct ttm_resource *last; 245 }; 246 247 /** 248 * struct ttm_lru_bulk_move 249 * 250 * @pos: first/last lru entry for resources in the each domain/priority 251 * 252 * Container for the current bulk move state. Should be used with 253 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move(). 254 */ 255 struct ttm_lru_bulk_move { 256 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; 257 }; 258 259 /** 260 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping + 261 * struct sg_table backed struct ttm_resource. 262 * @base: Embedded struct ttm_kmap_iter providing the usage interface. 263 * @iomap: struct io_mapping representing the underlying linear io_memory. 264 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource. 265 * @start: Offset that needs to be subtracted from @st to make 266 * sg_dma_address(st->sgl) - @start == 0 for @iomap start. 267 * @cache: Scatterlist traversal cache for fast lookups. 268 * @cache.sg: Pointer to the currently cached scatterlist segment. 269 * @cache.i: First index of @sg. PAGE_SIZE granularity. 270 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity. 271 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity. 272 */ 273 struct ttm_kmap_iter_iomap { 274 struct ttm_kmap_iter base; 275 struct io_mapping *iomap; 276 struct sg_table *st; 277 resource_size_t start; 278 struct { 279 struct scatterlist *sg; 280 pgoff_t i; 281 pgoff_t end; 282 pgoff_t offs; 283 } cache; 284 }; 285 286 /** 287 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io 288 * @base: The base iterator 289 * @dmap: Points to the starting address of the region 290 * @needs_unmap: Whether we need to unmap on fini 291 */ 292 struct ttm_kmap_iter_linear_io { 293 struct ttm_kmap_iter base; 294 struct iosys_map dmap; 295 bool needs_unmap; 296 }; 297 298 /** 299 * ttm_resource_manager_set_used 300 * 301 * @man: A memory manager object. 302 * @used: usage state to set. 303 * 304 * Set the manager in use flag. If disabled the manager is no longer 305 * used for object placement. 306 */ 307 static inline void 308 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used) 309 { 310 int i; 311 312 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++) 313 WARN_ON(!list_empty(&man->lru[i])); 314 man->use_type = used; 315 } 316 317 /** 318 * ttm_resource_manager_used 319 * 320 * @man: Manager to get used state for 321 * 322 * Get the in use flag for a manager. 323 * Returns: 324 * true is used, false if not. 325 */ 326 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man) 327 { 328 return man->use_type; 329 } 330 331 /** 332 * ttm_resource_manager_cleanup 333 * 334 * @man: A memory manager object. 335 * 336 * Cleanup the move fences from the memory manager object. 337 */ 338 static inline void 339 ttm_resource_manager_cleanup(struct ttm_resource_manager *man) 340 { 341 dma_fence_put(man->move); 342 man->move = NULL; 343 } 344 345 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk); 346 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk); 347 348 void ttm_resource_add_bulk_move(struct ttm_resource *res, 349 struct ttm_buffer_object *bo); 350 void ttm_resource_del_bulk_move(struct ttm_resource *res, 351 struct ttm_buffer_object *bo); 352 void ttm_resource_move_to_lru_tail(struct ttm_resource *res); 353 354 void ttm_resource_init(struct ttm_buffer_object *bo, 355 const struct ttm_place *place, 356 struct ttm_resource *res); 357 void ttm_resource_fini(struct ttm_resource_manager *man, 358 struct ttm_resource *res); 359 360 int ttm_resource_alloc(struct ttm_buffer_object *bo, 361 const struct ttm_place *place, 362 struct ttm_resource **res); 363 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); 364 bool ttm_resource_intersects(struct ttm_device *bdev, 365 struct ttm_resource *res, 366 const struct ttm_place *place, 367 size_t size); 368 bool ttm_resource_compatible(struct ttm_device *bdev, 369 struct ttm_resource *res, 370 const struct ttm_place *place, 371 size_t size); 372 bool ttm_resource_compat(struct ttm_resource *res, 373 struct ttm_placement *placement); 374 void ttm_resource_set_bo(struct ttm_resource *res, 375 struct ttm_buffer_object *bo); 376 377 void ttm_resource_manager_init(struct ttm_resource_manager *man, 378 struct ttm_device *bdev, 379 uint64_t size); 380 381 int ttm_resource_manager_evict_all(struct ttm_device *bdev, 382 struct ttm_resource_manager *man); 383 384 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man); 385 void ttm_resource_manager_debug(struct ttm_resource_manager *man, 386 struct drm_printer *p); 387 388 struct ttm_resource * 389 ttm_resource_manager_first(struct ttm_resource_manager *man, 390 struct ttm_resource_cursor *cursor); 391 struct ttm_resource * 392 ttm_resource_manager_next(struct ttm_resource_manager *man, 393 struct ttm_resource_cursor *cursor, 394 struct ttm_resource *res); 395 396 /** 397 * ttm_resource_manager_for_each_res - iterate over all resources 398 * @man: the resource manager 399 * @cursor: struct ttm_resource_cursor for the current position 400 * @res: the current resource 401 * 402 * Iterate over all the evictable resources in a resource manager. 403 */ 404 #define ttm_resource_manager_for_each_res(man, cursor, res) \ 405 for (res = ttm_resource_manager_first(man, cursor); res; \ 406 res = ttm_resource_manager_next(man, cursor, res)) 407 408 struct ttm_kmap_iter * 409 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, 410 struct io_mapping *iomap, 411 struct sg_table *st, 412 resource_size_t start); 413 414 struct ttm_kmap_iter_linear_io; 415 416 struct ttm_kmap_iter * 417 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, 418 struct ttm_device *bdev, 419 struct ttm_resource *mem); 420 421 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, 422 struct ttm_device *bdev, 423 struct ttm_resource *mem); 424 425 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, 426 struct dentry * parent, 427 const char *name); 428 #endif 429