1 /* 2 * Copyright 2020 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #ifndef _TTM_RESOURCE_H_ 26 #define _TTM_RESOURCE_H_ 27 28 #include <linux/types.h> 29 #include <linux/list.h> 30 #include <linux/mutex.h> 31 #include <linux/atomic.h> 32 #include <linux/dma-buf-map.h> 33 #include <linux/dma-fence.h> 34 35 #include <drm/drm_print.h> 36 #include <drm/ttm/ttm_caching.h> 37 #include <drm/ttm/ttm_kmap_iter.h> 38 39 #define TTM_MAX_BO_PRIORITY 4U 40 #define TTM_NUM_MEM_TYPES 8 41 42 struct ttm_device; 43 struct ttm_resource_manager; 44 struct ttm_resource; 45 struct ttm_place; 46 struct ttm_buffer_object; 47 struct ttm_placement; 48 struct dma_buf_map; 49 struct io_mapping; 50 struct sg_table; 51 struct scatterlist; 52 53 struct ttm_resource_manager_func { 54 /** 55 * struct ttm_resource_manager_func member alloc 56 * 57 * @man: Pointer to a memory type manager. 58 * @bo: Pointer to the buffer object we're allocating space for. 59 * @place: Placement details. 60 * @res: Resulting pointer to the ttm_resource. 61 * 62 * This function should allocate space in the memory type managed 63 * by @man. Placement details if applicable are given by @place. If 64 * successful, a filled in ttm_resource object should be returned in 65 * @res. @res::start should be set to a value identifying the beginning 66 * of the range allocated, and the function should return zero. 67 * If the manager can't fulfill the request -ENOSPC should be returned. 68 * If a system error occurred, preventing the request to be fulfilled, 69 * the function should return a negative error code. 70 * 71 * This function may not be called from within atomic context and needs 72 * to take care of its own locking to protect any data structures 73 * managing the space. 74 */ 75 int (*alloc)(struct ttm_resource_manager *man, 76 struct ttm_buffer_object *bo, 77 const struct ttm_place *place, 78 struct ttm_resource **res); 79 80 /** 81 * struct ttm_resource_manager_func member free 82 * 83 * @man: Pointer to a memory type manager. 84 * @res: Pointer to a struct ttm_resource to be freed. 85 * 86 * This function frees memory type resources previously allocated. 87 * May not be called from within atomic context. 88 */ 89 void (*free)(struct ttm_resource_manager *man, 90 struct ttm_resource *res); 91 92 /** 93 * struct ttm_resource_manager_func member debug 94 * 95 * @man: Pointer to a memory type manager. 96 * @printer: Prefix to be used in printout to identify the caller. 97 * 98 * This function is called to print out the state of the memory 99 * type manager to aid debugging of out-of-memory conditions. 100 * It may not be called from within atomic context. 101 */ 102 void (*debug)(struct ttm_resource_manager *man, 103 struct drm_printer *printer); 104 }; 105 106 /** 107 * struct ttm_resource_manager 108 * 109 * @use_type: The memory type is enabled. 110 * @use_tt: If a TT object should be used for the backing store. 111 * @size: Size of the managed region. 112 * @bdev: ttm device this manager belongs to 113 * @func: structure pointer implementing the range manager. See above 114 * @move_lock: lock for move fence 115 * @move: The fence of the last pipelined move operation. 116 * @lru: The lru list for this memory type. 117 * 118 * This structure is used to identify and manage memory types for a device. 119 */ 120 struct ttm_resource_manager { 121 /* 122 * No protection. Constant from start. 123 */ 124 bool use_type; 125 bool use_tt; 126 struct ttm_device *bdev; 127 uint64_t size; 128 const struct ttm_resource_manager_func *func; 129 spinlock_t move_lock; 130 131 /* 132 * Protected by @move_lock. 133 */ 134 struct dma_fence *move; 135 136 /* 137 * Protected by the bdev->lru_lock. 138 */ 139 struct list_head lru[TTM_MAX_BO_PRIORITY]; 140 141 /** 142 * @usage: How much of the resources are used, protected by the 143 * bdev->lru_lock. 144 */ 145 uint64_t usage; 146 }; 147 148 /** 149 * struct ttm_bus_placement 150 * 151 * @addr: mapped virtual address 152 * @offset: physical addr 153 * @is_iomem: is this io memory ? 154 * @caching: See enum ttm_caching 155 * 156 * Structure indicating the bus placement of an object. 157 */ 158 struct ttm_bus_placement { 159 void *addr; 160 phys_addr_t offset; 161 bool is_iomem; 162 enum ttm_caching caching; 163 }; 164 165 /** 166 * struct ttm_resource 167 * 168 * @start: Start of the allocation. 169 * @num_pages: Actual size of resource in pages. 170 * @mem_type: Resource type of the allocation. 171 * @placement: Placement flags. 172 * @bus: Placement on io bus accessible to the CPU 173 * @bo: weak reference to the BO, protected by ttm_device::lru_lock 174 * 175 * Structure indicating the placement and space resources used by a 176 * buffer object. 177 */ 178 struct ttm_resource { 179 unsigned long start; 180 unsigned long num_pages; 181 uint32_t mem_type; 182 uint32_t placement; 183 struct ttm_bus_placement bus; 184 struct ttm_buffer_object *bo; 185 186 /** 187 * @lru: Least recently used list, see &ttm_resource_manager.lru 188 */ 189 struct list_head lru; 190 }; 191 192 /** 193 * struct ttm_resource_cursor 194 * 195 * @priority: the current priority 196 * 197 * Cursor to iterate over the resources in a manager. 198 */ 199 struct ttm_resource_cursor { 200 unsigned int priority; 201 }; 202 203 /** 204 * struct ttm_lru_bulk_move_pos 205 * 206 * @first: first res in the bulk move range 207 * @last: last res in the bulk move range 208 * 209 * Range of resources for a lru bulk move. 210 */ 211 struct ttm_lru_bulk_move_pos { 212 struct ttm_resource *first; 213 struct ttm_resource *last; 214 }; 215 216 /** 217 * struct ttm_lru_bulk_move 218 * 219 * @tt: first/last lru entry for resources in the TT domain 220 * @vram: first/last lru entry for resources in the VRAM domain 221 * 222 * Container for the current bulk move state. Should be used with 223 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move(). 224 */ 225 struct ttm_lru_bulk_move { 226 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; 227 }; 228 229 /** 230 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping + 231 * struct sg_table backed struct ttm_resource. 232 * @base: Embedded struct ttm_kmap_iter providing the usage interface. 233 * @iomap: struct io_mapping representing the underlying linear io_memory. 234 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource. 235 * @start: Offset that needs to be subtracted from @st to make 236 * sg_dma_address(st->sgl) - @start == 0 for @iomap start. 237 * @cache: Scatterlist traversal cache for fast lookups. 238 * @cache.sg: Pointer to the currently cached scatterlist segment. 239 * @cache.i: First index of @sg. PAGE_SIZE granularity. 240 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity. 241 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity. 242 */ 243 struct ttm_kmap_iter_iomap { 244 struct ttm_kmap_iter base; 245 struct io_mapping *iomap; 246 struct sg_table *st; 247 resource_size_t start; 248 struct { 249 struct scatterlist *sg; 250 pgoff_t i; 251 pgoff_t end; 252 pgoff_t offs; 253 } cache; 254 }; 255 256 /** 257 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io 258 * @base: The base iterator 259 * @dmap: Points to the starting address of the region 260 * @needs_unmap: Whether we need to unmap on fini 261 */ 262 struct ttm_kmap_iter_linear_io { 263 struct ttm_kmap_iter base; 264 struct dma_buf_map dmap; 265 bool needs_unmap; 266 }; 267 268 /** 269 * ttm_resource_manager_set_used 270 * 271 * @man: A memory manager object. 272 * @used: usage state to set. 273 * 274 * Set the manager in use flag. If disabled the manager is no longer 275 * used for object placement. 276 */ 277 static inline void 278 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used) 279 { 280 int i; 281 282 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++) 283 WARN_ON(!list_empty(&man->lru[i])); 284 man->use_type = used; 285 } 286 287 /** 288 * ttm_resource_manager_used 289 * 290 * @man: Manager to get used state for 291 * 292 * Get the in use flag for a manager. 293 * Returns: 294 * true is used, false if not. 295 */ 296 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man) 297 { 298 return man->use_type; 299 } 300 301 /** 302 * ttm_resource_manager_cleanup 303 * 304 * @man: A memory manager object. 305 * 306 * Cleanup the move fences from the memory manager object. 307 */ 308 static inline void 309 ttm_resource_manager_cleanup(struct ttm_resource_manager *man) 310 { 311 dma_fence_put(man->move); 312 man->move = NULL; 313 } 314 315 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk); 316 void ttm_lru_bulk_move_add(struct ttm_lru_bulk_move *bulk, 317 struct ttm_resource *res); 318 void ttm_lru_bulk_move_del(struct ttm_lru_bulk_move *bulk, 319 struct ttm_resource *res); 320 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk); 321 322 void ttm_resource_move_to_lru_tail(struct ttm_resource *res); 323 324 void ttm_resource_init(struct ttm_buffer_object *bo, 325 const struct ttm_place *place, 326 struct ttm_resource *res); 327 void ttm_resource_fini(struct ttm_resource_manager *man, 328 struct ttm_resource *res); 329 330 int ttm_resource_alloc(struct ttm_buffer_object *bo, 331 const struct ttm_place *place, 332 struct ttm_resource **res); 333 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); 334 bool ttm_resource_compat(struct ttm_resource *res, 335 struct ttm_placement *placement); 336 void ttm_resource_set_bo(struct ttm_resource *res, 337 struct ttm_buffer_object *bo); 338 339 void ttm_resource_manager_init(struct ttm_resource_manager *man, 340 struct ttm_device *bdev, 341 uint64_t size); 342 343 int ttm_resource_manager_evict_all(struct ttm_device *bdev, 344 struct ttm_resource_manager *man); 345 346 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man); 347 void ttm_resource_manager_debug(struct ttm_resource_manager *man, 348 struct drm_printer *p); 349 350 struct ttm_resource * 351 ttm_resource_manager_first(struct ttm_resource_manager *man, 352 struct ttm_resource_cursor *cursor); 353 struct ttm_resource * 354 ttm_resource_manager_next(struct ttm_resource_manager *man, 355 struct ttm_resource_cursor *cursor, 356 struct ttm_resource *res); 357 358 /** 359 * ttm_resource_manager_for_each_res - iterate over all resources 360 * @man: the resource manager 361 * @cursor: struct ttm_resource_cursor for the current position 362 * @res: the current resource 363 * 364 * Iterate over all the evictable resources in a resource manager. 365 */ 366 #define ttm_resource_manager_for_each_res(man, cursor, res) \ 367 for (res = ttm_resource_manager_first(man, cursor); res; \ 368 res = ttm_resource_manager_next(man, cursor, res)) 369 370 struct ttm_kmap_iter * 371 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, 372 struct io_mapping *iomap, 373 struct sg_table *st, 374 resource_size_t start); 375 376 struct ttm_kmap_iter_linear_io; 377 378 struct ttm_kmap_iter * 379 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, 380 struct ttm_device *bdev, 381 struct ttm_resource *mem); 382 383 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, 384 struct ttm_device *bdev, 385 struct ttm_resource *mem); 386 #endif 387