1249d6048SJerome Glisse /************************************************************************** 2249d6048SJerome Glisse * 3249d6048SJerome Glisse * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX. USA. 4ba004e39SChris Wilson * Copyright 2016 Intel Corporation 5249d6048SJerome Glisse * All Rights Reserved. 6249d6048SJerome Glisse * 7249d6048SJerome Glisse * Permission is hereby granted, free of charge, to any person obtaining a 8249d6048SJerome Glisse * copy of this software and associated documentation files (the 9249d6048SJerome Glisse * "Software"), to deal in the Software without restriction, including 10249d6048SJerome Glisse * without limitation the rights to use, copy, modify, merge, publish, 11249d6048SJerome Glisse * distribute, sub license, and/or sell copies of the Software, and to 12249d6048SJerome Glisse * permit persons to whom the Software is furnished to do so, subject to 13249d6048SJerome Glisse * the following conditions: 14249d6048SJerome Glisse * 15249d6048SJerome Glisse * The above copyright notice and this permission notice (including the 16249d6048SJerome Glisse * next paragraph) shall be included in all copies or substantial portions 17249d6048SJerome Glisse * of the Software. 18249d6048SJerome Glisse * 19249d6048SJerome Glisse * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 20249d6048SJerome Glisse * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 21249d6048SJerome Glisse * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL 22249d6048SJerome Glisse * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, 23249d6048SJerome Glisse * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 24249d6048SJerome Glisse * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE 25249d6048SJerome Glisse * USE OR OTHER DEALINGS IN THE SOFTWARE. 26249d6048SJerome Glisse * 27249d6048SJerome Glisse * 28249d6048SJerome Glisse **************************************************************************/ 29249d6048SJerome Glisse /* 30249d6048SJerome Glisse * Authors: 31249d6048SJerome Glisse * Thomas Hellstrom <thomas-at-tungstengraphics-dot-com> 32249d6048SJerome Glisse */ 33249d6048SJerome Glisse 34249d6048SJerome Glisse #ifndef _DRM_MM_H_ 35249d6048SJerome Glisse #define _DRM_MM_H_ 36249d6048SJerome Glisse 37249d6048SJerome Glisse /* 38249d6048SJerome Glisse * Generic range manager structs 39249d6048SJerome Glisse */ 4086e81f0eSDavid Herrmann #include <linux/bug.h> 41202b52b7SChris Wilson #include <linux/rbtree.h> 4286e81f0eSDavid Herrmann #include <linux/kernel.h> 43589ee628SIngo Molnar #include <linux/mm_types.h> 44249d6048SJerome Glisse #include <linux/list.h> 4586e81f0eSDavid Herrmann #include <linux/spinlock.h> 465705670dSChris Wilson #ifdef CONFIG_DRM_DEBUG_MM 475705670dSChris Wilson #include <linux/stackdepot.h> 485705670dSChris Wilson #endif 49b5c3714fSDaniel Vetter #include <drm/drm_print.h> 50249d6048SJerome Glisse 51b3ee963fSChris Wilson #ifdef CONFIG_DRM_DEBUG_MM 52b3ee963fSChris Wilson #define DRM_MM_BUG_ON(expr) BUG_ON(expr) 53b3ee963fSChris Wilson #else 54b3ee963fSChris Wilson #define DRM_MM_BUG_ON(expr) BUILD_BUG_ON_INVALID(expr) 55b3ee963fSChris Wilson #endif 56b3ee963fSChris Wilson 574e64e553SChris Wilson /** 584e64e553SChris Wilson * enum drm_mm_insert_mode - control search and allocation behaviour 594e64e553SChris Wilson * 604e64e553SChris Wilson * The &struct drm_mm range manager supports finding a suitable modes using 614e64e553SChris Wilson * a number of search trees. These trees are oranised by size, by address and 624e64e553SChris Wilson * in most recent eviction order. This allows the user to find either the 634e64e553SChris Wilson * smallest hole to reuse, the lowest or highest address to reuse, or simply 644e64e553SChris Wilson * reuse the most recent eviction that fits. When allocating the &drm_mm_node 654e64e553SChris Wilson * from within the hole, the &drm_mm_insert_mode also dictate whether to 664e64e553SChris Wilson * allocate the lowest matching address or the highest. 674e64e553SChris Wilson */ 684e64e553SChris Wilson enum drm_mm_insert_mode { 694e64e553SChris Wilson /** 704e64e553SChris Wilson * @DRM_MM_INSERT_BEST: 714e64e553SChris Wilson * 724e64e553SChris Wilson * Search for the smallest hole (within the search range) that fits 734e64e553SChris Wilson * the desired node. 744e64e553SChris Wilson * 754e64e553SChris Wilson * Allocates the node from the bottom of the found hole. 764e64e553SChris Wilson */ 774e64e553SChris Wilson DRM_MM_INSERT_BEST = 0, 7831e5d7c6SDavid Herrmann 794e64e553SChris Wilson /** 804e64e553SChris Wilson * @DRM_MM_INSERT_LOW: 814e64e553SChris Wilson * 824e64e553SChris Wilson * Search for the lowest hole (address closest to 0, within the search 834e64e553SChris Wilson * range) that fits the desired node. 844e64e553SChris Wilson * 854e64e553SChris Wilson * Allocates the node from the bottom of the found hole. 864e64e553SChris Wilson */ 874e64e553SChris Wilson DRM_MM_INSERT_LOW, 8862347f9eSLauri Kasanen 894e64e553SChris Wilson /** 904e64e553SChris Wilson * @DRM_MM_INSERT_HIGH: 914e64e553SChris Wilson * 924e64e553SChris Wilson * Search for the highest hole (address closest to U64_MAX, within the 934e64e553SChris Wilson * search range) that fits the desired node. 944e64e553SChris Wilson * 954e64e553SChris Wilson * Allocates the node from the *top* of the found hole. The specified 964e64e553SChris Wilson * alignment for the node is applied to the base of the node 974e64e553SChris Wilson * (&drm_mm_node.start). 984e64e553SChris Wilson */ 994e64e553SChris Wilson DRM_MM_INSERT_HIGH, 1004e64e553SChris Wilson 1014e64e553SChris Wilson /** 1024e64e553SChris Wilson * @DRM_MM_INSERT_EVICT: 1034e64e553SChris Wilson * 1044e64e553SChris Wilson * Search for the most recently evicted hole (within the search range) 1054e64e553SChris Wilson * that fits the desired node. This is appropriate for use immediately 1064e64e553SChris Wilson * after performing an eviction scan (see drm_mm_scan_init()) and 1074e64e553SChris Wilson * removing the selected nodes to form a hole. 1084e64e553SChris Wilson * 1094e64e553SChris Wilson * Allocates the node from the bottom of the found hole. 1104e64e553SChris Wilson */ 1114e64e553SChris Wilson DRM_MM_INSERT_EVICT, 11283bc4ec3SChris Wilson 11383bc4ec3SChris Wilson /** 11483bc4ec3SChris Wilson * @DRM_MM_INSERT_ONCE: 11583bc4ec3SChris Wilson * 11683bc4ec3SChris Wilson * Only check the first hole for suitablity and report -ENOSPC 11783bc4ec3SChris Wilson * immediately otherwise, rather than check every hole until a 11883bc4ec3SChris Wilson * suitable one is found. Can only be used in conjunction with another 11983bc4ec3SChris Wilson * search method such as DRM_MM_INSERT_HIGH or DRM_MM_INSERT_LOW. 12083bc4ec3SChris Wilson */ 12183bc4ec3SChris Wilson DRM_MM_INSERT_ONCE = BIT(31), 12283bc4ec3SChris Wilson 12383bc4ec3SChris Wilson /** 12483bc4ec3SChris Wilson * @DRM_MM_INSERT_HIGHEST: 12583bc4ec3SChris Wilson * 12683bc4ec3SChris Wilson * Only check the highest hole (the hole with the largest address) and 12783bc4ec3SChris Wilson * insert the node at the top of the hole or report -ENOSPC if 12883bc4ec3SChris Wilson * unsuitable. 12983bc4ec3SChris Wilson * 13083bc4ec3SChris Wilson * Does not search all holes. 13183bc4ec3SChris Wilson */ 13283bc4ec3SChris Wilson DRM_MM_INSERT_HIGHEST = DRM_MM_INSERT_HIGH | DRM_MM_INSERT_ONCE, 13383bc4ec3SChris Wilson 13483bc4ec3SChris Wilson /** 13583bc4ec3SChris Wilson * @DRM_MM_INSERT_LOWEST: 13683bc4ec3SChris Wilson * 13783bc4ec3SChris Wilson * Only check the lowest hole (the hole with the smallest address) and 13883bc4ec3SChris Wilson * insert the node at the bottom of the hole or report -ENOSPC if 13983bc4ec3SChris Wilson * unsuitable. 14083bc4ec3SChris Wilson * 14183bc4ec3SChris Wilson * Does not search all holes. 14283bc4ec3SChris Wilson */ 14383bc4ec3SChris Wilson DRM_MM_INSERT_LOWEST = DRM_MM_INSERT_LOW | DRM_MM_INSERT_ONCE, 1444e64e553SChris Wilson }; 14562347f9eSLauri Kasanen 14605fc0321SDaniel Vetter /** 14705fc0321SDaniel Vetter * struct drm_mm_node - allocated block in the DRM allocator 14805fc0321SDaniel Vetter * 14905fc0321SDaniel Vetter * This represents an allocated block in a &drm_mm allocator. Except for 15005fc0321SDaniel Vetter * pre-reserved nodes inserted using drm_mm_reserve_node() the structure is 15105fc0321SDaniel Vetter * entirely opaque and should only be accessed through the provided funcions. 15205fc0321SDaniel Vetter * Since allocation of these nodes is entirely handled by the driver they can be 15305fc0321SDaniel Vetter * embedded. 15405fc0321SDaniel Vetter */ 155249d6048SJerome Glisse struct drm_mm_node { 15605fc0321SDaniel Vetter /** @color: Opaque driver-private tag. */ 15705fc0321SDaniel Vetter unsigned long color; 15805fc0321SDaniel Vetter /** @start: Start address of the allocated block. */ 15905fc0321SDaniel Vetter u64 start; 16005fc0321SDaniel Vetter /** @size: Size of the allocated block. */ 16105fc0321SDaniel Vetter u64 size; 16205fc0321SDaniel Vetter /* private: */ 1634e64e553SChris Wilson struct drm_mm *mm; 164d1024ce9SDaniel Vetter struct list_head node_list; 165ea7b1dd4SDaniel Vetter struct list_head hole_stack; 166202b52b7SChris Wilson struct rb_node rb; 1674e64e553SChris Wilson struct rb_node rb_hole_size; 1684e64e553SChris Wilson struct rb_node rb_hole_addr; 169202b52b7SChris Wilson u64 __subtree_last; 1704e64e553SChris Wilson u64 hole_size; 1710cdea445SNirmoy Das u64 subtree_max_hole; 1724ee92c71SChris Wilson unsigned long flags; 1734ee92c71SChris Wilson #define DRM_MM_NODE_ALLOCATED_BIT 0 1744ee92c71SChris Wilson #define DRM_MM_NODE_SCANNED_BIT 1 1755705670dSChris Wilson #ifdef CONFIG_DRM_DEBUG_MM 1765705670dSChris Wilson depot_stack_handle_t stack; 1775705670dSChris Wilson #endif 178249d6048SJerome Glisse }; 179249d6048SJerome Glisse 18005fc0321SDaniel Vetter /** 18105fc0321SDaniel Vetter * struct drm_mm - DRM allocator 18205fc0321SDaniel Vetter * 18305fc0321SDaniel Vetter * DRM range allocator with a few special functions and features geared towards 18405fc0321SDaniel Vetter * managing GPU memory. Except for the @color_adjust callback the structure is 18505fc0321SDaniel Vetter * entirely opaque and should only be accessed through the provided functions 18605fc0321SDaniel Vetter * and macros. This structure can be embedded into larger driver structures. 18705fc0321SDaniel Vetter */ 188249d6048SJerome Glisse struct drm_mm { 18905fc0321SDaniel Vetter /** 19005fc0321SDaniel Vetter * @color_adjust: 19105fc0321SDaniel Vetter * 19205fc0321SDaniel Vetter * Optional driver callback to further apply restrictions on a hole. The 19305fc0321SDaniel Vetter * node argument points at the node containing the hole from which the 19405fc0321SDaniel Vetter * block would be allocated (see drm_mm_hole_follows() and friends). The 19505fc0321SDaniel Vetter * other arguments are the size of the block to be allocated. The driver 19605fc0321SDaniel Vetter * can adjust the start and end as needed to e.g. insert guard pages. 19705fc0321SDaniel Vetter */ 19805fc0321SDaniel Vetter void (*color_adjust)(const struct drm_mm_node *node, 19905fc0321SDaniel Vetter unsigned long color, 20005fc0321SDaniel Vetter u64 *start, u64 *end); 20105fc0321SDaniel Vetter 20205fc0321SDaniel Vetter /* private: */ 20325985edcSLucas De Marchi /* List of all memory nodes that immediately precede a free hole. */ 204ea7b1dd4SDaniel Vetter struct list_head hole_stack; 205ea7b1dd4SDaniel Vetter /* head_node.node_list is the list of all memory nodes, ordered 206ea7b1dd4SDaniel Vetter * according to the (increasing) start address of the memory node. */ 207ea7b1dd4SDaniel Vetter struct drm_mm_node head_node; 208202b52b7SChris Wilson /* Keep an interval_tree for fast lookup of drm_mm_nodes by address. */ 209f808c13fSDavidlohr Bueso struct rb_root_cached interval_tree; 2102f7e8769SChris Wilson struct rb_root_cached holes_size; 2114e64e553SChris Wilson struct rb_root holes_addr; 212202b52b7SChris Wilson 2139a71e277SChris Wilson unsigned long scan_active; 2149a71e277SChris Wilson }; 2159a71e277SChris Wilson 21605fc0321SDaniel Vetter /** 21705fc0321SDaniel Vetter * struct drm_mm_scan - DRM allocator eviction roaster data 21805fc0321SDaniel Vetter * 21905fc0321SDaniel Vetter * This structure tracks data needed for the eviction roaster set up using 22005fc0321SDaniel Vetter * drm_mm_scan_init(), and used with drm_mm_scan_add_block() and 22105fc0321SDaniel Vetter * drm_mm_scan_remove_block(). The structure is entirely opaque and should only 22205fc0321SDaniel Vetter * be accessed through the provided functions and macros. It is meant to be 22305fc0321SDaniel Vetter * allocated temporarily by the driver on the stack. 22405fc0321SDaniel Vetter */ 2259a71e277SChris Wilson struct drm_mm_scan { 22605fc0321SDaniel Vetter /* private: */ 2279a71e277SChris Wilson struct drm_mm *mm; 2289a71e277SChris Wilson 2299a71e277SChris Wilson u64 size; 2309a71e277SChris Wilson u64 alignment; 2319a956b15SChris Wilson u64 remainder_mask; 2329a71e277SChris Wilson 2339a71e277SChris Wilson u64 range_start; 2349a71e277SChris Wilson u64 range_end; 2359a71e277SChris Wilson 2369a71e277SChris Wilson u64 hit_start; 2379a71e277SChris Wilson u64 hit_end; 2389a71e277SChris Wilson 2399a71e277SChris Wilson unsigned long color; 2404e64e553SChris Wilson enum drm_mm_insert_mode mode; 241249d6048SJerome Glisse }; 242249d6048SJerome Glisse 243e18c0412SDaniel Vetter /** 244e18c0412SDaniel Vetter * drm_mm_node_allocated - checks whether a node is allocated 245e18c0412SDaniel Vetter * @node: drm_mm_node to check 246e18c0412SDaniel Vetter * 247ba004e39SChris Wilson * Drivers are required to clear a node prior to using it with the 248ba004e39SChris Wilson * drm_mm range manager. 249ba004e39SChris Wilson * 250ba004e39SChris Wilson * Drivers should use this helper for proper encapsulation of drm_mm 251e18c0412SDaniel Vetter * internals. 252e18c0412SDaniel Vetter * 253e18c0412SDaniel Vetter * Returns: 254e18c0412SDaniel Vetter * True if the @node is allocated. 255e18c0412SDaniel Vetter */ 25645b186f1SChris Wilson static inline bool drm_mm_node_allocated(const struct drm_mm_node *node) 257b0b7af18SDaniel Vetter { 2584ee92c71SChris Wilson return test_bit(DRM_MM_NODE_ALLOCATED_BIT, &node->flags); 259b0b7af18SDaniel Vetter } 260b0b7af18SDaniel Vetter 261e18c0412SDaniel Vetter /** 262e18c0412SDaniel Vetter * drm_mm_initialized - checks whether an allocator is initialized 263e18c0412SDaniel Vetter * @mm: drm_mm to check 264e18c0412SDaniel Vetter * 265ba004e39SChris Wilson * Drivers should clear the struct drm_mm prior to initialisation if they 266ba004e39SChris Wilson * want to use this function. 267ba004e39SChris Wilson * 268ba004e39SChris Wilson * Drivers should use this helper for proper encapsulation of drm_mm 269e18c0412SDaniel Vetter * internals. 270e18c0412SDaniel Vetter * 271e18c0412SDaniel Vetter * Returns: 272e18c0412SDaniel Vetter * True if the @mm is initialized. 273e18c0412SDaniel Vetter */ 27445b186f1SChris Wilson static inline bool drm_mm_initialized(const struct drm_mm *mm) 27531a5b8ceSDaniel Vetter { 2762214ddc2SChris Wilson return READ_ONCE(mm->hole_stack.next); 27731a5b8ceSDaniel Vetter } 2789e8944abSChris Wilson 2793f85fb34SChris Wilson /** 2803f85fb34SChris Wilson * drm_mm_hole_follows - checks whether a hole follows this node 2813f85fb34SChris Wilson * @node: drm_mm_node to check 2823f85fb34SChris Wilson * 2833f85fb34SChris Wilson * Holes are embedded into the drm_mm using the tail of a drm_mm_node. 2843f85fb34SChris Wilson * If you wish to know whether a hole follows this particular node, 28505fc0321SDaniel Vetter * query this function. See also drm_mm_hole_node_start() and 28605fc0321SDaniel Vetter * drm_mm_hole_node_end(). 2873f85fb34SChris Wilson * 2883f85fb34SChris Wilson * Returns: 2893f85fb34SChris Wilson * True if a hole follows the @node. 2903f85fb34SChris Wilson */ 2913f85fb34SChris Wilson static inline bool drm_mm_hole_follows(const struct drm_mm_node *node) 2923f85fb34SChris Wilson { 2934e64e553SChris Wilson return node->hole_size; 2943f85fb34SChris Wilson } 2953f85fb34SChris Wilson 29645b186f1SChris Wilson static inline u64 __drm_mm_hole_node_start(const struct drm_mm_node *hole_node) 2979e8944abSChris Wilson { 2989e8944abSChris Wilson return hole_node->start + hole_node->size; 2999e8944abSChris Wilson } 3009e8944abSChris Wilson 301e18c0412SDaniel Vetter /** 302e18c0412SDaniel Vetter * drm_mm_hole_node_start - computes the start of the hole following @node 303e18c0412SDaniel Vetter * @hole_node: drm_mm_node which implicitly tracks the following hole 304e18c0412SDaniel Vetter * 305ba004e39SChris Wilson * This is useful for driver-specific debug dumpers. Otherwise drivers should 306ba004e39SChris Wilson * not inspect holes themselves. Drivers must check first whether a hole indeed 3073f85fb34SChris Wilson * follows by looking at drm_mm_hole_follows() 308e18c0412SDaniel Vetter * 309e18c0412SDaniel Vetter * Returns: 310e18c0412SDaniel Vetter * Start of the subsequent hole. 311e18c0412SDaniel Vetter */ 31245b186f1SChris Wilson static inline u64 drm_mm_hole_node_start(const struct drm_mm_node *hole_node) 3139e8944abSChris Wilson { 3143f85fb34SChris Wilson DRM_MM_BUG_ON(!drm_mm_hole_follows(hole_node)); 3159e8944abSChris Wilson return __drm_mm_hole_node_start(hole_node); 3169e8944abSChris Wilson } 3179e8944abSChris Wilson 31845b186f1SChris Wilson static inline u64 __drm_mm_hole_node_end(const struct drm_mm_node *hole_node) 3199e8944abSChris Wilson { 32087069f44SGeliang Tang return list_next_entry(hole_node, node_list)->start; 3219e8944abSChris Wilson } 3229e8944abSChris Wilson 323e18c0412SDaniel Vetter /** 324e18c0412SDaniel Vetter * drm_mm_hole_node_end - computes the end of the hole following @node 325e18c0412SDaniel Vetter * @hole_node: drm_mm_node which implicitly tracks the following hole 326e18c0412SDaniel Vetter * 327ba004e39SChris Wilson * This is useful for driver-specific debug dumpers. Otherwise drivers should 328ba004e39SChris Wilson * not inspect holes themselves. Drivers must check first whether a hole indeed 3293f85fb34SChris Wilson * follows by looking at drm_mm_hole_follows(). 330e18c0412SDaniel Vetter * 331e18c0412SDaniel Vetter * Returns: 332e18c0412SDaniel Vetter * End of the subsequent hole. 333e18c0412SDaniel Vetter */ 33445b186f1SChris Wilson static inline u64 drm_mm_hole_node_end(const struct drm_mm_node *hole_node) 3359e8944abSChris Wilson { 3369e8944abSChris Wilson return __drm_mm_hole_node_end(hole_node); 3379e8944abSChris Wilson } 3389e8944abSChris Wilson 3392bc98c86SChris Wilson /** 3402bc98c86SChris Wilson * drm_mm_nodes - list of nodes under the drm_mm range manager 3412bc98c86SChris Wilson * @mm: the struct drm_mm range manger 3422bc98c86SChris Wilson * 3432bc98c86SChris Wilson * As the drm_mm range manager hides its node_list deep with its 3442bc98c86SChris Wilson * structure, extracting it looks painful and repetitive. This is 3452bc98c86SChris Wilson * not expected to be used outside of the drm_mm_for_each_node() 3462bc98c86SChris Wilson * macros and similar internal functions. 3472bc98c86SChris Wilson * 3482bc98c86SChris Wilson * Returns: 3492bc98c86SChris Wilson * The node list, may be empty. 3502bc98c86SChris Wilson */ 3512bc98c86SChris Wilson #define drm_mm_nodes(mm) (&(mm)->head_node.node_list) 352ad579002SChris Wilson 353e18c0412SDaniel Vetter /** 354e18c0412SDaniel Vetter * drm_mm_for_each_node - iterator to walk over all allocated nodes 35505fc0321SDaniel Vetter * @entry: &struct drm_mm_node to assign to in each iteration step 35605fc0321SDaniel Vetter * @mm: &drm_mm allocator to walk 357e18c0412SDaniel Vetter * 358e18c0412SDaniel Vetter * This iterator walks over all nodes in the range allocator. It is implemented 35905fc0321SDaniel Vetter * with list_for_each(), so not save against removal of elements. 360e18c0412SDaniel Vetter */ 361ad579002SChris Wilson #define drm_mm_for_each_node(entry, mm) \ 3622bc98c86SChris Wilson list_for_each_entry(entry, drm_mm_nodes(mm), node_list) 363ad579002SChris Wilson 364ad579002SChris Wilson /** 365ad579002SChris Wilson * drm_mm_for_each_node_safe - iterator to walk over all allocated nodes 36605fc0321SDaniel Vetter * @entry: &struct drm_mm_node to assign to in each iteration step 36705fc0321SDaniel Vetter * @next: &struct drm_mm_node to store the next step 36805fc0321SDaniel Vetter * @mm: &drm_mm allocator to walk 369ad579002SChris Wilson * 370ad579002SChris Wilson * This iterator walks over all nodes in the range allocator. It is implemented 37105fc0321SDaniel Vetter * with list_for_each_safe(), so save against removal of elements. 372ad579002SChris Wilson */ 373ad579002SChris Wilson #define drm_mm_for_each_node_safe(entry, next, mm) \ 3742bc98c86SChris Wilson list_for_each_entry_safe(entry, next, drm_mm_nodes(mm), node_list) 3759e8944abSChris Wilson 376e18c0412SDaniel Vetter /** 377e18c0412SDaniel Vetter * drm_mm_for_each_hole - iterator to walk over all holes 3784e64e553SChris Wilson * @pos: &drm_mm_node used internally to track progress 37905fc0321SDaniel Vetter * @mm: &drm_mm allocator to walk 380e18c0412SDaniel Vetter * @hole_start: ulong variable to assign the hole start to on each iteration 381e18c0412SDaniel Vetter * @hole_end: ulong variable to assign the hole end to on each iteration 382e18c0412SDaniel Vetter * 383e18c0412SDaniel Vetter * This iterator walks over all holes in the range allocator. It is implemented 38405fc0321SDaniel Vetter * with list_for_each(), so not save against removal of elements. @entry is used 385e18c0412SDaniel Vetter * internally and will not reflect a real drm_mm_node for the very first hole. 386e18c0412SDaniel Vetter * Hence users of this iterator may not access it. 387e18c0412SDaniel Vetter * 388e18c0412SDaniel Vetter * Implementation Note: 389e18c0412SDaniel Vetter * We need to inline list_for_each_entry in order to be able to set hole_start 390e18c0412SDaniel Vetter * and hole_end on each iteration while keeping the macro sane. 3919e8944abSChris Wilson */ 3924e64e553SChris Wilson #define drm_mm_for_each_hole(pos, mm, hole_start, hole_end) \ 3934e64e553SChris Wilson for (pos = list_first_entry(&(mm)->hole_stack, \ 3944e64e553SChris Wilson typeof(*pos), hole_stack); \ 3954e64e553SChris Wilson &pos->hole_stack != &(mm)->hole_stack ? \ 3964e64e553SChris Wilson hole_start = drm_mm_hole_node_start(pos), \ 3974e64e553SChris Wilson hole_end = hole_start + pos->hole_size, \ 3984e64e553SChris Wilson 1 : 0; \ 3994e64e553SChris Wilson pos = list_next_entry(pos, hole_stack)) 40062347f9eSLauri Kasanen 401249d6048SJerome Glisse /* 402249d6048SJerome Glisse * Basic range manager support (drm_mm.c) 403249d6048SJerome Glisse */ 404e18c0412SDaniel Vetter int drm_mm_reserve_node(struct drm_mm *mm, struct drm_mm_node *node); 4054e64e553SChris Wilson int drm_mm_insert_node_in_range(struct drm_mm *mm, 406b8103450SChris Wilson struct drm_mm_node *node, 407440fd528SThierry Reding u64 size, 40871733207SChris Wilson u64 alignment, 409b8103450SChris Wilson unsigned long color, 410440fd528SThierry Reding u64 start, 411440fd528SThierry Reding u64 end, 4124e64e553SChris Wilson enum drm_mm_insert_mode mode); 41331e5d7c6SDavid Herrmann 414adb040b8SChris Wilson /** 415adb040b8SChris Wilson * drm_mm_insert_node_generic - search for space and insert @node 416adb040b8SChris Wilson * @mm: drm_mm to allocate from 417adb040b8SChris Wilson * @node: preallocate node to insert 418adb040b8SChris Wilson * @size: size of the allocation 419adb040b8SChris Wilson * @alignment: alignment of the allocation 420adb040b8SChris Wilson * @color: opaque tag value to use for this node 4214e64e553SChris Wilson * @mode: fine-tune the allocation search and placement 422adb040b8SChris Wilson * 4230a2adb02SLiviu Dudau * This is a simplified version of drm_mm_insert_node_in_range() with no 42405fc0321SDaniel Vetter * range restrictions applied. 42505fc0321SDaniel Vetter * 426adb040b8SChris Wilson * The preallocated node must be cleared to 0. 427adb040b8SChris Wilson * 428adb040b8SChris Wilson * Returns: 429adb040b8SChris Wilson * 0 on success, -ENOSPC if there's no suitable hole. 430adb040b8SChris Wilson */ 431adb040b8SChris Wilson static inline int 432adb040b8SChris Wilson drm_mm_insert_node_generic(struct drm_mm *mm, struct drm_mm_node *node, 433adb040b8SChris Wilson u64 size, u64 alignment, 434adb040b8SChris Wilson unsigned long color, 4354e64e553SChris Wilson enum drm_mm_insert_mode mode) 436adb040b8SChris Wilson { 4374e64e553SChris Wilson return drm_mm_insert_node_in_range(mm, node, 4384e64e553SChris Wilson size, alignment, color, 4394e64e553SChris Wilson 0, U64_MAX, mode); 440adb040b8SChris Wilson } 441adb040b8SChris Wilson 442adb040b8SChris Wilson /** 443adb040b8SChris Wilson * drm_mm_insert_node - search for space and insert @node 444adb040b8SChris Wilson * @mm: drm_mm to allocate from 445adb040b8SChris Wilson * @node: preallocate node to insert 446adb040b8SChris Wilson * @size: size of the allocation 447adb040b8SChris Wilson * 448adb040b8SChris Wilson * This is a simplified version of drm_mm_insert_node_generic() with @color set 449adb040b8SChris Wilson * to 0. 450adb040b8SChris Wilson * 451adb040b8SChris Wilson * The preallocated node must be cleared to 0. 452adb040b8SChris Wilson * 453adb040b8SChris Wilson * Returns: 454adb040b8SChris Wilson * 0 on success, -ENOSPC if there's no suitable hole. 455adb040b8SChris Wilson */ 456adb040b8SChris Wilson static inline int drm_mm_insert_node(struct drm_mm *mm, 457adb040b8SChris Wilson struct drm_mm_node *node, 4584e64e553SChris Wilson u64 size) 459adb040b8SChris Wilson { 4604e64e553SChris Wilson return drm_mm_insert_node_generic(mm, node, size, 0, 0, 0); 461adb040b8SChris Wilson } 462adb040b8SChris Wilson 463e18c0412SDaniel Vetter void drm_mm_remove_node(struct drm_mm_node *node); 464e18c0412SDaniel Vetter void drm_mm_replace_node(struct drm_mm_node *old, struct drm_mm_node *new); 46545b186f1SChris Wilson void drm_mm_init(struct drm_mm *mm, u64 start, u64 size); 466e18c0412SDaniel Vetter void drm_mm_takedown(struct drm_mm *mm); 467ac9bb7b7SChris Wilson 468ac9bb7b7SChris Wilson /** 469ac9bb7b7SChris Wilson * drm_mm_clean - checks whether an allocator is clean 470ac9bb7b7SChris Wilson * @mm: drm_mm allocator to check 471ac9bb7b7SChris Wilson * 472ac9bb7b7SChris Wilson * Returns: 473ac9bb7b7SChris Wilson * True if the allocator is completely free, false if there's still a node 474ac9bb7b7SChris Wilson * allocated in it. 475ac9bb7b7SChris Wilson */ 476ac9bb7b7SChris Wilson static inline bool drm_mm_clean(const struct drm_mm *mm) 477ac9bb7b7SChris Wilson { 478ac9bb7b7SChris Wilson return list_empty(drm_mm_nodes(mm)); 479ac9bb7b7SChris Wilson } 480249d6048SJerome Glisse 481202b52b7SChris Wilson struct drm_mm_node * 48245b186f1SChris Wilson __drm_mm_interval_first(const struct drm_mm *mm, u64 start, u64 last); 483202b52b7SChris Wilson 484522e85ddSChris Wilson /** 485522e85ddSChris Wilson * drm_mm_for_each_node_in_range - iterator to walk over a range of 486522e85ddSChris Wilson * allocated nodes 4878b2fb7b6SChris Wilson * @node__: drm_mm_node structure to assign to in each iteration step 4888b2fb7b6SChris Wilson * @mm__: drm_mm allocator to walk 4898b2fb7b6SChris Wilson * @start__: starting offset, the first node will overlap this 4908b2fb7b6SChris Wilson * @end__: ending offset, the last node will start before this (but may overlap) 491522e85ddSChris Wilson * 492522e85ddSChris Wilson * This iterator walks over all nodes in the range allocator that lie 493522e85ddSChris Wilson * between @start and @end. It is implemented similarly to list_for_each(), 494522e85ddSChris Wilson * but using the internal interval tree to accelerate the search for the 495522e85ddSChris Wilson * starting node, and so not safe against removal of elements. It assumes 496522e85ddSChris Wilson * that @end is within (or is the upper limit of) the drm_mm allocator. 497bbba9693SChris Wilson * If [@start, @end] are beyond the range of the drm_mm, the iterator may walk 498bbba9693SChris Wilson * over the special _unallocated_ &drm_mm.head_node, and may even continue 499bbba9693SChris Wilson * indefinitely. 500522e85ddSChris Wilson */ 5018b2fb7b6SChris Wilson #define drm_mm_for_each_node_in_range(node__, mm__, start__, end__) \ 5028b2fb7b6SChris Wilson for (node__ = __drm_mm_interval_first((mm__), (start__), (end__)-1); \ 503bbba9693SChris Wilson node__->start < (end__); \ 5048b2fb7b6SChris Wilson node__ = list_next_entry(node__, node_list)) 505202b52b7SChris Wilson 5069a71e277SChris Wilson void drm_mm_scan_init_with_range(struct drm_mm_scan *scan, 5079a71e277SChris Wilson struct drm_mm *mm, 5080b04d474SChris Wilson u64 size, u64 alignment, unsigned long color, 5090b04d474SChris Wilson u64 start, u64 end, 5104e64e553SChris Wilson enum drm_mm_insert_mode mode); 5112c4b3895SChris Wilson 5122c4b3895SChris Wilson /** 5132c4b3895SChris Wilson * drm_mm_scan_init - initialize lru scanning 5142c4b3895SChris Wilson * @scan: scan state 5152c4b3895SChris Wilson * @mm: drm_mm to scan 5162c4b3895SChris Wilson * @size: size of the allocation 5172c4b3895SChris Wilson * @alignment: alignment of the allocation 5182c4b3895SChris Wilson * @color: opaque tag value to use for the allocation 5194e64e553SChris Wilson * @mode: fine-tune the allocation search and placement 5202c4b3895SChris Wilson * 52105fc0321SDaniel Vetter * This is a simplified version of drm_mm_scan_init_with_range() with no range 52205fc0321SDaniel Vetter * restrictions applied. 52305fc0321SDaniel Vetter * 5242c4b3895SChris Wilson * This simply sets up the scanning routines with the parameters for the desired 5250b04d474SChris Wilson * hole. 5262c4b3895SChris Wilson * 5272c4b3895SChris Wilson * Warning: 5282c4b3895SChris Wilson * As long as the scan list is non-empty, no other operations than 5292c4b3895SChris Wilson * adding/removing nodes to/from the scan list are allowed. 5302c4b3895SChris Wilson */ 5312c4b3895SChris Wilson static inline void drm_mm_scan_init(struct drm_mm_scan *scan, 5322c4b3895SChris Wilson struct drm_mm *mm, 5332c4b3895SChris Wilson u64 size, 5342c4b3895SChris Wilson u64 alignment, 5350b04d474SChris Wilson unsigned long color, 5364e64e553SChris Wilson enum drm_mm_insert_mode mode) 5372c4b3895SChris Wilson { 5380b04d474SChris Wilson drm_mm_scan_init_with_range(scan, mm, 5390b04d474SChris Wilson size, alignment, color, 5404e64e553SChris Wilson 0, U64_MAX, mode); 5412c4b3895SChris Wilson } 5422c4b3895SChris Wilson 5439a71e277SChris Wilson bool drm_mm_scan_add_block(struct drm_mm_scan *scan, 5449a71e277SChris Wilson struct drm_mm_node *node); 5459a71e277SChris Wilson bool drm_mm_scan_remove_block(struct drm_mm_scan *scan, 5469a71e277SChris Wilson struct drm_mm_node *node); 5473fa489daSChris Wilson struct drm_mm_node *drm_mm_scan_color_evict(struct drm_mm_scan *scan); 548709ea971SDaniel Vetter 549b5c3714fSDaniel Vetter void drm_mm_print(const struct drm_mm *mm, struct drm_printer *p); 550fa8a1238SDave Airlie 551249d6048SJerome Glisse #endif 552