1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2014-2018 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_internal.h" 7 #include "gem/i915_gem_object.h" 8 9 #include "i915_drv.h" 10 #include "intel_engine_pm.h" 11 #include "intel_gt_buffer_pool.h" 12 13 static struct list_head * 14 bucket_for_size(struct intel_gt_buffer_pool *pool, size_t sz) 15 { 16 int n; 17 18 /* 19 * Compute a power-of-two bucket, but throw everything greater than 20 * 16KiB into the same bucket: i.e. the buckets hold objects of 21 * (1 page, 2 pages, 4 pages, 8+ pages). 22 */ 23 n = fls(sz >> PAGE_SHIFT) - 1; 24 if (n >= ARRAY_SIZE(pool->cache_list)) 25 n = ARRAY_SIZE(pool->cache_list) - 1; 26 27 return &pool->cache_list[n]; 28 } 29 30 static void node_free(struct intel_gt_buffer_pool_node *node) 31 { 32 i915_gem_object_put(node->obj); 33 i915_active_fini(&node->active); 34 kfree_rcu(node, rcu); 35 } 36 37 static bool pool_free_older_than(struct intel_gt_buffer_pool *pool, long keep) 38 { 39 struct intel_gt_buffer_pool_node *node, *stale = NULL; 40 bool active = false; 41 int n; 42 43 /* Free buffers that have not been used in the past second */ 44 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) { 45 struct list_head *list = &pool->cache_list[n]; 46 47 if (list_empty(list)) 48 continue; 49 50 if (spin_trylock_irq(&pool->lock)) { 51 struct list_head *pos; 52 53 /* Most recent at head; oldest at tail */ 54 list_for_each_prev(pos, list) { 55 unsigned long age; 56 57 node = list_entry(pos, typeof(*node), link); 58 59 age = READ_ONCE(node->age); 60 if (!age || jiffies - age < keep) 61 break; 62 63 /* Check we are the first to claim this node */ 64 if (!xchg(&node->age, 0)) 65 break; 66 67 node->free = stale; 68 stale = node; 69 } 70 if (!list_is_last(pos, list)) 71 __list_del_many(pos, list); 72 73 spin_unlock_irq(&pool->lock); 74 } 75 76 active |= !list_empty(list); 77 } 78 79 while ((node = stale)) { 80 stale = stale->free; 81 node_free(node); 82 } 83 84 return active; 85 } 86 87 static void pool_free_work(struct work_struct *wrk) 88 { 89 struct intel_gt_buffer_pool *pool = 90 container_of(wrk, typeof(*pool), work.work); 91 92 if (pool_free_older_than(pool, HZ)) 93 schedule_delayed_work(&pool->work, 94 round_jiffies_up_relative(HZ)); 95 } 96 97 static void pool_retire(struct i915_active *ref) 98 { 99 struct intel_gt_buffer_pool_node *node = 100 container_of(ref, typeof(*node), active); 101 struct intel_gt_buffer_pool *pool = node->pool; 102 struct list_head *list = bucket_for_size(pool, node->obj->base.size); 103 unsigned long flags; 104 105 if (node->pinned) { 106 i915_gem_object_unpin_pages(node->obj); 107 108 /* Return this object to the shrinker pool */ 109 i915_gem_object_make_purgeable(node->obj); 110 node->pinned = false; 111 } 112 113 GEM_BUG_ON(node->age); 114 spin_lock_irqsave(&pool->lock, flags); 115 list_add_rcu(&node->link, list); 116 WRITE_ONCE(node->age, jiffies ?: 1); /* 0 reserved for active nodes */ 117 spin_unlock_irqrestore(&pool->lock, flags); 118 119 schedule_delayed_work(&pool->work, 120 round_jiffies_up_relative(HZ)); 121 } 122 123 void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node) 124 { 125 assert_object_held(node->obj); 126 127 if (node->pinned) 128 return; 129 130 __i915_gem_object_pin_pages(node->obj); 131 /* Hide this pinned object from the shrinker until retired */ 132 i915_gem_object_make_unshrinkable(node->obj); 133 node->pinned = true; 134 } 135 136 static struct intel_gt_buffer_pool_node * 137 node_create(struct intel_gt_buffer_pool *pool, size_t sz, 138 enum i915_map_type type) 139 { 140 struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool); 141 struct intel_gt_buffer_pool_node *node; 142 struct drm_i915_gem_object *obj; 143 144 node = kmalloc(sizeof(*node), 145 GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN); 146 if (!node) 147 return ERR_PTR(-ENOMEM); 148 149 node->age = 0; 150 node->pool = pool; 151 node->pinned = false; 152 i915_active_init(&node->active, NULL, pool_retire, 0); 153 154 obj = i915_gem_object_create_internal(gt->i915, sz); 155 if (IS_ERR(obj)) { 156 i915_active_fini(&node->active); 157 kfree(node); 158 return ERR_CAST(obj); 159 } 160 161 i915_gem_object_set_readonly(obj); 162 163 node->type = type; 164 node->obj = obj; 165 return node; 166 } 167 168 struct intel_gt_buffer_pool_node * 169 intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size, 170 enum i915_map_type type) 171 { 172 struct intel_gt_buffer_pool *pool = >->buffer_pool; 173 struct intel_gt_buffer_pool_node *node; 174 struct list_head *list; 175 int ret; 176 177 size = PAGE_ALIGN(size); 178 list = bucket_for_size(pool, size); 179 180 rcu_read_lock(); 181 list_for_each_entry_rcu(node, list, link) { 182 unsigned long age; 183 184 if (node->obj->base.size < size) 185 continue; 186 187 if (node->type != type) 188 continue; 189 190 age = READ_ONCE(node->age); 191 if (!age) 192 continue; 193 194 if (cmpxchg(&node->age, age, 0) == age) { 195 spin_lock_irq(&pool->lock); 196 list_del_rcu(&node->link); 197 spin_unlock_irq(&pool->lock); 198 break; 199 } 200 } 201 rcu_read_unlock(); 202 203 if (&node->link == list) { 204 node = node_create(pool, size, type); 205 if (IS_ERR(node)) 206 return node; 207 } 208 209 ret = i915_active_acquire(&node->active); 210 if (ret) { 211 node_free(node); 212 return ERR_PTR(ret); 213 } 214 215 return node; 216 } 217 218 void intel_gt_init_buffer_pool(struct intel_gt *gt) 219 { 220 struct intel_gt_buffer_pool *pool = >->buffer_pool; 221 int n; 222 223 spin_lock_init(&pool->lock); 224 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) 225 INIT_LIST_HEAD(&pool->cache_list[n]); 226 INIT_DELAYED_WORK(&pool->work, pool_free_work); 227 } 228 229 void intel_gt_flush_buffer_pool(struct intel_gt *gt) 230 { 231 struct intel_gt_buffer_pool *pool = >->buffer_pool; 232 233 do { 234 while (pool_free_older_than(pool, 0)) 235 ; 236 } while (cancel_delayed_work_sync(&pool->work)); 237 } 238 239 void intel_gt_fini_buffer_pool(struct intel_gt *gt) 240 { 241 struct intel_gt_buffer_pool *pool = >->buffer_pool; 242 int n; 243 244 for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) 245 GEM_BUG_ON(!list_empty(&pool->cache_list[n])); 246 } 247