1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2014-2018 Intel Corporation
4  */
5 
6 #include "gem/i915_gem_internal.h"
7 #include "gem/i915_gem_object.h"
8 
9 #include "i915_drv.h"
10 #include "intel_engine_pm.h"
11 #include "intel_gt_buffer_pool.h"
12 
13 static struct list_head *
14 bucket_for_size(struct intel_gt_buffer_pool *pool, size_t sz)
15 {
16 	int n;
17 
18 	/*
19 	 * Compute a power-of-two bucket, but throw everything greater than
20 	 * 16KiB into the same bucket: i.e. the buckets hold objects of
21 	 * (1 page, 2 pages, 4 pages, 8+ pages).
22 	 */
23 	n = fls(sz >> PAGE_SHIFT) - 1;
24 	if (n >= ARRAY_SIZE(pool->cache_list))
25 		n = ARRAY_SIZE(pool->cache_list) - 1;
26 
27 	return &pool->cache_list[n];
28 }
29 
30 static void node_free(struct intel_gt_buffer_pool_node *node)
31 {
32 	i915_gem_object_put(node->obj);
33 	i915_active_fini(&node->active);
34 	kfree_rcu(node, rcu);
35 }
36 
37 static bool pool_free_older_than(struct intel_gt_buffer_pool *pool, long keep)
38 {
39 	struct intel_gt_buffer_pool_node *node, *stale = NULL;
40 	bool active = false;
41 	int n;
42 
43 	/* Free buffers that have not been used in the past second */
44 	for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++) {
45 		struct list_head *list = &pool->cache_list[n];
46 
47 		if (list_empty(list))
48 			continue;
49 
50 		if (spin_trylock_irq(&pool->lock)) {
51 			struct list_head *pos;
52 
53 			/* Most recent at head; oldest at tail */
54 			list_for_each_prev(pos, list) {
55 				unsigned long age;
56 
57 				node = list_entry(pos, typeof(*node), link);
58 
59 				age = READ_ONCE(node->age);
60 				if (!age || jiffies - age < keep)
61 					break;
62 
63 				/* Check we are the first to claim this node */
64 				if (!xchg(&node->age, 0))
65 					break;
66 
67 				node->free = stale;
68 				stale = node;
69 			}
70 			if (!list_is_last(pos, list))
71 				__list_del_many(pos, list);
72 
73 			spin_unlock_irq(&pool->lock);
74 		}
75 
76 		active |= !list_empty(list);
77 	}
78 
79 	while ((node = stale)) {
80 		stale = stale->free;
81 		node_free(node);
82 	}
83 
84 	return active;
85 }
86 
87 static void pool_free_work(struct work_struct *wrk)
88 {
89 	struct intel_gt_buffer_pool *pool =
90 		container_of(wrk, typeof(*pool), work.work);
91 	struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
92 
93 	if (pool_free_older_than(pool, HZ))
94 		queue_delayed_work(gt->i915->unordered_wq, &pool->work,
95 				   round_jiffies_up_relative(HZ));
96 }
97 
98 static void pool_retire(struct i915_active *ref)
99 {
100 	struct intel_gt_buffer_pool_node *node =
101 		container_of(ref, typeof(*node), active);
102 	struct intel_gt_buffer_pool *pool = node->pool;
103 	struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
104 	struct list_head *list = bucket_for_size(pool, node->obj->base.size);
105 	unsigned long flags;
106 
107 	if (node->pinned) {
108 		i915_gem_object_unpin_pages(node->obj);
109 
110 		/* Return this object to the shrinker pool */
111 		i915_gem_object_make_purgeable(node->obj);
112 		node->pinned = false;
113 	}
114 
115 	GEM_BUG_ON(node->age);
116 	spin_lock_irqsave(&pool->lock, flags);
117 	list_add_rcu(&node->link, list);
118 	WRITE_ONCE(node->age, jiffies ?: 1); /* 0 reserved for active nodes */
119 	spin_unlock_irqrestore(&pool->lock, flags);
120 
121 	queue_delayed_work(gt->i915->unordered_wq, &pool->work,
122 			   round_jiffies_up_relative(HZ));
123 }
124 
125 void intel_gt_buffer_pool_mark_used(struct intel_gt_buffer_pool_node *node)
126 {
127 	assert_object_held(node->obj);
128 
129 	if (node->pinned)
130 		return;
131 
132 	__i915_gem_object_pin_pages(node->obj);
133 	/* Hide this pinned object from the shrinker until retired */
134 	i915_gem_object_make_unshrinkable(node->obj);
135 	node->pinned = true;
136 }
137 
138 static struct intel_gt_buffer_pool_node *
139 node_create(struct intel_gt_buffer_pool *pool, size_t sz,
140 	    enum i915_map_type type)
141 {
142 	struct intel_gt *gt = container_of(pool, struct intel_gt, buffer_pool);
143 	struct intel_gt_buffer_pool_node *node;
144 	struct drm_i915_gem_object *obj;
145 
146 	node = kmalloc(sizeof(*node),
147 		       GFP_KERNEL | __GFP_RETRY_MAYFAIL | __GFP_NOWARN);
148 	if (!node)
149 		return ERR_PTR(-ENOMEM);
150 
151 	node->age = 0;
152 	node->pool = pool;
153 	node->pinned = false;
154 	i915_active_init(&node->active, NULL, pool_retire, 0);
155 
156 	obj = i915_gem_object_create_internal(gt->i915, sz);
157 	if (IS_ERR(obj)) {
158 		i915_active_fini(&node->active);
159 		kfree(node);
160 		return ERR_CAST(obj);
161 	}
162 
163 	i915_gem_object_set_readonly(obj);
164 
165 	node->type = type;
166 	node->obj = obj;
167 	return node;
168 }
169 
170 struct intel_gt_buffer_pool_node *
171 intel_gt_get_buffer_pool(struct intel_gt *gt, size_t size,
172 			 enum i915_map_type type)
173 {
174 	struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
175 	struct intel_gt_buffer_pool_node *node;
176 	struct list_head *list;
177 	int ret;
178 
179 	size = PAGE_ALIGN(size);
180 	list = bucket_for_size(pool, size);
181 
182 	rcu_read_lock();
183 	list_for_each_entry_rcu(node, list, link) {
184 		unsigned long age;
185 
186 		if (node->obj->base.size < size)
187 			continue;
188 
189 		if (node->type != type)
190 			continue;
191 
192 		age = READ_ONCE(node->age);
193 		if (!age)
194 			continue;
195 
196 		if (cmpxchg(&node->age, age, 0) == age) {
197 			spin_lock_irq(&pool->lock);
198 			list_del_rcu(&node->link);
199 			spin_unlock_irq(&pool->lock);
200 			break;
201 		}
202 	}
203 	rcu_read_unlock();
204 
205 	if (&node->link == list) {
206 		node = node_create(pool, size, type);
207 		if (IS_ERR(node))
208 			return node;
209 	}
210 
211 	ret = i915_active_acquire(&node->active);
212 	if (ret) {
213 		node_free(node);
214 		return ERR_PTR(ret);
215 	}
216 
217 	return node;
218 }
219 
220 void intel_gt_init_buffer_pool(struct intel_gt *gt)
221 {
222 	struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
223 	int n;
224 
225 	spin_lock_init(&pool->lock);
226 	for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
227 		INIT_LIST_HEAD(&pool->cache_list[n]);
228 	INIT_DELAYED_WORK(&pool->work, pool_free_work);
229 }
230 
231 void intel_gt_flush_buffer_pool(struct intel_gt *gt)
232 {
233 	struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
234 
235 	do {
236 		while (pool_free_older_than(pool, 0))
237 			;
238 	} while (cancel_delayed_work_sync(&pool->work));
239 }
240 
241 void intel_gt_fini_buffer_pool(struct intel_gt *gt)
242 {
243 	struct intel_gt_buffer_pool *pool = &gt->buffer_pool;
244 	int n;
245 
246 	for (n = 0; n < ARRAY_SIZE(pool->cache_list); n++)
247 		GEM_BUG_ON(!list_empty(&pool->cache_list[n]));
248 }
249