1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2016 Red Hat
4  * Author: Rob Clark <robdclark@gmail.com>
5  */
6 
7 #include <linux/vmalloc.h>
8 #include <linux/sched/mm.h>
9 
10 #include "msm_drv.h"
11 #include "msm_gem.h"
12 #include "msm_gpu.h"
13 #include "msm_gpu_trace.h"
14 
15 /* Default disabled for now until it has some more testing on the different
16  * iommu combinations that can be paired with the driver:
17  */
18 static bool enable_eviction = true;
19 MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers");
20 module_param(enable_eviction, bool, 0600);
21 
22 static bool can_swap(void)
23 {
24 	return enable_eviction && get_nr_swap_pages() > 0;
25 }
26 
27 static bool can_block(struct shrink_control *sc)
28 {
29 	if (!(sc->gfp_mask & __GFP_DIRECT_RECLAIM))
30 		return false;
31 	return current_is_kswapd() || (sc->gfp_mask & __GFP_RECLAIM);
32 }
33 
34 static unsigned long
35 msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc)
36 {
37 	struct msm_drm_private *priv =
38 		container_of(shrinker, struct msm_drm_private, shrinker);
39 	unsigned count = priv->lru.dontneed.count;
40 
41 	if (can_swap())
42 		count += priv->lru.willneed.count;
43 
44 	return count;
45 }
46 
47 static bool
48 purge(struct drm_gem_object *obj)
49 {
50 	if (!is_purgeable(to_msm_bo(obj)))
51 		return false;
52 
53 	if (msm_gem_active(obj))
54 		return false;
55 
56 	msm_gem_purge(obj);
57 
58 	return true;
59 }
60 
61 static bool
62 evict(struct drm_gem_object *obj)
63 {
64 	if (is_unevictable(to_msm_bo(obj)))
65 		return false;
66 
67 	if (msm_gem_active(obj))
68 		return false;
69 
70 	msm_gem_evict(obj);
71 
72 	return true;
73 }
74 
75 static bool
76 wait_for_idle(struct drm_gem_object *obj)
77 {
78 	enum dma_resv_usage usage = dma_resv_usage_rw(true);
79 	return dma_resv_wait_timeout(obj->resv, usage, false, 1000) > 0;
80 }
81 
82 static bool
83 active_purge(struct drm_gem_object *obj)
84 {
85 	if (!wait_for_idle(obj))
86 		return false;
87 
88 	return purge(obj);
89 }
90 
91 static bool
92 active_evict(struct drm_gem_object *obj)
93 {
94 	if (!wait_for_idle(obj))
95 		return false;
96 
97 	return evict(obj);
98 }
99 
100 static unsigned long
101 msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc)
102 {
103 	struct msm_drm_private *priv =
104 		container_of(shrinker, struct msm_drm_private, shrinker);
105 	struct {
106 		struct drm_gem_lru *lru;
107 		bool (*shrink)(struct drm_gem_object *obj);
108 		bool cond;
109 		unsigned long freed;
110 		unsigned long remaining;
111 	} stages[] = {
112 		/* Stages of progressively more aggressive/expensive reclaim: */
113 		{ &priv->lru.dontneed, purge,        true },
114 		{ &priv->lru.willneed, evict,        can_swap() },
115 		{ &priv->lru.dontneed, active_purge, can_block(sc) },
116 		{ &priv->lru.willneed, active_evict, can_swap() && can_block(sc) },
117 	};
118 	long nr = sc->nr_to_scan;
119 	unsigned long freed = 0;
120 	unsigned long remaining = 0;
121 
122 	for (unsigned i = 0; (nr > 0) && (i < ARRAY_SIZE(stages)); i++) {
123 		if (!stages[i].cond)
124 			continue;
125 		stages[i].freed =
126 			drm_gem_lru_scan(stages[i].lru, nr,
127 					&stages[i].remaining,
128 					 stages[i].shrink);
129 		nr -= stages[i].freed;
130 		freed += stages[i].freed;
131 		remaining += stages[i].remaining;
132 	}
133 
134 	if (freed) {
135 		trace_msm_gem_shrink(sc->nr_to_scan, stages[0].freed,
136 				     stages[1].freed, stages[2].freed,
137 				     stages[3].freed);
138 	}
139 
140 	return (freed > 0 && remaining > 0) ? freed : SHRINK_STOP;
141 }
142 
143 #ifdef CONFIG_DEBUG_FS
144 unsigned long
145 msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan)
146 {
147 	struct msm_drm_private *priv = dev->dev_private;
148 	struct shrink_control sc = {
149 		.nr_to_scan = nr_to_scan,
150 	};
151 	int ret;
152 
153 	fs_reclaim_acquire(GFP_KERNEL);
154 	ret = msm_gem_shrinker_scan(&priv->shrinker, &sc);
155 	fs_reclaim_release(GFP_KERNEL);
156 
157 	return ret;
158 }
159 #endif
160 
161 /* since we don't know any better, lets bail after a few
162  * and if necessary the shrinker will be invoked again.
163  * Seems better than unmapping *everything*
164  */
165 static const int vmap_shrink_limit = 15;
166 
167 static bool
168 vmap_shrink(struct drm_gem_object *obj)
169 {
170 	if (!is_vunmapable(to_msm_bo(obj)))
171 		return false;
172 
173 	msm_gem_vunmap(obj);
174 
175 	return true;
176 }
177 
178 static int
179 msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr)
180 {
181 	struct msm_drm_private *priv =
182 		container_of(nb, struct msm_drm_private, vmap_notifier);
183 	struct drm_gem_lru *lrus[] = {
184 		&priv->lru.dontneed,
185 		&priv->lru.willneed,
186 		&priv->lru.pinned,
187 		NULL,
188 	};
189 	unsigned idx, unmapped = 0;
190 	unsigned long remaining = 0;
191 
192 	for (idx = 0; lrus[idx] && unmapped < vmap_shrink_limit; idx++) {
193 		unmapped += drm_gem_lru_scan(lrus[idx],
194 					     vmap_shrink_limit - unmapped,
195 					     &remaining,
196 					     vmap_shrink);
197 	}
198 
199 	*(unsigned long *)ptr += unmapped;
200 
201 	if (unmapped > 0)
202 		trace_msm_gem_purge_vmaps(unmapped);
203 
204 	return NOTIFY_DONE;
205 }
206 
207 /**
208  * msm_gem_shrinker_init - Initialize msm shrinker
209  * @dev: drm device
210  *
211  * This function registers and sets up the msm shrinker.
212  */
213 void msm_gem_shrinker_init(struct drm_device *dev)
214 {
215 	struct msm_drm_private *priv = dev->dev_private;
216 	priv->shrinker.count_objects = msm_gem_shrinker_count;
217 	priv->shrinker.scan_objects = msm_gem_shrinker_scan;
218 	priv->shrinker.seeks = DEFAULT_SEEKS;
219 	WARN_ON(register_shrinker(&priv->shrinker, "drm-msm_gem"));
220 
221 	priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap;
222 	WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier));
223 }
224 
225 /**
226  * msm_gem_shrinker_cleanup - Clean up msm shrinker
227  * @dev: drm device
228  *
229  * This function unregisters the msm shrinker.
230  */
231 void msm_gem_shrinker_cleanup(struct drm_device *dev)
232 {
233 	struct msm_drm_private *priv = dev->dev_private;
234 
235 	if (priv->shrinker.nr_deferred) {
236 		WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier));
237 		unregister_shrinker(&priv->shrinker);
238 	}
239 }
240