1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2016 Red Hat 4 * Author: Rob Clark <robdclark@gmail.com> 5 */ 6 7 #include <linux/vmalloc.h> 8 9 #include "msm_drv.h" 10 #include "msm_gem.h" 11 #include "msm_gpu.h" 12 #include "msm_gpu_trace.h" 13 14 /* Default disabled for now until it has some more testing on the different 15 * iommu combinations that can be paired with the driver: 16 */ 17 bool enable_eviction = false; 18 MODULE_PARM_DESC(enable_eviction, "Enable swappable GEM buffers"); 19 module_param(enable_eviction, bool, 0600); 20 21 static bool can_swap(void) 22 { 23 return enable_eviction && get_nr_swap_pages() > 0; 24 } 25 26 static unsigned long 27 msm_gem_shrinker_count(struct shrinker *shrinker, struct shrink_control *sc) 28 { 29 struct msm_drm_private *priv = 30 container_of(shrinker, struct msm_drm_private, shrinker); 31 unsigned count = priv->shrinkable_count; 32 33 if (can_swap()) 34 count += priv->evictable_count; 35 36 return count; 37 } 38 39 static bool 40 purge(struct msm_gem_object *msm_obj) 41 { 42 if (!is_purgeable(msm_obj)) 43 return false; 44 45 /* 46 * This will move the obj out of still_in_list to 47 * the purged list 48 */ 49 msm_gem_purge(&msm_obj->base); 50 51 return true; 52 } 53 54 static bool 55 evict(struct msm_gem_object *msm_obj) 56 { 57 if (is_unevictable(msm_obj)) 58 return false; 59 60 msm_gem_evict(&msm_obj->base); 61 62 return true; 63 } 64 65 static unsigned long 66 scan(struct msm_drm_private *priv, unsigned nr_to_scan, struct list_head *list, 67 bool (*shrink)(struct msm_gem_object *msm_obj)) 68 { 69 unsigned freed = 0; 70 struct list_head still_in_list; 71 72 INIT_LIST_HEAD(&still_in_list); 73 74 mutex_lock(&priv->mm_lock); 75 76 while (freed < nr_to_scan) { 77 struct msm_gem_object *msm_obj = list_first_entry_or_null( 78 list, typeof(*msm_obj), mm_list); 79 80 if (!msm_obj) 81 break; 82 83 list_move_tail(&msm_obj->mm_list, &still_in_list); 84 85 /* 86 * If it is in the process of being freed, msm_gem_free_object 87 * can be blocked on mm_lock waiting to remove it. So just 88 * skip it. 89 */ 90 if (!kref_get_unless_zero(&msm_obj->base.refcount)) 91 continue; 92 93 /* 94 * Now that we own a reference, we can drop mm_lock for the 95 * rest of the loop body, to reduce contention with the 96 * retire_submit path (which could make more objects purgeable) 97 */ 98 99 mutex_unlock(&priv->mm_lock); 100 101 /* 102 * Note that this still needs to be trylock, since we can 103 * hit shrinker in response to trying to get backing pages 104 * for this obj (ie. while it's lock is already held) 105 */ 106 if (!msm_gem_trylock(&msm_obj->base)) 107 goto tail; 108 109 if (shrink(msm_obj)) 110 freed += msm_obj->base.size >> PAGE_SHIFT; 111 112 msm_gem_unlock(&msm_obj->base); 113 114 tail: 115 drm_gem_object_put(&msm_obj->base); 116 mutex_lock(&priv->mm_lock); 117 } 118 119 list_splice_tail(&still_in_list, list); 120 mutex_unlock(&priv->mm_lock); 121 122 return freed; 123 } 124 125 static unsigned long 126 msm_gem_shrinker_scan(struct shrinker *shrinker, struct shrink_control *sc) 127 { 128 struct msm_drm_private *priv = 129 container_of(shrinker, struct msm_drm_private, shrinker); 130 unsigned long freed; 131 132 freed = scan(priv, sc->nr_to_scan, &priv->inactive_dontneed, purge); 133 134 if (freed > 0) 135 trace_msm_gem_purge(freed << PAGE_SHIFT); 136 137 if (can_swap() && freed < sc->nr_to_scan) { 138 int evicted = scan(priv, sc->nr_to_scan - freed, 139 &priv->inactive_willneed, evict); 140 141 if (evicted > 0) 142 trace_msm_gem_evict(evicted << PAGE_SHIFT); 143 144 freed += evicted; 145 } 146 147 return (freed > 0) ? freed : SHRINK_STOP; 148 } 149 150 #ifdef CONFIG_DEBUG_FS 151 unsigned long 152 msm_gem_shrinker_shrink(struct drm_device *dev, unsigned long nr_to_scan) 153 { 154 struct msm_drm_private *priv = dev->dev_private; 155 struct shrink_control sc = { 156 .nr_to_scan = nr_to_scan, 157 }; 158 int ret; 159 160 fs_reclaim_acquire(GFP_KERNEL); 161 ret = msm_gem_shrinker_scan(&priv->shrinker, &sc); 162 fs_reclaim_release(GFP_KERNEL); 163 164 return ret; 165 } 166 #endif 167 168 /* since we don't know any better, lets bail after a few 169 * and if necessary the shrinker will be invoked again. 170 * Seems better than unmapping *everything* 171 */ 172 static const int vmap_shrink_limit = 15; 173 174 static bool 175 vmap_shrink(struct msm_gem_object *msm_obj) 176 { 177 if (!is_vunmapable(msm_obj)) 178 return false; 179 180 msm_gem_vunmap(&msm_obj->base); 181 182 return true; 183 } 184 185 static int 186 msm_gem_shrinker_vmap(struct notifier_block *nb, unsigned long event, void *ptr) 187 { 188 struct msm_drm_private *priv = 189 container_of(nb, struct msm_drm_private, vmap_notifier); 190 struct list_head *mm_lists[] = { 191 &priv->inactive_dontneed, 192 &priv->inactive_willneed, 193 priv->gpu ? &priv->gpu->active_list : NULL, 194 NULL, 195 }; 196 unsigned idx, unmapped = 0; 197 198 for (idx = 0; mm_lists[idx] && unmapped < vmap_shrink_limit; idx++) { 199 unmapped += scan(priv, vmap_shrink_limit - unmapped, 200 mm_lists[idx], vmap_shrink); 201 } 202 203 *(unsigned long *)ptr += unmapped; 204 205 if (unmapped > 0) 206 trace_msm_gem_purge_vmaps(unmapped); 207 208 return NOTIFY_DONE; 209 } 210 211 /** 212 * msm_gem_shrinker_init - Initialize msm shrinker 213 * @dev: drm device 214 * 215 * This function registers and sets up the msm shrinker. 216 */ 217 void msm_gem_shrinker_init(struct drm_device *dev) 218 { 219 struct msm_drm_private *priv = dev->dev_private; 220 priv->shrinker.count_objects = msm_gem_shrinker_count; 221 priv->shrinker.scan_objects = msm_gem_shrinker_scan; 222 priv->shrinker.seeks = DEFAULT_SEEKS; 223 WARN_ON(register_shrinker(&priv->shrinker)); 224 225 priv->vmap_notifier.notifier_call = msm_gem_shrinker_vmap; 226 WARN_ON(register_vmap_purge_notifier(&priv->vmap_notifier)); 227 } 228 229 /** 230 * msm_gem_shrinker_cleanup - Clean up msm shrinker 231 * @dev: drm device 232 * 233 * This function unregisters the msm shrinker. 234 */ 235 void msm_gem_shrinker_cleanup(struct drm_device *dev) 236 { 237 struct msm_drm_private *priv = dev->dev_private; 238 239 if (priv->shrinker.nr_deferred) { 240 WARN_ON(unregister_vmap_purge_notifier(&priv->vmap_notifier)); 241 unregister_shrinker(&priv->shrinker); 242 } 243 } 244