1 #include <linux/swap_cgroup.h> 2 #include <linux/vmalloc.h> 3 #include <linux/mm.h> 4 5 #include <linux/swapops.h> /* depends on mm.h include */ 6 7 static DEFINE_MUTEX(swap_cgroup_mutex); 8 struct swap_cgroup_ctrl { 9 struct page **map; 10 unsigned long length; 11 spinlock_t lock; 12 }; 13 14 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES]; 15 16 struct swap_cgroup { 17 unsigned short id; 18 }; 19 #define SC_PER_PAGE (PAGE_SIZE/sizeof(struct swap_cgroup)) 20 21 /* 22 * SwapCgroup implements "lookup" and "exchange" operations. 23 * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge 24 * against SwapCache. At swap_free(), this is accessed directly from swap. 25 * 26 * This means, 27 * - we have no race in "exchange" when we're accessed via SwapCache because 28 * SwapCache(and its swp_entry) is under lock. 29 * - When called via swap_free(), there is no user of this entry and no race. 30 * Then, we don't need lock around "exchange". 31 * 32 * TODO: we can push these buffers out to HIGHMEM. 33 */ 34 35 /* 36 * allocate buffer for swap_cgroup. 37 */ 38 static int swap_cgroup_prepare(int type) 39 { 40 struct page *page; 41 struct swap_cgroup_ctrl *ctrl; 42 unsigned long idx, max; 43 44 ctrl = &swap_cgroup_ctrl[type]; 45 46 for (idx = 0; idx < ctrl->length; idx++) { 47 page = alloc_page(GFP_KERNEL | __GFP_ZERO); 48 if (!page) 49 goto not_enough_page; 50 ctrl->map[idx] = page; 51 52 if (!(idx % SWAP_CLUSTER_MAX)) 53 cond_resched(); 54 } 55 return 0; 56 not_enough_page: 57 max = idx; 58 for (idx = 0; idx < max; idx++) 59 __free_page(ctrl->map[idx]); 60 61 return -ENOMEM; 62 } 63 64 static struct swap_cgroup *__lookup_swap_cgroup(struct swap_cgroup_ctrl *ctrl, 65 pgoff_t offset) 66 { 67 struct page *mappage; 68 struct swap_cgroup *sc; 69 70 mappage = ctrl->map[offset / SC_PER_PAGE]; 71 sc = page_address(mappage); 72 return sc + offset % SC_PER_PAGE; 73 } 74 75 static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent, 76 struct swap_cgroup_ctrl **ctrlp) 77 { 78 pgoff_t offset = swp_offset(ent); 79 struct swap_cgroup_ctrl *ctrl; 80 81 ctrl = &swap_cgroup_ctrl[swp_type(ent)]; 82 if (ctrlp) 83 *ctrlp = ctrl; 84 return __lookup_swap_cgroup(ctrl, offset); 85 } 86 87 /** 88 * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry. 89 * @ent: swap entry to be cmpxchged 90 * @old: old id 91 * @new: new id 92 * 93 * Returns old id at success, 0 at failure. 94 * (There is no mem_cgroup using 0 as its id) 95 */ 96 unsigned short swap_cgroup_cmpxchg(swp_entry_t ent, 97 unsigned short old, unsigned short new) 98 { 99 struct swap_cgroup_ctrl *ctrl; 100 struct swap_cgroup *sc; 101 unsigned long flags; 102 unsigned short retval; 103 104 sc = lookup_swap_cgroup(ent, &ctrl); 105 106 spin_lock_irqsave(&ctrl->lock, flags); 107 retval = sc->id; 108 if (retval == old) 109 sc->id = new; 110 else 111 retval = 0; 112 spin_unlock_irqrestore(&ctrl->lock, flags); 113 return retval; 114 } 115 116 /** 117 * swap_cgroup_record - record mem_cgroup for a set of swap entries 118 * @ent: the first swap entry to be recorded into 119 * @id: mem_cgroup to be recorded 120 * @nr_ents: number of swap entries to be recorded 121 * 122 * Returns old value at success, 0 at failure. 123 * (Of course, old value can be 0.) 124 */ 125 unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id, 126 unsigned int nr_ents) 127 { 128 struct swap_cgroup_ctrl *ctrl; 129 struct swap_cgroup *sc; 130 unsigned short old; 131 unsigned long flags; 132 pgoff_t offset = swp_offset(ent); 133 pgoff_t end = offset + nr_ents; 134 135 sc = lookup_swap_cgroup(ent, &ctrl); 136 137 spin_lock_irqsave(&ctrl->lock, flags); 138 old = sc->id; 139 for (;;) { 140 VM_BUG_ON(sc->id != old); 141 sc->id = id; 142 offset++; 143 if (offset == end) 144 break; 145 if (offset % SC_PER_PAGE) 146 sc++; 147 else 148 sc = __lookup_swap_cgroup(ctrl, offset); 149 } 150 spin_unlock_irqrestore(&ctrl->lock, flags); 151 152 return old; 153 } 154 155 /** 156 * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry 157 * @ent: swap entry to be looked up. 158 * 159 * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID) 160 */ 161 unsigned short lookup_swap_cgroup_id(swp_entry_t ent) 162 { 163 return lookup_swap_cgroup(ent, NULL)->id; 164 } 165 166 int swap_cgroup_swapon(int type, unsigned long max_pages) 167 { 168 void *array; 169 unsigned long array_size; 170 unsigned long length; 171 struct swap_cgroup_ctrl *ctrl; 172 173 if (!do_swap_account) 174 return 0; 175 176 length = DIV_ROUND_UP(max_pages, SC_PER_PAGE); 177 array_size = length * sizeof(void *); 178 179 array = vzalloc(array_size); 180 if (!array) 181 goto nomem; 182 183 ctrl = &swap_cgroup_ctrl[type]; 184 mutex_lock(&swap_cgroup_mutex); 185 ctrl->length = length; 186 ctrl->map = array; 187 spin_lock_init(&ctrl->lock); 188 if (swap_cgroup_prepare(type)) { 189 /* memory shortage */ 190 ctrl->map = NULL; 191 ctrl->length = 0; 192 mutex_unlock(&swap_cgroup_mutex); 193 vfree(array); 194 goto nomem; 195 } 196 mutex_unlock(&swap_cgroup_mutex); 197 198 return 0; 199 nomem: 200 pr_info("couldn't allocate enough memory for swap_cgroup\n"); 201 pr_info("swap_cgroup can be disabled by swapaccount=0 boot option\n"); 202 return -ENOMEM; 203 } 204 205 void swap_cgroup_swapoff(int type) 206 { 207 struct page **map; 208 unsigned long i, length; 209 struct swap_cgroup_ctrl *ctrl; 210 211 if (!do_swap_account) 212 return; 213 214 mutex_lock(&swap_cgroup_mutex); 215 ctrl = &swap_cgroup_ctrl[type]; 216 map = ctrl->map; 217 length = ctrl->length; 218 ctrl->map = NULL; 219 ctrl->length = 0; 220 mutex_unlock(&swap_cgroup_mutex); 221 222 if (map) { 223 for (i = 0; i < length; i++) { 224 struct page *page = map[i]; 225 if (page) 226 __free_page(page); 227 if (!(i % SWAP_CLUSTER_MAX)) 228 cond_resched(); 229 } 230 vfree(map); 231 } 232 } 233