xref: /openbmc/linux/mm/swap_cgroup.c (revision 94cdda6b)
1 #include <linux/swap_cgroup.h>
2 #include <linux/vmalloc.h>
3 #include <linux/mm.h>
4 
5 #include <linux/swapops.h> /* depends on mm.h include */
6 
7 static DEFINE_MUTEX(swap_cgroup_mutex);
8 struct swap_cgroup_ctrl {
9 	struct page **map;
10 	unsigned long length;
11 	spinlock_t	lock;
12 };
13 
14 static struct swap_cgroup_ctrl swap_cgroup_ctrl[MAX_SWAPFILES];
15 
16 struct swap_cgroup {
17 	unsigned short		id;
18 };
19 #define SC_PER_PAGE	(PAGE_SIZE/sizeof(struct swap_cgroup))
20 
21 /*
22  * SwapCgroup implements "lookup" and "exchange" operations.
23  * In typical usage, this swap_cgroup is accessed via memcg's charge/uncharge
24  * against SwapCache. At swap_free(), this is accessed directly from swap.
25  *
26  * This means,
27  *  - we have no race in "exchange" when we're accessed via SwapCache because
28  *    SwapCache(and its swp_entry) is under lock.
29  *  - When called via swap_free(), there is no user of this entry and no race.
30  * Then, we don't need lock around "exchange".
31  *
32  * TODO: we can push these buffers out to HIGHMEM.
33  */
34 
35 /*
36  * allocate buffer for swap_cgroup.
37  */
38 static int swap_cgroup_prepare(int type)
39 {
40 	struct page *page;
41 	struct swap_cgroup_ctrl *ctrl;
42 	unsigned long idx, max;
43 
44 	ctrl = &swap_cgroup_ctrl[type];
45 
46 	for (idx = 0; idx < ctrl->length; idx++) {
47 		page = alloc_page(GFP_KERNEL | __GFP_ZERO);
48 		if (!page)
49 			goto not_enough_page;
50 		ctrl->map[idx] = page;
51 	}
52 	return 0;
53 not_enough_page:
54 	max = idx;
55 	for (idx = 0; idx < max; idx++)
56 		__free_page(ctrl->map[idx]);
57 
58 	return -ENOMEM;
59 }
60 
61 static struct swap_cgroup *lookup_swap_cgroup(swp_entry_t ent,
62 					struct swap_cgroup_ctrl **ctrlp)
63 {
64 	pgoff_t offset = swp_offset(ent);
65 	struct swap_cgroup_ctrl *ctrl;
66 	struct page *mappage;
67 	struct swap_cgroup *sc;
68 
69 	ctrl = &swap_cgroup_ctrl[swp_type(ent)];
70 	if (ctrlp)
71 		*ctrlp = ctrl;
72 
73 	mappage = ctrl->map[offset / SC_PER_PAGE];
74 	sc = page_address(mappage);
75 	return sc + offset % SC_PER_PAGE;
76 }
77 
78 /**
79  * swap_cgroup_cmpxchg - cmpxchg mem_cgroup's id for this swp_entry.
80  * @ent: swap entry to be cmpxchged
81  * @old: old id
82  * @new: new id
83  *
84  * Returns old id at success, 0 at failure.
85  * (There is no mem_cgroup using 0 as its id)
86  */
87 unsigned short swap_cgroup_cmpxchg(swp_entry_t ent,
88 					unsigned short old, unsigned short new)
89 {
90 	struct swap_cgroup_ctrl *ctrl;
91 	struct swap_cgroup *sc;
92 	unsigned long flags;
93 	unsigned short retval;
94 
95 	sc = lookup_swap_cgroup(ent, &ctrl);
96 
97 	spin_lock_irqsave(&ctrl->lock, flags);
98 	retval = sc->id;
99 	if (retval == old)
100 		sc->id = new;
101 	else
102 		retval = 0;
103 	spin_unlock_irqrestore(&ctrl->lock, flags);
104 	return retval;
105 }
106 
107 /**
108  * swap_cgroup_record - record mem_cgroup for this swp_entry.
109  * @ent: swap entry to be recorded into
110  * @id: mem_cgroup to be recorded
111  *
112  * Returns old value at success, 0 at failure.
113  * (Of course, old value can be 0.)
114  */
115 unsigned short swap_cgroup_record(swp_entry_t ent, unsigned short id)
116 {
117 	struct swap_cgroup_ctrl *ctrl;
118 	struct swap_cgroup *sc;
119 	unsigned short old;
120 	unsigned long flags;
121 
122 	sc = lookup_swap_cgroup(ent, &ctrl);
123 
124 	spin_lock_irqsave(&ctrl->lock, flags);
125 	old = sc->id;
126 	sc->id = id;
127 	spin_unlock_irqrestore(&ctrl->lock, flags);
128 
129 	return old;
130 }
131 
132 /**
133  * lookup_swap_cgroup_id - lookup mem_cgroup id tied to swap entry
134  * @ent: swap entry to be looked up.
135  *
136  * Returns ID of mem_cgroup at success. 0 at failure. (0 is invalid ID)
137  */
138 unsigned short lookup_swap_cgroup_id(swp_entry_t ent)
139 {
140 	return lookup_swap_cgroup(ent, NULL)->id;
141 }
142 
143 int swap_cgroup_swapon(int type, unsigned long max_pages)
144 {
145 	void *array;
146 	unsigned long array_size;
147 	unsigned long length;
148 	struct swap_cgroup_ctrl *ctrl;
149 
150 	if (!do_swap_account)
151 		return 0;
152 
153 	length = DIV_ROUND_UP(max_pages, SC_PER_PAGE);
154 	array_size = length * sizeof(void *);
155 
156 	array = vzalloc(array_size);
157 	if (!array)
158 		goto nomem;
159 
160 	ctrl = &swap_cgroup_ctrl[type];
161 	mutex_lock(&swap_cgroup_mutex);
162 	ctrl->length = length;
163 	ctrl->map = array;
164 	spin_lock_init(&ctrl->lock);
165 	if (swap_cgroup_prepare(type)) {
166 		/* memory shortage */
167 		ctrl->map = NULL;
168 		ctrl->length = 0;
169 		mutex_unlock(&swap_cgroup_mutex);
170 		vfree(array);
171 		goto nomem;
172 	}
173 	mutex_unlock(&swap_cgroup_mutex);
174 
175 	return 0;
176 nomem:
177 	printk(KERN_INFO "couldn't allocate enough memory for swap_cgroup.\n");
178 	printk(KERN_INFO
179 		"swap_cgroup can be disabled by swapaccount=0 boot option\n");
180 	return -ENOMEM;
181 }
182 
183 void swap_cgroup_swapoff(int type)
184 {
185 	struct page **map;
186 	unsigned long i, length;
187 	struct swap_cgroup_ctrl *ctrl;
188 
189 	if (!do_swap_account)
190 		return;
191 
192 	mutex_lock(&swap_cgroup_mutex);
193 	ctrl = &swap_cgroup_ctrl[type];
194 	map = ctrl->map;
195 	length = ctrl->length;
196 	ctrl->map = NULL;
197 	ctrl->length = 0;
198 	mutex_unlock(&swap_cgroup_mutex);
199 
200 	if (map) {
201 		for (i = 0; i < length; i++) {
202 			struct page *page = map[i];
203 			if (page)
204 				__free_page(page);
205 		}
206 		vfree(map);
207 	}
208 }
209