1 /*
2  * Collaborative memory management interface.
3  *
4  * Copyright (C) 2008 IBM Corporation
5  * Author(s): Brian King (brking@linux.vnet.ibm.com),
6  *
7  * This program is free software; you can redistribute it and/or modify
8  * it under the terms of the GNU General Public License as published by
9  * the Free Software Foundation; either version 2 of the License, or
10  * (at your option) any later version.
11  *
12  * This program is distributed in the hope that it will be useful,
13  * but WITHOUT ANY WARRANTY; without even the implied warranty of
14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
15  * GNU General Public License for more details.
16  *
17  * You should have received a copy of the GNU General Public License
18  * along with this program; if not, write to the Free Software
19  * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
20  *
21  */
22 
23 #include <linux/ctype.h>
24 #include <linux/delay.h>
25 #include <linux/errno.h>
26 #include <linux/fs.h>
27 #include <linux/gfp.h>
28 #include <linux/kthread.h>
29 #include <linux/module.h>
30 #include <linux/oom.h>
31 #include <linux/reboot.h>
32 #include <linux/sched.h>
33 #include <linux/stringify.h>
34 #include <linux/swap.h>
35 #include <linux/device.h>
36 #include <asm/firmware.h>
37 #include <asm/hvcall.h>
38 #include <asm/mmu.h>
39 #include <asm/pgalloc.h>
40 #include <linux/uaccess.h>
41 #include <linux/memory.h>
42 #include <asm/plpar_wrappers.h>
43 
44 #include "pseries.h"
45 
46 #define CMM_DRIVER_VERSION	"1.0.0"
47 #define CMM_DEFAULT_DELAY	1
48 #define CMM_HOTPLUG_DELAY	5
49 #define CMM_DEBUG			0
50 #define CMM_DISABLE		0
51 #define CMM_OOM_KB		1024
52 #define CMM_MIN_MEM_MB		256
53 #define KB2PAGES(_p)		((_p)>>(PAGE_SHIFT-10))
54 #define PAGES2KB(_p)		((_p)<<(PAGE_SHIFT-10))
55 /*
56  * The priority level tries to ensure that this notifier is called as
57  * late as possible to reduce thrashing in the shared memory pool.
58  */
59 #define CMM_MEM_HOTPLUG_PRI	1
60 #define CMM_MEM_ISOLATE_PRI	15
61 
62 static unsigned int delay = CMM_DEFAULT_DELAY;
63 static unsigned int hotplug_delay = CMM_HOTPLUG_DELAY;
64 static unsigned int oom_kb = CMM_OOM_KB;
65 static unsigned int cmm_debug = CMM_DEBUG;
66 static unsigned int cmm_disabled = CMM_DISABLE;
67 static unsigned long min_mem_mb = CMM_MIN_MEM_MB;
68 static struct device cmm_dev;
69 
70 MODULE_AUTHOR("Brian King <brking@linux.vnet.ibm.com>");
71 MODULE_DESCRIPTION("IBM System p Collaborative Memory Manager");
72 MODULE_LICENSE("GPL");
73 MODULE_VERSION(CMM_DRIVER_VERSION);
74 
75 module_param_named(delay, delay, uint, 0644);
76 MODULE_PARM_DESC(delay, "Delay (in seconds) between polls to query hypervisor paging requests. "
77 		 "[Default=" __stringify(CMM_DEFAULT_DELAY) "]");
78 module_param_named(hotplug_delay, hotplug_delay, uint, 0644);
79 MODULE_PARM_DESC(hotplug_delay, "Delay (in seconds) after memory hotplug remove "
80 		 "before loaning resumes. "
81 		 "[Default=" __stringify(CMM_HOTPLUG_DELAY) "]");
82 module_param_named(oom_kb, oom_kb, uint, 0644);
83 MODULE_PARM_DESC(oom_kb, "Amount of memory in kb to free on OOM. "
84 		 "[Default=" __stringify(CMM_OOM_KB) "]");
85 module_param_named(min_mem_mb, min_mem_mb, ulong, 0644);
86 MODULE_PARM_DESC(min_mem_mb, "Minimum amount of memory (in MB) to not balloon. "
87 		 "[Default=" __stringify(CMM_MIN_MEM_MB) "]");
88 module_param_named(debug, cmm_debug, uint, 0644);
89 MODULE_PARM_DESC(debug, "Enable module debugging logging. Set to 1 to enable. "
90 		 "[Default=" __stringify(CMM_DEBUG) "]");
91 
92 #define CMM_NR_PAGES ((PAGE_SIZE - sizeof(void *) - sizeof(unsigned long)) / sizeof(unsigned long))
93 
94 #define cmm_dbg(...) if (cmm_debug) { printk(KERN_INFO "cmm: "__VA_ARGS__); }
95 
96 struct cmm_page_array {
97 	struct cmm_page_array *next;
98 	unsigned long index;
99 	unsigned long page[CMM_NR_PAGES];
100 };
101 
102 static unsigned long loaned_pages;
103 static unsigned long loaned_pages_target;
104 static unsigned long oom_freed_pages;
105 
106 static struct cmm_page_array *cmm_page_list;
107 static DEFINE_SPINLOCK(cmm_lock);
108 
109 static DEFINE_MUTEX(hotplug_mutex);
110 static int hotplug_occurred; /* protected by the hotplug mutex */
111 
112 static struct task_struct *cmm_thread_ptr;
113 
114 static long plpar_page_set_loaned(unsigned long vpa)
115 {
116 	unsigned long cmo_page_sz = cmo_get_page_size();
117 	long rc = 0;
118 	int i;
119 
120 	for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
121 		rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED, vpa + i, 0);
122 
123 	for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
124 		plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE,
125 				   vpa + i - cmo_page_sz, 0);
126 
127 	return rc;
128 }
129 
130 static long plpar_page_set_active(unsigned long vpa)
131 {
132 	unsigned long cmo_page_sz = cmo_get_page_size();
133 	long rc = 0;
134 	int i;
135 
136 	for (i = 0; !rc && i < PAGE_SIZE; i += cmo_page_sz)
137 		rc = plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_ACTIVE, vpa + i, 0);
138 
139 	for (i -= cmo_page_sz; rc && i != 0; i -= cmo_page_sz)
140 		plpar_hcall_norets(H_PAGE_INIT, H_PAGE_SET_LOANED,
141 				   vpa + i - cmo_page_sz, 0);
142 
143 	return rc;
144 }
145 
146 /**
147  * cmm_alloc_pages - Allocate pages and mark them as loaned
148  * @nr:	number of pages to allocate
149  *
150  * Return value:
151  * 	number of pages requested to be allocated which were not
152  **/
153 static long cmm_alloc_pages(long nr)
154 {
155 	struct cmm_page_array *pa, *npa;
156 	unsigned long addr;
157 	long rc;
158 
159 	cmm_dbg("Begin request for %ld pages\n", nr);
160 
161 	while (nr) {
162 		/* Exit if a hotplug operation is in progress or occurred */
163 		if (mutex_trylock(&hotplug_mutex)) {
164 			if (hotplug_occurred) {
165 				mutex_unlock(&hotplug_mutex);
166 				break;
167 			}
168 			mutex_unlock(&hotplug_mutex);
169 		} else {
170 			break;
171 		}
172 
173 		addr = __get_free_page(GFP_NOIO | __GFP_NOWARN |
174 				       __GFP_NORETRY | __GFP_NOMEMALLOC);
175 		if (!addr)
176 			break;
177 		spin_lock(&cmm_lock);
178 		pa = cmm_page_list;
179 		if (!pa || pa->index >= CMM_NR_PAGES) {
180 			/* Need a new page for the page list. */
181 			spin_unlock(&cmm_lock);
182 			npa = (struct cmm_page_array *)__get_free_page(
183 					GFP_NOIO | __GFP_NOWARN |
184 					__GFP_NORETRY | __GFP_NOMEMALLOC);
185 			if (!npa) {
186 				pr_info("%s: Can not allocate new page list\n", __func__);
187 				free_page(addr);
188 				break;
189 			}
190 			spin_lock(&cmm_lock);
191 			pa = cmm_page_list;
192 
193 			if (!pa || pa->index >= CMM_NR_PAGES) {
194 				npa->next = pa;
195 				npa->index = 0;
196 				pa = npa;
197 				cmm_page_list = pa;
198 			} else
199 				free_page((unsigned long) npa);
200 		}
201 
202 		if ((rc = plpar_page_set_loaned(__pa(addr)))) {
203 			pr_err("%s: Can not set page to loaned. rc=%ld\n", __func__, rc);
204 			spin_unlock(&cmm_lock);
205 			free_page(addr);
206 			break;
207 		}
208 
209 		pa->page[pa->index++] = addr;
210 		loaned_pages++;
211 		totalram_pages--;
212 		spin_unlock(&cmm_lock);
213 		nr--;
214 	}
215 
216 	cmm_dbg("End request with %ld pages unfulfilled\n", nr);
217 	return nr;
218 }
219 
220 /**
221  * cmm_free_pages - Free pages and mark them as active
222  * @nr:	number of pages to free
223  *
224  * Return value:
225  * 	number of pages requested to be freed which were not
226  **/
227 static long cmm_free_pages(long nr)
228 {
229 	struct cmm_page_array *pa;
230 	unsigned long addr;
231 
232 	cmm_dbg("Begin free of %ld pages.\n", nr);
233 	spin_lock(&cmm_lock);
234 	pa = cmm_page_list;
235 	while (nr) {
236 		if (!pa || pa->index <= 0)
237 			break;
238 		addr = pa->page[--pa->index];
239 
240 		if (pa->index == 0) {
241 			pa = pa->next;
242 			free_page((unsigned long) cmm_page_list);
243 			cmm_page_list = pa;
244 		}
245 
246 		plpar_page_set_active(__pa(addr));
247 		free_page(addr);
248 		loaned_pages--;
249 		nr--;
250 		totalram_pages++;
251 	}
252 	spin_unlock(&cmm_lock);
253 	cmm_dbg("End request with %ld pages unfulfilled\n", nr);
254 	return nr;
255 }
256 
257 /**
258  * cmm_oom_notify - OOM notifier
259  * @self:	notifier block struct
260  * @dummy:	not used
261  * @parm:	returned - number of pages freed
262  *
263  * Return value:
264  * 	NOTIFY_OK
265  **/
266 static int cmm_oom_notify(struct notifier_block *self,
267 			  unsigned long dummy, void *parm)
268 {
269 	unsigned long *freed = parm;
270 	long nr = KB2PAGES(oom_kb);
271 
272 	cmm_dbg("OOM processing started\n");
273 	nr = cmm_free_pages(nr);
274 	loaned_pages_target = loaned_pages;
275 	*freed += KB2PAGES(oom_kb) - nr;
276 	oom_freed_pages += KB2PAGES(oom_kb) - nr;
277 	cmm_dbg("OOM processing complete\n");
278 	return NOTIFY_OK;
279 }
280 
281 /**
282  * cmm_get_mpp - Read memory performance parameters
283  *
284  * Makes hcall to query the current page loan request from the hypervisor.
285  *
286  * Return value:
287  * 	nothing
288  **/
289 static void cmm_get_mpp(void)
290 {
291 	int rc;
292 	struct hvcall_mpp_data mpp_data;
293 	signed long active_pages_target, page_loan_request, target;
294 	signed long total_pages = totalram_pages + loaned_pages;
295 	signed long min_mem_pages = (min_mem_mb * 1024 * 1024) / PAGE_SIZE;
296 
297 	rc = h_get_mpp(&mpp_data);
298 
299 	if (rc != H_SUCCESS)
300 		return;
301 
302 	page_loan_request = div_s64((s64)mpp_data.loan_request, PAGE_SIZE);
303 	target = page_loan_request + (signed long)loaned_pages;
304 
305 	if (target < 0 || total_pages < min_mem_pages)
306 		target = 0;
307 
308 	if (target > oom_freed_pages)
309 		target -= oom_freed_pages;
310 	else
311 		target = 0;
312 
313 	active_pages_target = total_pages - target;
314 
315 	if (min_mem_pages > active_pages_target)
316 		target = total_pages - min_mem_pages;
317 
318 	if (target < 0)
319 		target = 0;
320 
321 	loaned_pages_target = target;
322 
323 	cmm_dbg("delta = %ld, loaned = %lu, target = %lu, oom = %lu, totalram = %lu\n",
324 		page_loan_request, loaned_pages, loaned_pages_target,
325 		oom_freed_pages, totalram_pages);
326 }
327 
328 static struct notifier_block cmm_oom_nb = {
329 	.notifier_call = cmm_oom_notify
330 };
331 
332 /**
333  * cmm_thread - CMM task thread
334  * @dummy:	not used
335  *
336  * Return value:
337  * 	0
338  **/
339 static int cmm_thread(void *dummy)
340 {
341 	unsigned long timeleft;
342 
343 	while (1) {
344 		timeleft = msleep_interruptible(delay * 1000);
345 
346 		if (kthread_should_stop() || timeleft)
347 			break;
348 
349 		if (mutex_trylock(&hotplug_mutex)) {
350 			if (hotplug_occurred) {
351 				hotplug_occurred = 0;
352 				mutex_unlock(&hotplug_mutex);
353 				cmm_dbg("Hotplug operation has occurred, "
354 						"loaning activity suspended "
355 						"for %d seconds.\n",
356 						hotplug_delay);
357 				timeleft = msleep_interruptible(hotplug_delay *
358 						1000);
359 				if (kthread_should_stop() || timeleft)
360 					break;
361 				continue;
362 			}
363 			mutex_unlock(&hotplug_mutex);
364 		} else {
365 			cmm_dbg("Hotplug operation in progress, activity "
366 					"suspended\n");
367 			continue;
368 		}
369 
370 		cmm_get_mpp();
371 
372 		if (loaned_pages_target > loaned_pages) {
373 			if (cmm_alloc_pages(loaned_pages_target - loaned_pages))
374 				loaned_pages_target = loaned_pages;
375 		} else if (loaned_pages_target < loaned_pages)
376 			cmm_free_pages(loaned_pages - loaned_pages_target);
377 	}
378 	return 0;
379 }
380 
381 #define CMM_SHOW(name, format, args...)			\
382 	static ssize_t show_##name(struct device *dev,	\
383 				   struct device_attribute *attr,	\
384 				   char *buf)			\
385 	{							\
386 		return sprintf(buf, format, ##args);		\
387 	}							\
388 	static DEVICE_ATTR(name, 0444, show_##name, NULL)
389 
390 CMM_SHOW(loaned_kb, "%lu\n", PAGES2KB(loaned_pages));
391 CMM_SHOW(loaned_target_kb, "%lu\n", PAGES2KB(loaned_pages_target));
392 
393 static ssize_t show_oom_pages(struct device *dev,
394 			      struct device_attribute *attr, char *buf)
395 {
396 	return sprintf(buf, "%lu\n", PAGES2KB(oom_freed_pages));
397 }
398 
399 static ssize_t store_oom_pages(struct device *dev,
400 			       struct device_attribute *attr,
401 			       const char *buf, size_t count)
402 {
403 	unsigned long val = simple_strtoul (buf, NULL, 10);
404 
405 	if (!capable(CAP_SYS_ADMIN))
406 		return -EPERM;
407 	if (val != 0)
408 		return -EBADMSG;
409 
410 	oom_freed_pages = 0;
411 	return count;
412 }
413 
414 static DEVICE_ATTR(oom_freed_kb, 0644,
415 		   show_oom_pages, store_oom_pages);
416 
417 static struct device_attribute *cmm_attrs[] = {
418 	&dev_attr_loaned_kb,
419 	&dev_attr_loaned_target_kb,
420 	&dev_attr_oom_freed_kb,
421 };
422 
423 static struct bus_type cmm_subsys = {
424 	.name = "cmm",
425 	.dev_name = "cmm",
426 };
427 
428 /**
429  * cmm_sysfs_register - Register with sysfs
430  *
431  * Return value:
432  * 	0 on success / other on failure
433  **/
434 static int cmm_sysfs_register(struct device *dev)
435 {
436 	int i, rc;
437 
438 	if ((rc = subsys_system_register(&cmm_subsys, NULL)))
439 		return rc;
440 
441 	dev->id = 0;
442 	dev->bus = &cmm_subsys;
443 
444 	if ((rc = device_register(dev)))
445 		goto subsys_unregister;
446 
447 	for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++) {
448 		if ((rc = device_create_file(dev, cmm_attrs[i])))
449 			goto fail;
450 	}
451 
452 	return 0;
453 
454 fail:
455 	while (--i >= 0)
456 		device_remove_file(dev, cmm_attrs[i]);
457 	device_unregister(dev);
458 subsys_unregister:
459 	bus_unregister(&cmm_subsys);
460 	return rc;
461 }
462 
463 /**
464  * cmm_unregister_sysfs - Unregister from sysfs
465  *
466  **/
467 static void cmm_unregister_sysfs(struct device *dev)
468 {
469 	int i;
470 
471 	for (i = 0; i < ARRAY_SIZE(cmm_attrs); i++)
472 		device_remove_file(dev, cmm_attrs[i]);
473 	device_unregister(dev);
474 	bus_unregister(&cmm_subsys);
475 }
476 
477 /**
478  * cmm_reboot_notifier - Make sure pages are not still marked as "loaned"
479  *
480  **/
481 static int cmm_reboot_notifier(struct notifier_block *nb,
482 			       unsigned long action, void *unused)
483 {
484 	if (action == SYS_RESTART) {
485 		if (cmm_thread_ptr)
486 			kthread_stop(cmm_thread_ptr);
487 		cmm_thread_ptr = NULL;
488 		cmm_free_pages(loaned_pages);
489 	}
490 	return NOTIFY_DONE;
491 }
492 
493 static struct notifier_block cmm_reboot_nb = {
494 	.notifier_call = cmm_reboot_notifier,
495 };
496 
497 /**
498  * cmm_count_pages - Count the number of pages loaned in a particular range.
499  *
500  * @arg: memory_isolate_notify structure with address range and count
501  *
502  * Return value:
503  *      0 on success
504  **/
505 static unsigned long cmm_count_pages(void *arg)
506 {
507 	struct memory_isolate_notify *marg = arg;
508 	struct cmm_page_array *pa;
509 	unsigned long start = (unsigned long)pfn_to_kaddr(marg->start_pfn);
510 	unsigned long end = start + (marg->nr_pages << PAGE_SHIFT);
511 	unsigned long idx;
512 
513 	spin_lock(&cmm_lock);
514 	pa = cmm_page_list;
515 	while (pa) {
516 		if ((unsigned long)pa >= start && (unsigned long)pa < end)
517 			marg->pages_found++;
518 		for (idx = 0; idx < pa->index; idx++)
519 			if (pa->page[idx] >= start && pa->page[idx] < end)
520 				marg->pages_found++;
521 		pa = pa->next;
522 	}
523 	spin_unlock(&cmm_lock);
524 	return 0;
525 }
526 
527 /**
528  * cmm_memory_isolate_cb - Handle memory isolation notifier calls
529  * @self:	notifier block struct
530  * @action:	action to take
531  * @arg:	struct memory_isolate_notify data for handler
532  *
533  * Return value:
534  *	NOTIFY_OK or notifier error based on subfunction return value
535  **/
536 static int cmm_memory_isolate_cb(struct notifier_block *self,
537 				 unsigned long action, void *arg)
538 {
539 	int ret = 0;
540 
541 	if (action == MEM_ISOLATE_COUNT)
542 		ret = cmm_count_pages(arg);
543 
544 	return notifier_from_errno(ret);
545 }
546 
547 static struct notifier_block cmm_mem_isolate_nb = {
548 	.notifier_call = cmm_memory_isolate_cb,
549 	.priority = CMM_MEM_ISOLATE_PRI
550 };
551 
552 /**
553  * cmm_mem_going_offline - Unloan pages where memory is to be removed
554  * @arg: memory_notify structure with page range to be offlined
555  *
556  * Return value:
557  *	0 on success
558  **/
559 static int cmm_mem_going_offline(void *arg)
560 {
561 	struct memory_notify *marg = arg;
562 	unsigned long start_page = (unsigned long)pfn_to_kaddr(marg->start_pfn);
563 	unsigned long end_page = start_page + (marg->nr_pages << PAGE_SHIFT);
564 	struct cmm_page_array *pa_curr, *pa_last, *npa;
565 	unsigned long idx;
566 	unsigned long freed = 0;
567 
568 	cmm_dbg("Memory going offline, searching 0x%lx (%ld pages).\n",
569 			start_page, marg->nr_pages);
570 	spin_lock(&cmm_lock);
571 
572 	/* Search the page list for pages in the range to be offlined */
573 	pa_last = pa_curr = cmm_page_list;
574 	while (pa_curr) {
575 		for (idx = (pa_curr->index - 1); (idx + 1) > 0; idx--) {
576 			if ((pa_curr->page[idx] < start_page) ||
577 			    (pa_curr->page[idx] >= end_page))
578 				continue;
579 
580 			plpar_page_set_active(__pa(pa_curr->page[idx]));
581 			free_page(pa_curr->page[idx]);
582 			freed++;
583 			loaned_pages--;
584 			totalram_pages++;
585 			pa_curr->page[idx] = pa_last->page[--pa_last->index];
586 			if (pa_last->index == 0) {
587 				if (pa_curr == pa_last)
588 					pa_curr = pa_last->next;
589 				pa_last = pa_last->next;
590 				free_page((unsigned long)cmm_page_list);
591 				cmm_page_list = pa_last;
592 			}
593 		}
594 		pa_curr = pa_curr->next;
595 	}
596 
597 	/* Search for page list structures in the range to be offlined */
598 	pa_last = NULL;
599 	pa_curr = cmm_page_list;
600 	while (pa_curr) {
601 		if (((unsigned long)pa_curr >= start_page) &&
602 				((unsigned long)pa_curr < end_page)) {
603 			npa = (struct cmm_page_array *)__get_free_page(
604 					GFP_NOIO | __GFP_NOWARN |
605 					__GFP_NORETRY | __GFP_NOMEMALLOC);
606 			if (!npa) {
607 				spin_unlock(&cmm_lock);
608 				cmm_dbg("Failed to allocate memory for list "
609 						"management. Memory hotplug "
610 						"failed.\n");
611 				return -ENOMEM;
612 			}
613 			memcpy(npa, pa_curr, PAGE_SIZE);
614 			if (pa_curr == cmm_page_list)
615 				cmm_page_list = npa;
616 			if (pa_last)
617 				pa_last->next = npa;
618 			free_page((unsigned long) pa_curr);
619 			freed++;
620 			pa_curr = npa;
621 		}
622 
623 		pa_last = pa_curr;
624 		pa_curr = pa_curr->next;
625 	}
626 
627 	spin_unlock(&cmm_lock);
628 	cmm_dbg("Released %ld pages in the search range.\n", freed);
629 
630 	return 0;
631 }
632 
633 /**
634  * cmm_memory_cb - Handle memory hotplug notifier calls
635  * @self:	notifier block struct
636  * @action:	action to take
637  * @arg:	struct memory_notify data for handler
638  *
639  * Return value:
640  *	NOTIFY_OK or notifier error based on subfunction return value
641  *
642  **/
643 static int cmm_memory_cb(struct notifier_block *self,
644 			unsigned long action, void *arg)
645 {
646 	int ret = 0;
647 
648 	switch (action) {
649 	case MEM_GOING_OFFLINE:
650 		mutex_lock(&hotplug_mutex);
651 		hotplug_occurred = 1;
652 		ret = cmm_mem_going_offline(arg);
653 		break;
654 	case MEM_OFFLINE:
655 	case MEM_CANCEL_OFFLINE:
656 		mutex_unlock(&hotplug_mutex);
657 		cmm_dbg("Memory offline operation complete.\n");
658 		break;
659 	case MEM_GOING_ONLINE:
660 	case MEM_ONLINE:
661 	case MEM_CANCEL_ONLINE:
662 		break;
663 	}
664 
665 	return notifier_from_errno(ret);
666 }
667 
668 static struct notifier_block cmm_mem_nb = {
669 	.notifier_call = cmm_memory_cb,
670 	.priority = CMM_MEM_HOTPLUG_PRI
671 };
672 
673 /**
674  * cmm_init - Module initialization
675  *
676  * Return value:
677  * 	0 on success / other on failure
678  **/
679 static int cmm_init(void)
680 {
681 	int rc = -ENOMEM;
682 
683 	if (!firmware_has_feature(FW_FEATURE_CMO))
684 		return -EOPNOTSUPP;
685 
686 	if ((rc = register_oom_notifier(&cmm_oom_nb)) < 0)
687 		return rc;
688 
689 	if ((rc = register_reboot_notifier(&cmm_reboot_nb)))
690 		goto out_oom_notifier;
691 
692 	if ((rc = cmm_sysfs_register(&cmm_dev)))
693 		goto out_reboot_notifier;
694 
695 	if (register_memory_notifier(&cmm_mem_nb) ||
696 	    register_memory_isolate_notifier(&cmm_mem_isolate_nb))
697 		goto out_unregister_notifier;
698 
699 	if (cmm_disabled)
700 		return rc;
701 
702 	cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread");
703 	if (IS_ERR(cmm_thread_ptr)) {
704 		rc = PTR_ERR(cmm_thread_ptr);
705 		goto out_unregister_notifier;
706 	}
707 
708 	return rc;
709 
710 out_unregister_notifier:
711 	unregister_memory_notifier(&cmm_mem_nb);
712 	unregister_memory_isolate_notifier(&cmm_mem_isolate_nb);
713 	cmm_unregister_sysfs(&cmm_dev);
714 out_reboot_notifier:
715 	unregister_reboot_notifier(&cmm_reboot_nb);
716 out_oom_notifier:
717 	unregister_oom_notifier(&cmm_oom_nb);
718 	return rc;
719 }
720 
721 /**
722  * cmm_exit - Module exit
723  *
724  * Return value:
725  * 	nothing
726  **/
727 static void cmm_exit(void)
728 {
729 	if (cmm_thread_ptr)
730 		kthread_stop(cmm_thread_ptr);
731 	unregister_oom_notifier(&cmm_oom_nb);
732 	unregister_reboot_notifier(&cmm_reboot_nb);
733 	unregister_memory_notifier(&cmm_mem_nb);
734 	unregister_memory_isolate_notifier(&cmm_mem_isolate_nb);
735 	cmm_free_pages(loaned_pages);
736 	cmm_unregister_sysfs(&cmm_dev);
737 }
738 
739 /**
740  * cmm_set_disable - Disable/Enable CMM
741  *
742  * Return value:
743  * 	0 on success / other on failure
744  **/
745 static int cmm_set_disable(const char *val, const struct kernel_param *kp)
746 {
747 	int disable = simple_strtoul(val, NULL, 10);
748 
749 	if (disable != 0 && disable != 1)
750 		return -EINVAL;
751 
752 	if (disable && !cmm_disabled) {
753 		if (cmm_thread_ptr)
754 			kthread_stop(cmm_thread_ptr);
755 		cmm_thread_ptr = NULL;
756 		cmm_free_pages(loaned_pages);
757 	} else if (!disable && cmm_disabled) {
758 		cmm_thread_ptr = kthread_run(cmm_thread, NULL, "cmmthread");
759 		if (IS_ERR(cmm_thread_ptr))
760 			return PTR_ERR(cmm_thread_ptr);
761 	}
762 
763 	cmm_disabled = disable;
764 	return 0;
765 }
766 
767 module_param_call(disable, cmm_set_disable, param_get_uint,
768 		  &cmm_disabled, 0644);
769 MODULE_PARM_DESC(disable, "Disable CMM. Set to 1 to disable. "
770 		 "[Default=" __stringify(CMM_DISABLE) "]");
771 
772 module_init(cmm_init);
773 module_exit(cmm_exit);
774