1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/errno.h>
3 #include <linux/gfp.h>
4 #include <linux/kernel.h>
5 #include <linux/mm.h>
6 #include <linux/memremap.h>
7 #include <linux/slab.h>
8 
9 #include <asm/page.h>
10 
11 #include <xen/balloon.h>
12 #include <xen/page.h>
13 #include <xen/xen.h>
14 
15 static DEFINE_MUTEX(list_lock);
16 static struct page *page_list;
17 static unsigned int list_count;
18 
19 static struct resource *target_resource;
20 
21 /*
22  * If arch is not happy with system "iomem_resource" being used for
23  * the region allocation it can provide it's own view by creating specific
24  * Xen resource with unused regions of guest physical address space provided
25  * by the hypervisor.
26  */
arch_xen_unpopulated_init(struct resource ** res)27 int __weak __init arch_xen_unpopulated_init(struct resource **res)
28 {
29 	*res = &iomem_resource;
30 
31 	return 0;
32 }
33 
fill_list(unsigned int nr_pages)34 static int fill_list(unsigned int nr_pages)
35 {
36 	struct dev_pagemap *pgmap;
37 	struct resource *res, *tmp_res = NULL;
38 	void *vaddr;
39 	unsigned int i, alloc_pages = round_up(nr_pages, PAGES_PER_SECTION);
40 	struct range mhp_range;
41 	int ret;
42 
43 	res = kzalloc(sizeof(*res), GFP_KERNEL);
44 	if (!res)
45 		return -ENOMEM;
46 
47 	res->name = "Xen scratch";
48 	res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
49 
50 	mhp_range = mhp_get_pluggable_range(true);
51 
52 	ret = allocate_resource(target_resource, res,
53 				alloc_pages * PAGE_SIZE, mhp_range.start, mhp_range.end,
54 				PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL);
55 	if (ret < 0) {
56 		pr_err("Cannot allocate new IOMEM resource\n");
57 		goto err_resource;
58 	}
59 
60 	/*
61 	 * Reserve the region previously allocated from Xen resource to avoid
62 	 * re-using it by someone else.
63 	 */
64 	if (target_resource != &iomem_resource) {
65 		tmp_res = kzalloc(sizeof(*tmp_res), GFP_KERNEL);
66 		if (!tmp_res) {
67 			ret = -ENOMEM;
68 			goto err_insert;
69 		}
70 
71 		tmp_res->name = res->name;
72 		tmp_res->start = res->start;
73 		tmp_res->end = res->end;
74 		tmp_res->flags = res->flags;
75 
76 		ret = request_resource(&iomem_resource, tmp_res);
77 		if (ret < 0) {
78 			pr_err("Cannot request resource %pR (%d)\n", tmp_res, ret);
79 			kfree(tmp_res);
80 			goto err_insert;
81 		}
82 	}
83 
84 	pgmap = kzalloc(sizeof(*pgmap), GFP_KERNEL);
85 	if (!pgmap) {
86 		ret = -ENOMEM;
87 		goto err_pgmap;
88 	}
89 
90 	pgmap->type = MEMORY_DEVICE_GENERIC;
91 	pgmap->range = (struct range) {
92 		.start = res->start,
93 		.end = res->end,
94 	};
95 	pgmap->nr_range = 1;
96 	pgmap->owner = res;
97 
98 #ifdef CONFIG_XEN_HAVE_PVMMU
99         /*
100          * memremap will build page tables for the new memory so
101          * the p2m must contain invalid entries so the correct
102          * non-present PTEs will be written.
103          *
104          * If a failure occurs, the original (identity) p2m entries
105          * are not restored since this region is now known not to
106          * conflict with any devices.
107          */
108 	if (!xen_feature(XENFEAT_auto_translated_physmap)) {
109 		xen_pfn_t pfn = PFN_DOWN(res->start);
110 
111 		for (i = 0; i < alloc_pages; i++) {
112 			if (!set_phys_to_machine(pfn + i, INVALID_P2M_ENTRY)) {
113 				pr_warn("set_phys_to_machine() failed, no memory added\n");
114 				ret = -ENOMEM;
115 				goto err_memremap;
116 			}
117                 }
118 	}
119 #endif
120 
121 	vaddr = memremap_pages(pgmap, NUMA_NO_NODE);
122 	if (IS_ERR(vaddr)) {
123 		pr_err("Cannot remap memory range\n");
124 		ret = PTR_ERR(vaddr);
125 		goto err_memremap;
126 	}
127 
128 	for (i = 0; i < alloc_pages; i++) {
129 		struct page *pg = virt_to_page(vaddr + PAGE_SIZE * i);
130 
131 		pg->zone_device_data = page_list;
132 		page_list = pg;
133 		list_count++;
134 	}
135 
136 	return 0;
137 
138 err_memremap:
139 	kfree(pgmap);
140 err_pgmap:
141 	if (tmp_res) {
142 		release_resource(tmp_res);
143 		kfree(tmp_res);
144 	}
145 err_insert:
146 	release_resource(res);
147 err_resource:
148 	kfree(res);
149 	return ret;
150 }
151 
152 /**
153  * xen_alloc_unpopulated_pages - alloc unpopulated pages
154  * @nr_pages: Number of pages
155  * @pages: pages returned
156  * @return 0 on success, error otherwise
157  */
xen_alloc_unpopulated_pages(unsigned int nr_pages,struct page ** pages)158 int xen_alloc_unpopulated_pages(unsigned int nr_pages, struct page **pages)
159 {
160 	unsigned int i;
161 	int ret = 0;
162 
163 	/*
164 	 * Fallback to default behavior if we do not have any suitable resource
165 	 * to allocate required region from and as the result we won't be able to
166 	 * construct pages.
167 	 */
168 	if (!target_resource)
169 		return xen_alloc_ballooned_pages(nr_pages, pages);
170 
171 	mutex_lock(&list_lock);
172 	if (list_count < nr_pages) {
173 		ret = fill_list(nr_pages - list_count);
174 		if (ret)
175 			goto out;
176 	}
177 
178 	for (i = 0; i < nr_pages; i++) {
179 		struct page *pg = page_list;
180 
181 		BUG_ON(!pg);
182 		page_list = pg->zone_device_data;
183 		list_count--;
184 		pages[i] = pg;
185 
186 #ifdef CONFIG_XEN_HAVE_PVMMU
187 		if (!xen_feature(XENFEAT_auto_translated_physmap)) {
188 			ret = xen_alloc_p2m_entry(page_to_pfn(pg));
189 			if (ret < 0) {
190 				unsigned int j;
191 
192 				for (j = 0; j <= i; j++) {
193 					pages[j]->zone_device_data = page_list;
194 					page_list = pages[j];
195 					list_count++;
196 				}
197 				goto out;
198 			}
199 		}
200 #endif
201 	}
202 
203 out:
204 	mutex_unlock(&list_lock);
205 	return ret;
206 }
207 EXPORT_SYMBOL(xen_alloc_unpopulated_pages);
208 
209 /**
210  * xen_free_unpopulated_pages - return unpopulated pages
211  * @nr_pages: Number of pages
212  * @pages: pages to return
213  */
xen_free_unpopulated_pages(unsigned int nr_pages,struct page ** pages)214 void xen_free_unpopulated_pages(unsigned int nr_pages, struct page **pages)
215 {
216 	unsigned int i;
217 
218 	if (!target_resource) {
219 		xen_free_ballooned_pages(nr_pages, pages);
220 		return;
221 	}
222 
223 	mutex_lock(&list_lock);
224 	for (i = 0; i < nr_pages; i++) {
225 		pages[i]->zone_device_data = page_list;
226 		page_list = pages[i];
227 		list_count++;
228 	}
229 	mutex_unlock(&list_lock);
230 }
231 EXPORT_SYMBOL(xen_free_unpopulated_pages);
232 
unpopulated_init(void)233 static int __init unpopulated_init(void)
234 {
235 	int ret;
236 
237 	if (!xen_domain())
238 		return -ENODEV;
239 
240 	ret = arch_xen_unpopulated_init(&target_resource);
241 	if (ret) {
242 		pr_err("xen:unpopulated: Cannot initialize target resource\n");
243 		target_resource = NULL;
244 	}
245 
246 	return ret;
247 }
248 early_initcall(unpopulated_init);
249