mempolicy.c (7f9f879243d6cf5d2d60d12065e93189cc343387) mempolicy.c (c00b6b9610991c042ff4c3153daaa3ea8522c210)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Simple NUMA memory policy for the Linux kernel.
4 *
5 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
6 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should

--- 2182 unchanged lines hidden (view full) ---

2191 page = __alloc_pages(gfp, order,
2192 policy_node(gfp, pol, numa_node_id()),
2193 policy_nodemask(gfp, pol));
2194
2195 return page;
2196}
2197EXPORT_SYMBOL(alloc_pages);
2198
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Simple NUMA memory policy for the Linux kernel.
4 *
5 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
6 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should

--- 2182 unchanged lines hidden (view full) ---

2191 page = __alloc_pages(gfp, order,
2192 policy_node(gfp, pol, numa_node_id()),
2193 policy_nodemask(gfp, pol));
2194
2195 return page;
2196}
2197EXPORT_SYMBOL(alloc_pages);
2198
2199struct folio *folio_alloc(gfp_t gfp, unsigned order)
2199static unsigned long alloc_pages_bulk_array_interleave(gfp_t gfp,
2200 struct mempolicy *pol, unsigned long nr_pages,
2201 struct page **page_array)
2200{
2202{
2201 struct page *page = alloc_pages(gfp | __GFP_COMP, order);
2203 int nodes;
2204 unsigned long nr_pages_per_node;
2205 int delta;
2206 int i;
2207 unsigned long nr_allocated;
2208 unsigned long total_allocated = 0;
2202
2209
2203 if (page && order > 1)
2204 prep_transhuge_page(page);
2205 return (struct folio *)page;
2210 nodes = nodes_weight(pol->nodes);
2211 nr_pages_per_node = nr_pages / nodes;
2212 delta = nr_pages - nodes * nr_pages_per_node;
2213
2214 for (i = 0; i < nodes; i++) {
2215 if (delta) {
2216 nr_allocated = __alloc_pages_bulk(gfp,
2217 interleave_nodes(pol), NULL,
2218 nr_pages_per_node + 1, NULL,
2219 page_array);
2220 delta--;
2221 } else {
2222 nr_allocated = __alloc_pages_bulk(gfp,
2223 interleave_nodes(pol), NULL,
2224 nr_pages_per_node, NULL, page_array);
2225 }
2226
2227 page_array += nr_allocated;
2228 total_allocated += nr_allocated;
2229 }
2230
2231 return total_allocated;
2206}
2232}
2207EXPORT_SYMBOL(folio_alloc);
2208
2233
2234static unsigned long alloc_pages_bulk_array_preferred_many(gfp_t gfp, int nid,
2235 struct mempolicy *pol, unsigned long nr_pages,
2236 struct page **page_array)
2237{
2238 gfp_t preferred_gfp;
2239 unsigned long nr_allocated = 0;
2240
2241 preferred_gfp = gfp | __GFP_NOWARN;
2242 preferred_gfp &= ~(__GFP_DIRECT_RECLAIM | __GFP_NOFAIL);
2243
2244 nr_allocated = __alloc_pages_bulk(preferred_gfp, nid, &pol->nodes,
2245 nr_pages, NULL, page_array);
2246
2247 if (nr_allocated < nr_pages)
2248 nr_allocated += __alloc_pages_bulk(gfp, numa_node_id(), NULL,
2249 nr_pages - nr_allocated, NULL,
2250 page_array + nr_allocated);
2251 return nr_allocated;
2252}
2253
2254/* alloc pages bulk and mempolicy should be considered at the
2255 * same time in some situation such as vmalloc.
2256 *
2257 * It can accelerate memory allocation especially interleaving
2258 * allocate memory.
2259 */
2260unsigned long alloc_pages_bulk_array_mempolicy(gfp_t gfp,
2261 unsigned long nr_pages, struct page **page_array)
2262{
2263 struct mempolicy *pol = &default_policy;
2264
2265 if (!in_interrupt() && !(gfp & __GFP_THISNODE))
2266 pol = get_task_policy(current);
2267
2268 if (pol->mode == MPOL_INTERLEAVE)
2269 return alloc_pages_bulk_array_interleave(gfp, pol,
2270 nr_pages, page_array);
2271
2272 if (pol->mode == MPOL_PREFERRED_MANY)
2273 return alloc_pages_bulk_array_preferred_many(gfp,
2274 numa_node_id(), pol, nr_pages, page_array);
2275
2276 return __alloc_pages_bulk(gfp, policy_node(gfp, pol, numa_node_id()),
2277 policy_nodemask(gfp, pol), nr_pages, NULL,
2278 page_array);
2279}
2280
2209int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2210{
2211 struct mempolicy *pol = mpol_dup(vma_policy(src));
2212
2213 if (IS_ERR(pol))
2214 return PTR_ERR(pol);
2215 dst->vm_policy = pol;
2216 return 0;

--- 832 unchanged lines hidden --- 		2281int vma_dup_policy(struct vm_area_struct *src, struct vm_area_struct *dst)
2282{
2283 struct mempolicy *pol = mpol_dup(vma_policy(src));
2284
2285 if (IS_ERR(pol))
2286 return PTR_ERR(pol);
2287 dst->vm_policy = pol;
2288 return 0;

--- 832 unchanged lines hidden ---