mempolicy.c (0a2c1e8183163a31fe8c9838f3108aacf9c05c4a) mempolicy.c (d451b89dcd183da725eda84dfb8a46c0b32a4234)
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Simple NUMA memory policy for the Linux kernel.
4 *
5 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
6 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should

--- 413 unchanged lines hidden (view full) ---

422 unsigned long flags;
423 nodemask_t *nmask;
424 unsigned long start;
425 unsigned long end;
426 struct vm_area_struct *first;
427};
428
429/*
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Simple NUMA memory policy for the Linux kernel.
4 *
5 * Copyright 2003,2004 Andi Kleen, SuSE Labs.
6 * (C) Copyright 2005 Christoph Lameter, Silicon Graphics, Inc.
7 *
8 * NUMA policy allows the user to give hints in which node(s) memory should

--- 413 unchanged lines hidden (view full) ---

422 unsigned long flags;
423 nodemask_t *nmask;
424 unsigned long start;
425 unsigned long end;
426 struct vm_area_struct *first;
427};
428
429/*
430 * Check if the page's nid is in qp->nmask.
430 * Check if the folio's nid is in qp->nmask.
431 *
432 * If MPOL_MF_INVERT is set in qp->flags, check if the nid is
433 * in the invert of qp->nmask.
434 */
431 *
432 * If MPOL_MF_INVERT is set in qp->flags, check if the nid is
433 * in the invert of qp->nmask.
434 */
435static inline bool queue_pages_required(struct page *page,
435static inline bool queue_folio_required(struct folio *folio,
436 struct queue_pages *qp)
437{
436 struct queue_pages *qp)
437{
438 int nid = page_to_nid(page);
438 int nid = folio_nid(folio);
439 unsigned long flags = qp->flags;
440
441 return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);
442}
443
444/*
445 * queue_folios_pmd() has three possible return values:
446 * 0 - folios are placed on the right node or queued successfully, or

--- 17 unchanged lines hidden (view full) ---

464 ret = -EIO;
465 goto unlock;
466 }
467 folio = pfn_folio(pmd_pfn(*pmd));
468 if (is_huge_zero_page(&folio->page)) {
469 walk->action = ACTION_CONTINUE;
470 goto unlock;
471 }
439 unsigned long flags = qp->flags;
440
441 return node_isset(nid, *qp->nmask) == !(flags & MPOL_MF_INVERT);
442}
443
444/*
445 * queue_folios_pmd() has three possible return values:
446 * 0 - folios are placed on the right node or queued successfully, or

--- 17 unchanged lines hidden (view full) ---

464 ret = -EIO;
465 goto unlock;
466 }
467 folio = pfn_folio(pmd_pfn(*pmd));
468 if (is_huge_zero_page(&folio->page)) {
469 walk->action = ACTION_CONTINUE;
470 goto unlock;
471 }
472 if (!queue_pages_required(&folio->page, qp))
472 if (!queue_folio_required(folio, qp))
473 goto unlock;
474
475 flags = qp->flags;
476 /* go to folio migration */
477 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
478 if (!vma_migratable(walk->vma) ||
479 migrate_page_add(&folio->page, qp->pagelist, flags)) {
480 ret = 1;

--- 44 unchanged lines hidden (view full) ---

525 if (!folio || folio_is_zone_device(folio))
526 continue;
527 /*
528 * vm_normal_folio() filters out zero pages, but there might
529 * still be reserved folios to skip, perhaps in a VDSO.
530 */
531 if (folio_test_reserved(folio))
532 continue;
473 goto unlock;
474
475 flags = qp->flags;
476 /* go to folio migration */
477 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
478 if (!vma_migratable(walk->vma) ||
479 migrate_page_add(&folio->page, qp->pagelist, flags)) {
480 ret = 1;

--- 44 unchanged lines hidden (view full) ---

525 if (!folio || folio_is_zone_device(folio))
526 continue;
527 /*
528 * vm_normal_folio() filters out zero pages, but there might
529 * still be reserved folios to skip, perhaps in a VDSO.
530 */
531 if (folio_test_reserved(folio))
532 continue;
533 if (!queue_pages_required(&folio->page, qp))
533 if (!queue_folio_required(folio, qp))
534 continue;
535 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
536 /* MPOL_MF_STRICT must be specified if we get here */
537 if (!vma_migratable(vma)) {
538 has_unmovable = true;
539 break;
540 }
541

--- 28 unchanged lines hidden (view full) ---

570 spinlock_t *ptl;
571 pte_t entry;
572
573 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
574 entry = huge_ptep_get(pte);
575 if (!pte_present(entry))
576 goto unlock;
577 folio = pfn_folio(pte_pfn(entry));
534 continue;
535 if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
536 /* MPOL_MF_STRICT must be specified if we get here */
537 if (!vma_migratable(vma)) {
538 has_unmovable = true;
539 break;
540 }
541

--- 28 unchanged lines hidden (view full) ---

570 spinlock_t *ptl;
571 pte_t entry;
572
573 ptl = huge_pte_lock(hstate_vma(walk->vma), walk->mm, pte);
574 entry = huge_ptep_get(pte);
575 if (!pte_present(entry))
576 goto unlock;
577 folio = pfn_folio(pte_pfn(entry));
578 if (!queue_pages_required(&folio->page, qp))
578 if (!queue_folio_required(folio, qp))
579 goto unlock;
580
581 if (flags == MPOL_MF_STRICT) {
582 /*
583 * STRICT alone means only detecting misplaced folio and no
584 * need to further check other vma.
585 */
586 ret = -EIO;

--- 2577 unchanged lines hidden --- 		579 goto unlock;
580
581 if (flags == MPOL_MF_STRICT) {
582 /*
583 * STRICT alone means only detecting misplaced folio and no
584 * need to further check other vma.
585 */
586 ret = -EIO;

--- 2577 unchanged lines hidden ---