mprotect.c (692d42d411b7db6a76382537fccbee3a12a2bcdb) mprotect.c (2bad466cc9d9b4c3b4b16eb9c03c919b59561316)
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * mm/mprotect.c
4 *
5 * (C) Copyright 1994 Linus Torvalds
6 * (C) Copyright 2002 Christoph Hellwig
7 *
8 * Address space accounting code <alan@lxorguk.ukuu.org.uk>

--- 262 unchanged lines hidden (view full) ---

271
272 if (!pte_same(oldpte, newpte)) {
273 set_pte_at(vma->vm_mm, addr, pte, newpte);
274 pages++;
275 }
276 } else {
277 /* It must be an none page, or what else?.. */
278 WARN_ON_ONCE(!pte_none(oldpte));
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * mm/mprotect.c
4 *
5 * (C) Copyright 1994 Linus Torvalds
6 * (C) Copyright 2002 Christoph Hellwig
7 *
8 * Address space accounting code <alan@lxorguk.ukuu.org.uk>

--- 262 unchanged lines hidden (view full) ---

271
272 if (!pte_same(oldpte, newpte)) {
273 set_pte_at(vma->vm_mm, addr, pte, newpte);
274 pages++;
275 }
276 } else {
277 /* It must be an none page, or what else?.. */
278 WARN_ON_ONCE(!pte_none(oldpte));
279 if (unlikely(uffd_wp && !vma_is_anonymous(vma))) {
279
280 /*
281 * Nobody plays with any none ptes besides
282 * userfaultfd when applying the protections.
283 */
284 if (likely(!uffd_wp))
285 continue;
286
287 if (userfaultfd_wp_use_markers(vma)) {
280 /*
281 * For file-backed mem, we need to be able to
282 * wr-protect a none pte, because even if the
283 * pte is none, the page/swap cache could
284 * exist. Doing that by install a marker.
285 */
286 set_pte_at(vma->vm_mm, addr, pte,
287 make_pte_marker(PTE_MARKER_UFFD_WP));

--- 27 unchanged lines hidden (view full) ---

315 if (unlikely(pmd_bad(pmdval))) {
316 pmd_clear_bad(pmd);
317 return 1;
318 }
319
320 return 0;
321}
322
288 /*
289 * For file-backed mem, we need to be able to
290 * wr-protect a none pte, because even if the
291 * pte is none, the page/swap cache could
292 * exist. Doing that by install a marker.
293 */
294 set_pte_at(vma->vm_mm, addr, pte,
295 make_pte_marker(PTE_MARKER_UFFD_WP));

--- 27 unchanged lines hidden (view full) ---

323 if (unlikely(pmd_bad(pmdval))) {
324 pmd_clear_bad(pmd);
325 return 1;
326 }
327
328 return 0;
329}
330
323/* Return true if we're uffd wr-protecting file-backed memory, or false */
331/*
332 * Return true if we want to split THPs into PTE mappings in change
333 * protection procedure, false otherwise.
334 */
324static inline bool
335static inline bool
325uffd_wp_protect_file(struct vm_area_struct *vma, unsigned long cp_flags)
336pgtable_split_needed(struct vm_area_struct *vma, unsigned long cp_flags)
326{
337{
338 /*
339 * pte markers only resides in pte level, if we need pte markers,
340 * we need to split. We cannot wr-protect shmem thp because file
341 * thp is handled differently when split by erasing the pmd so far.
342 */
327 return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma);
328}
329
330/*
343 return (cp_flags & MM_CP_UFFD_WP) && !vma_is_anonymous(vma);
344}
345
346/*
331 * If wr-protecting the range for file-backed, populate pgtable for the case
332 * when pgtable is empty but page cache exists. When {pte|pmd|...}_alloc()
333 * failed we treat it the same way as pgtable allocation failures during
334 * page faults by kicking OOM and returning error.
347 * Return true if we want to populate pgtables in change protection
348 * procedure, false otherwise
335 */
349 */
350static inline bool
351pgtable_populate_needed(struct vm_area_struct *vma, unsigned long cp_flags)
352{
353 /* If not within ioctl(UFFDIO_WRITEPROTECT), then don't bother */
354 if (!(cp_flags & MM_CP_UFFD_WP))
355 return false;
356
357 /* Populate if the userfaultfd mode requires pte markers */
358 return userfaultfd_wp_use_markers(vma);
359}
360
361/*
362 * Populate the pgtable underneath for whatever reason if requested.
363 * When {pte|pmd|...}_alloc() failed we treat it the same way as pgtable
364 * allocation failures during page faults by kicking OOM and returning
365 * error.
366 */
336#define change_pmd_prepare(vma, pmd, cp_flags) \
337 ({ \
338 long err = 0; \
367#define change_pmd_prepare(vma, pmd, cp_flags) \
368 ({ \
369 long err = 0; \
339 if (unlikely(uffd_wp_protect_file(vma, cp_flags))) { \
370 if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \
340 if (pte_alloc(vma->vm_mm, pmd)) \
341 err = -ENOMEM; \
342 } \
343 err; \
344 })
345
346/*
347 * This is the general pud/p4d/pgd version of change_pmd_prepare(). We need to
348 * have separate change_pmd_prepare() because pte_alloc() returns 0 on success,
349 * while {pmd|pud|p4d}_alloc() returns the valid pointer on success.
350 */
351#define change_prepare(vma, high, low, addr, cp_flags) \
352 ({ \
353 long err = 0; \
371 if (pte_alloc(vma->vm_mm, pmd)) \
372 err = -ENOMEM; \
373 } \
374 err; \
375 })
376
377/*
378 * This is the general pud/p4d/pgd version of change_pmd_prepare(). We need to
379 * have separate change_pmd_prepare() because pte_alloc() returns 0 on success,
380 * while {pmd|pud|p4d}_alloc() returns the valid pointer on success.
381 */
382#define change_prepare(vma, high, low, addr, cp_flags) \
383 ({ \
384 long err = 0; \
354 if (unlikely(uffd_wp_protect_file(vma, cp_flags))) { \
385 if (unlikely(pgtable_populate_needed(vma, cp_flags))) { \
355 low##_t *p = low##_alloc(vma->vm_mm, high, addr); \
356 if (p == NULL) \
357 err = -ENOMEM; \
358 } \
359 err; \
360 })
361
362static inline long change_pmd_range(struct mmu_gather *tlb,

--- 36 unchanged lines hidden (view full) ---

399 mmu_notifier_range_init(&range,
400 MMU_NOTIFY_PROTECTION_VMA, 0,
401 vma->vm_mm, addr, end);
402 mmu_notifier_invalidate_range_start(&range);
403 }
404
405 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
406 if ((next - addr != HPAGE_PMD_SIZE) ||
386 low##_t *p = low##_alloc(vma->vm_mm, high, addr); \
387 if (p == NULL) \
388 err = -ENOMEM; \
389 } \
390 err; \
391 })
392
393static inline long change_pmd_range(struct mmu_gather *tlb,

--- 36 unchanged lines hidden (view full) ---

430 mmu_notifier_range_init(&range,
431 MMU_NOTIFY_PROTECTION_VMA, 0,
432 vma->vm_mm, addr, end);
433 mmu_notifier_invalidate_range_start(&range);
434 }
435
436 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
437 if ((next - addr != HPAGE_PMD_SIZE) ||
407 uffd_wp_protect_file(vma, cp_flags)) {
438 pgtable_split_needed(vma, cp_flags)) {
408 __split_huge_pmd(vma, pmd, addr, false, NULL);
409 /*
410 * For file-backed, the pmd could have been
411 * cleared; make sure pmd populated if
412 * necessary, then fall-through to pte level.
413 */
414 ret = change_pmd_prepare(vma, pmd, cp_flags);
415 if (ret) {

--- 493 unchanged lines hidden --- 		439 __split_huge_pmd(vma, pmd, addr, false, NULL);
440 /*
441 * For file-backed, the pmd could have been
442 * cleared; make sure pmd populated if
443 * necessary, then fall-through to pte level.
444 */
445 ret = change_pmd_prepare(vma, pmd, cp_flags);
446 if (ret) {

--- 493 unchanged lines hidden ---