1 #ifndef _ASM_X86_PGTABLE_H 2 #define _ASM_X86_PGTABLE_H 3 4 #include <asm/page.h> 5 #include <asm/e820.h> 6 7 #include <asm/pgtable_types.h> 8 9 /* 10 * Macro to mark a page protection value as UC- 11 */ 12 #define pgprot_noncached(prot) \ 13 ((boot_cpu_data.x86 > 3) \ 14 ? (__pgprot(pgprot_val(prot) | _PAGE_CACHE_UC_MINUS)) \ 15 : (prot)) 16 17 #ifndef __ASSEMBLY__ 18 19 #include <asm/x86_init.h> 20 21 /* 22 * ZERO_PAGE is a global shared page that is always zero: used 23 * for zero-mapped memory areas etc.. 24 */ 25 extern unsigned long empty_zero_page[PAGE_SIZE / sizeof(unsigned long)] 26 __visible; 27 #define ZERO_PAGE(vaddr) (virt_to_page(empty_zero_page)) 28 29 extern spinlock_t pgd_lock; 30 extern struct list_head pgd_list; 31 32 extern struct mm_struct *pgd_page_get_mm(struct page *page); 33 34 #ifdef CONFIG_PARAVIRT 35 #include <asm/paravirt.h> 36 #else /* !CONFIG_PARAVIRT */ 37 #define set_pte(ptep, pte) native_set_pte(ptep, pte) 38 #define set_pte_at(mm, addr, ptep, pte) native_set_pte_at(mm, addr, ptep, pte) 39 #define set_pmd_at(mm, addr, pmdp, pmd) native_set_pmd_at(mm, addr, pmdp, pmd) 40 41 #define set_pte_atomic(ptep, pte) \ 42 native_set_pte_atomic(ptep, pte) 43 44 #define set_pmd(pmdp, pmd) native_set_pmd(pmdp, pmd) 45 46 #ifndef __PAGETABLE_PUD_FOLDED 47 #define set_pgd(pgdp, pgd) native_set_pgd(pgdp, pgd) 48 #define pgd_clear(pgd) native_pgd_clear(pgd) 49 #endif 50 51 #ifndef set_pud 52 # define set_pud(pudp, pud) native_set_pud(pudp, pud) 53 #endif 54 55 #ifndef __PAGETABLE_PMD_FOLDED 56 #define pud_clear(pud) native_pud_clear(pud) 57 #endif 58 59 #define pte_clear(mm, addr, ptep) native_pte_clear(mm, addr, ptep) 60 #define pmd_clear(pmd) native_pmd_clear(pmd) 61 62 #define pte_update(mm, addr, ptep) do { } while (0) 63 #define pte_update_defer(mm, addr, ptep) do { } while (0) 64 #define pmd_update(mm, addr, ptep) do { } while (0) 65 #define pmd_update_defer(mm, addr, ptep) do { } while (0) 66 67 #define pgd_val(x) native_pgd_val(x) 68 #define __pgd(x) native_make_pgd(x) 69 70 #ifndef __PAGETABLE_PUD_FOLDED 71 #define pud_val(x) native_pud_val(x) 72 #define __pud(x) native_make_pud(x) 73 #endif 74 75 #ifndef __PAGETABLE_PMD_FOLDED 76 #define pmd_val(x) native_pmd_val(x) 77 #define __pmd(x) native_make_pmd(x) 78 #endif 79 80 #define pte_val(x) native_pte_val(x) 81 #define __pte(x) native_make_pte(x) 82 83 #define arch_end_context_switch(prev) do {} while(0) 84 85 #endif /* CONFIG_PARAVIRT */ 86 87 /* 88 * The following only work if pte_present() is true. 89 * Undefined behaviour if not.. 90 */ 91 static inline int pte_dirty(pte_t pte) 92 { 93 return pte_flags(pte) & _PAGE_DIRTY; 94 } 95 96 static inline int pte_young(pte_t pte) 97 { 98 return pte_flags(pte) & _PAGE_ACCESSED; 99 } 100 101 static inline int pmd_young(pmd_t pmd) 102 { 103 return pmd_flags(pmd) & _PAGE_ACCESSED; 104 } 105 106 static inline int pte_write(pte_t pte) 107 { 108 return pte_flags(pte) & _PAGE_RW; 109 } 110 111 static inline int pte_file(pte_t pte) 112 { 113 return pte_flags(pte) & _PAGE_FILE; 114 } 115 116 static inline int pte_huge(pte_t pte) 117 { 118 return pte_flags(pte) & _PAGE_PSE; 119 } 120 121 static inline int pte_global(pte_t pte) 122 { 123 return pte_flags(pte) & _PAGE_GLOBAL; 124 } 125 126 static inline int pte_exec(pte_t pte) 127 { 128 return !(pte_flags(pte) & _PAGE_NX); 129 } 130 131 static inline int pte_special(pte_t pte) 132 { 133 return pte_flags(pte) & _PAGE_SPECIAL; 134 } 135 136 static inline unsigned long pte_pfn(pte_t pte) 137 { 138 return (pte_val(pte) & PTE_PFN_MASK) >> PAGE_SHIFT; 139 } 140 141 static inline unsigned long pmd_pfn(pmd_t pmd) 142 { 143 return (pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT; 144 } 145 146 static inline unsigned long pud_pfn(pud_t pud) 147 { 148 return (pud_val(pud) & PTE_PFN_MASK) >> PAGE_SHIFT; 149 } 150 151 #define pte_page(pte) pfn_to_page(pte_pfn(pte)) 152 153 static inline int pmd_large(pmd_t pte) 154 { 155 return pmd_flags(pte) & _PAGE_PSE; 156 } 157 158 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 159 static inline int pmd_trans_splitting(pmd_t pmd) 160 { 161 return pmd_val(pmd) & _PAGE_SPLITTING; 162 } 163 164 static inline int pmd_trans_huge(pmd_t pmd) 165 { 166 return pmd_val(pmd) & _PAGE_PSE; 167 } 168 169 static inline int has_transparent_hugepage(void) 170 { 171 return cpu_has_pse; 172 } 173 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 174 175 static inline pte_t pte_set_flags(pte_t pte, pteval_t set) 176 { 177 pteval_t v = native_pte_val(pte); 178 179 return native_make_pte(v | set); 180 } 181 182 static inline pte_t pte_clear_flags(pte_t pte, pteval_t clear) 183 { 184 pteval_t v = native_pte_val(pte); 185 186 return native_make_pte(v & ~clear); 187 } 188 189 static inline pte_t pte_mkclean(pte_t pte) 190 { 191 return pte_clear_flags(pte, _PAGE_DIRTY); 192 } 193 194 static inline pte_t pte_mkold(pte_t pte) 195 { 196 return pte_clear_flags(pte, _PAGE_ACCESSED); 197 } 198 199 static inline pte_t pte_wrprotect(pte_t pte) 200 { 201 return pte_clear_flags(pte, _PAGE_RW); 202 } 203 204 static inline pte_t pte_mkexec(pte_t pte) 205 { 206 return pte_clear_flags(pte, _PAGE_NX); 207 } 208 209 static inline pte_t pte_mkdirty(pte_t pte) 210 { 211 return pte_set_flags(pte, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); 212 } 213 214 static inline pte_t pte_mkyoung(pte_t pte) 215 { 216 return pte_set_flags(pte, _PAGE_ACCESSED); 217 } 218 219 static inline pte_t pte_mkwrite(pte_t pte) 220 { 221 return pte_set_flags(pte, _PAGE_RW); 222 } 223 224 static inline pte_t pte_mkhuge(pte_t pte) 225 { 226 return pte_set_flags(pte, _PAGE_PSE); 227 } 228 229 static inline pte_t pte_clrhuge(pte_t pte) 230 { 231 return pte_clear_flags(pte, _PAGE_PSE); 232 } 233 234 static inline pte_t pte_mkglobal(pte_t pte) 235 { 236 return pte_set_flags(pte, _PAGE_GLOBAL); 237 } 238 239 static inline pte_t pte_clrglobal(pte_t pte) 240 { 241 return pte_clear_flags(pte, _PAGE_GLOBAL); 242 } 243 244 static inline pte_t pte_mkspecial(pte_t pte) 245 { 246 return pte_set_flags(pte, _PAGE_SPECIAL); 247 } 248 249 static inline pmd_t pmd_set_flags(pmd_t pmd, pmdval_t set) 250 { 251 pmdval_t v = native_pmd_val(pmd); 252 253 return __pmd(v | set); 254 } 255 256 static inline pmd_t pmd_clear_flags(pmd_t pmd, pmdval_t clear) 257 { 258 pmdval_t v = native_pmd_val(pmd); 259 260 return __pmd(v & ~clear); 261 } 262 263 static inline pmd_t pmd_mkold(pmd_t pmd) 264 { 265 return pmd_clear_flags(pmd, _PAGE_ACCESSED); 266 } 267 268 static inline pmd_t pmd_wrprotect(pmd_t pmd) 269 { 270 return pmd_clear_flags(pmd, _PAGE_RW); 271 } 272 273 static inline pmd_t pmd_mkdirty(pmd_t pmd) 274 { 275 return pmd_set_flags(pmd, _PAGE_DIRTY | _PAGE_SOFT_DIRTY); 276 } 277 278 static inline pmd_t pmd_mkhuge(pmd_t pmd) 279 { 280 return pmd_set_flags(pmd, _PAGE_PSE); 281 } 282 283 static inline pmd_t pmd_mkyoung(pmd_t pmd) 284 { 285 return pmd_set_flags(pmd, _PAGE_ACCESSED); 286 } 287 288 static inline pmd_t pmd_mkwrite(pmd_t pmd) 289 { 290 return pmd_set_flags(pmd, _PAGE_RW); 291 } 292 293 static inline pmd_t pmd_mknotpresent(pmd_t pmd) 294 { 295 return pmd_clear_flags(pmd, _PAGE_PRESENT); 296 } 297 298 static inline int pte_soft_dirty(pte_t pte) 299 { 300 return pte_flags(pte) & _PAGE_SOFT_DIRTY; 301 } 302 303 static inline int pmd_soft_dirty(pmd_t pmd) 304 { 305 return pmd_flags(pmd) & _PAGE_SOFT_DIRTY; 306 } 307 308 static inline pte_t pte_mksoft_dirty(pte_t pte) 309 { 310 return pte_set_flags(pte, _PAGE_SOFT_DIRTY); 311 } 312 313 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) 314 { 315 return pmd_set_flags(pmd, _PAGE_SOFT_DIRTY); 316 } 317 318 static inline pte_t pte_file_clear_soft_dirty(pte_t pte) 319 { 320 return pte_clear_flags(pte, _PAGE_SOFT_DIRTY); 321 } 322 323 static inline pte_t pte_file_mksoft_dirty(pte_t pte) 324 { 325 return pte_set_flags(pte, _PAGE_SOFT_DIRTY); 326 } 327 328 static inline int pte_file_soft_dirty(pte_t pte) 329 { 330 return pte_flags(pte) & _PAGE_SOFT_DIRTY; 331 } 332 333 /* 334 * Mask out unsupported bits in a present pgprot. Non-present pgprots 335 * can use those bits for other purposes, so leave them be. 336 */ 337 static inline pgprotval_t massage_pgprot(pgprot_t pgprot) 338 { 339 pgprotval_t protval = pgprot_val(pgprot); 340 341 if (protval & _PAGE_PRESENT) 342 protval &= __supported_pte_mask; 343 344 return protval; 345 } 346 347 static inline pte_t pfn_pte(unsigned long page_nr, pgprot_t pgprot) 348 { 349 return __pte(((phys_addr_t)page_nr << PAGE_SHIFT) | 350 massage_pgprot(pgprot)); 351 } 352 353 static inline pmd_t pfn_pmd(unsigned long page_nr, pgprot_t pgprot) 354 { 355 return __pmd(((phys_addr_t)page_nr << PAGE_SHIFT) | 356 massage_pgprot(pgprot)); 357 } 358 359 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) 360 { 361 pteval_t val = pte_val(pte); 362 363 /* 364 * Chop off the NX bit (if present), and add the NX portion of 365 * the newprot (if present): 366 */ 367 val &= _PAGE_CHG_MASK; 368 val |= massage_pgprot(newprot) & ~_PAGE_CHG_MASK; 369 370 return __pte(val); 371 } 372 373 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) 374 { 375 pmdval_t val = pmd_val(pmd); 376 377 val &= _HPAGE_CHG_MASK; 378 val |= massage_pgprot(newprot) & ~_HPAGE_CHG_MASK; 379 380 return __pmd(val); 381 } 382 383 /* mprotect needs to preserve PAT bits when updating vm_page_prot */ 384 #define pgprot_modify pgprot_modify 385 static inline pgprot_t pgprot_modify(pgprot_t oldprot, pgprot_t newprot) 386 { 387 pgprotval_t preservebits = pgprot_val(oldprot) & _PAGE_CHG_MASK; 388 pgprotval_t addbits = pgprot_val(newprot); 389 return __pgprot(preservebits | addbits); 390 } 391 392 #define pte_pgprot(x) __pgprot(pte_flags(x) & PTE_FLAGS_MASK) 393 394 #define canon_pgprot(p) __pgprot(massage_pgprot(p)) 395 396 static inline int is_new_memtype_allowed(u64 paddr, unsigned long size, 397 unsigned long flags, 398 unsigned long new_flags) 399 { 400 /* 401 * PAT type is always WB for untracked ranges, so no need to check. 402 */ 403 if (x86_platform.is_untracked_pat_range(paddr, paddr + size)) 404 return 1; 405 406 /* 407 * Certain new memtypes are not allowed with certain 408 * requested memtype: 409 * - request is uncached, return cannot be write-back 410 * - request is write-combine, return cannot be write-back 411 */ 412 if ((flags == _PAGE_CACHE_UC_MINUS && 413 new_flags == _PAGE_CACHE_WB) || 414 (flags == _PAGE_CACHE_WC && 415 new_flags == _PAGE_CACHE_WB)) { 416 return 0; 417 } 418 419 return 1; 420 } 421 422 pmd_t *populate_extra_pmd(unsigned long vaddr); 423 pte_t *populate_extra_pte(unsigned long vaddr); 424 #endif /* __ASSEMBLY__ */ 425 426 #ifdef CONFIG_X86_32 427 # include <asm/pgtable_32.h> 428 #else 429 # include <asm/pgtable_64.h> 430 #endif 431 432 #ifndef __ASSEMBLY__ 433 #include <linux/mm_types.h> 434 #include <linux/mmdebug.h> 435 #include <linux/log2.h> 436 437 static inline int pte_none(pte_t pte) 438 { 439 return !pte.pte; 440 } 441 442 #define __HAVE_ARCH_PTE_SAME 443 static inline int pte_same(pte_t a, pte_t b) 444 { 445 return a.pte == b.pte; 446 } 447 448 static inline int pte_present(pte_t a) 449 { 450 return pte_flags(a) & (_PAGE_PRESENT | _PAGE_PROTNONE | 451 _PAGE_NUMA); 452 } 453 454 #define pte_accessible pte_accessible 455 static inline bool pte_accessible(struct mm_struct *mm, pte_t a) 456 { 457 if (pte_flags(a) & _PAGE_PRESENT) 458 return true; 459 460 if ((pte_flags(a) & (_PAGE_PROTNONE | _PAGE_NUMA)) && 461 mm_tlb_flush_pending(mm)) 462 return true; 463 464 return false; 465 } 466 467 static inline int pte_hidden(pte_t pte) 468 { 469 return pte_flags(pte) & _PAGE_HIDDEN; 470 } 471 472 static inline int pmd_present(pmd_t pmd) 473 { 474 /* 475 * Checking for _PAGE_PSE is needed too because 476 * split_huge_page will temporarily clear the present bit (but 477 * the _PAGE_PSE flag will remain set at all times while the 478 * _PAGE_PRESENT bit is clear). 479 */ 480 return pmd_flags(pmd) & (_PAGE_PRESENT | _PAGE_PROTNONE | _PAGE_PSE | 481 _PAGE_NUMA); 482 } 483 484 static inline int pmd_none(pmd_t pmd) 485 { 486 /* Only check low word on 32-bit platforms, since it might be 487 out of sync with upper half. */ 488 return (unsigned long)native_pmd_val(pmd) == 0; 489 } 490 491 static inline unsigned long pmd_page_vaddr(pmd_t pmd) 492 { 493 return (unsigned long)__va(pmd_val(pmd) & PTE_PFN_MASK); 494 } 495 496 /* 497 * Currently stuck as a macro due to indirect forward reference to 498 * linux/mmzone.h's __section_mem_map_addr() definition: 499 */ 500 #define pmd_page(pmd) pfn_to_page((pmd_val(pmd) & PTE_PFN_MASK) >> PAGE_SHIFT) 501 502 /* 503 * the pmd page can be thought of an array like this: pmd_t[PTRS_PER_PMD] 504 * 505 * this macro returns the index of the entry in the pmd page which would 506 * control the given virtual address 507 */ 508 static inline unsigned long pmd_index(unsigned long address) 509 { 510 return (address >> PMD_SHIFT) & (PTRS_PER_PMD - 1); 511 } 512 513 /* 514 * Conversion functions: convert a page and protection to a page entry, 515 * and a page entry and page directory to the page they refer to. 516 * 517 * (Currently stuck as a macro because of indirect forward reference 518 * to linux/mm.h:page_to_nid()) 519 */ 520 #define mk_pte(page, pgprot) pfn_pte(page_to_pfn(page), (pgprot)) 521 522 /* 523 * the pte page can be thought of an array like this: pte_t[PTRS_PER_PTE] 524 * 525 * this function returns the index of the entry in the pte page which would 526 * control the given virtual address 527 */ 528 static inline unsigned long pte_index(unsigned long address) 529 { 530 return (address >> PAGE_SHIFT) & (PTRS_PER_PTE - 1); 531 } 532 533 static inline pte_t *pte_offset_kernel(pmd_t *pmd, unsigned long address) 534 { 535 return (pte_t *)pmd_page_vaddr(*pmd) + pte_index(address); 536 } 537 538 static inline int pmd_bad(pmd_t pmd) 539 { 540 #ifdef CONFIG_NUMA_BALANCING 541 /* pmd_numa check */ 542 if ((pmd_flags(pmd) & (_PAGE_NUMA|_PAGE_PRESENT)) == _PAGE_NUMA) 543 return 0; 544 #endif 545 return (pmd_flags(pmd) & ~_PAGE_USER) != _KERNPG_TABLE; 546 } 547 548 static inline unsigned long pages_to_mb(unsigned long npg) 549 { 550 return npg >> (20 - PAGE_SHIFT); 551 } 552 553 #if PAGETABLE_LEVELS > 2 554 static inline int pud_none(pud_t pud) 555 { 556 return native_pud_val(pud) == 0; 557 } 558 559 static inline int pud_present(pud_t pud) 560 { 561 return pud_flags(pud) & _PAGE_PRESENT; 562 } 563 564 static inline unsigned long pud_page_vaddr(pud_t pud) 565 { 566 return (unsigned long)__va((unsigned long)pud_val(pud) & PTE_PFN_MASK); 567 } 568 569 /* 570 * Currently stuck as a macro due to indirect forward reference to 571 * linux/mmzone.h's __section_mem_map_addr() definition: 572 */ 573 #define pud_page(pud) pfn_to_page(pud_val(pud) >> PAGE_SHIFT) 574 575 /* Find an entry in the second-level page table.. */ 576 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) 577 { 578 return (pmd_t *)pud_page_vaddr(*pud) + pmd_index(address); 579 } 580 581 static inline int pud_large(pud_t pud) 582 { 583 return (pud_val(pud) & (_PAGE_PSE | _PAGE_PRESENT)) == 584 (_PAGE_PSE | _PAGE_PRESENT); 585 } 586 587 static inline int pud_bad(pud_t pud) 588 { 589 return (pud_flags(pud) & ~(_KERNPG_TABLE | _PAGE_USER)) != 0; 590 } 591 #else 592 static inline int pud_large(pud_t pud) 593 { 594 return 0; 595 } 596 #endif /* PAGETABLE_LEVELS > 2 */ 597 598 #if PAGETABLE_LEVELS > 3 599 static inline int pgd_present(pgd_t pgd) 600 { 601 return pgd_flags(pgd) & _PAGE_PRESENT; 602 } 603 604 static inline unsigned long pgd_page_vaddr(pgd_t pgd) 605 { 606 return (unsigned long)__va((unsigned long)pgd_val(pgd) & PTE_PFN_MASK); 607 } 608 609 /* 610 * Currently stuck as a macro due to indirect forward reference to 611 * linux/mmzone.h's __section_mem_map_addr() definition: 612 */ 613 #define pgd_page(pgd) pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT) 614 615 /* to find an entry in a page-table-directory. */ 616 static inline unsigned long pud_index(unsigned long address) 617 { 618 return (address >> PUD_SHIFT) & (PTRS_PER_PUD - 1); 619 } 620 621 static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address) 622 { 623 return (pud_t *)pgd_page_vaddr(*pgd) + pud_index(address); 624 } 625 626 static inline int pgd_bad(pgd_t pgd) 627 { 628 return (pgd_flags(pgd) & ~_PAGE_USER) != _KERNPG_TABLE; 629 } 630 631 static inline int pgd_none(pgd_t pgd) 632 { 633 return !native_pgd_val(pgd); 634 } 635 #endif /* PAGETABLE_LEVELS > 3 */ 636 637 #endif /* __ASSEMBLY__ */ 638 639 /* 640 * the pgd page can be thought of an array like this: pgd_t[PTRS_PER_PGD] 641 * 642 * this macro returns the index of the entry in the pgd page which would 643 * control the given virtual address 644 */ 645 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD - 1)) 646 647 /* 648 * pgd_offset() returns a (pgd_t *) 649 * pgd_index() is used get the offset into the pgd page's array of pgd_t's; 650 */ 651 #define pgd_offset(mm, address) ((mm)->pgd + pgd_index((address))) 652 /* 653 * a shortcut which implies the use of the kernel's pgd, instead 654 * of a process's 655 */ 656 #define pgd_offset_k(address) pgd_offset(&init_mm, (address)) 657 658 659 #define KERNEL_PGD_BOUNDARY pgd_index(PAGE_OFFSET) 660 #define KERNEL_PGD_PTRS (PTRS_PER_PGD - KERNEL_PGD_BOUNDARY) 661 662 #ifndef __ASSEMBLY__ 663 664 extern int direct_gbpages; 665 void init_mem_mapping(void); 666 void early_alloc_pgt_buf(void); 667 668 /* local pte updates need not use xchg for locking */ 669 static inline pte_t native_local_ptep_get_and_clear(pte_t *ptep) 670 { 671 pte_t res = *ptep; 672 673 /* Pure native function needs no input for mm, addr */ 674 native_pte_clear(NULL, 0, ptep); 675 return res; 676 } 677 678 static inline pmd_t native_local_pmdp_get_and_clear(pmd_t *pmdp) 679 { 680 pmd_t res = *pmdp; 681 682 native_pmd_clear(pmdp); 683 return res; 684 } 685 686 static inline void native_set_pte_at(struct mm_struct *mm, unsigned long addr, 687 pte_t *ptep , pte_t pte) 688 { 689 native_set_pte(ptep, pte); 690 } 691 692 static inline void native_set_pmd_at(struct mm_struct *mm, unsigned long addr, 693 pmd_t *pmdp , pmd_t pmd) 694 { 695 native_set_pmd(pmdp, pmd); 696 } 697 698 #ifndef CONFIG_PARAVIRT 699 /* 700 * Rules for using pte_update - it must be called after any PTE update which 701 * has not been done using the set_pte / clear_pte interfaces. It is used by 702 * shadow mode hypervisors to resynchronize the shadow page tables. Kernel PTE 703 * updates should either be sets, clears, or set_pte_atomic for P->P 704 * transitions, which means this hook should only be called for user PTEs. 705 * This hook implies a P->P protection or access change has taken place, which 706 * requires a subsequent TLB flush. The notification can optionally be delayed 707 * until the TLB flush event by using the pte_update_defer form of the 708 * interface, but care must be taken to assure that the flush happens while 709 * still holding the same page table lock so that the shadow and primary pages 710 * do not become out of sync on SMP. 711 */ 712 #define pte_update(mm, addr, ptep) do { } while (0) 713 #define pte_update_defer(mm, addr, ptep) do { } while (0) 714 #endif 715 716 /* 717 * We only update the dirty/accessed state if we set 718 * the dirty bit by hand in the kernel, since the hardware 719 * will do the accessed bit for us, and we don't want to 720 * race with other CPU's that might be updating the dirty 721 * bit at the same time. 722 */ 723 struct vm_area_struct; 724 725 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 726 extern int ptep_set_access_flags(struct vm_area_struct *vma, 727 unsigned long address, pte_t *ptep, 728 pte_t entry, int dirty); 729 730 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG 731 extern int ptep_test_and_clear_young(struct vm_area_struct *vma, 732 unsigned long addr, pte_t *ptep); 733 734 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 735 extern int ptep_clear_flush_young(struct vm_area_struct *vma, 736 unsigned long address, pte_t *ptep); 737 738 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR 739 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, 740 pte_t *ptep) 741 { 742 pte_t pte = native_ptep_get_and_clear(ptep); 743 pte_update(mm, addr, ptep); 744 return pte; 745 } 746 747 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL 748 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, 749 unsigned long addr, pte_t *ptep, 750 int full) 751 { 752 pte_t pte; 753 if (full) { 754 /* 755 * Full address destruction in progress; paravirt does not 756 * care about updates and native needs no locking 757 */ 758 pte = native_local_ptep_get_and_clear(ptep); 759 } else { 760 pte = ptep_get_and_clear(mm, addr, ptep); 761 } 762 return pte; 763 } 764 765 #define __HAVE_ARCH_PTEP_SET_WRPROTECT 766 static inline void ptep_set_wrprotect(struct mm_struct *mm, 767 unsigned long addr, pte_t *ptep) 768 { 769 clear_bit(_PAGE_BIT_RW, (unsigned long *)&ptep->pte); 770 pte_update(mm, addr, ptep); 771 } 772 773 #define flush_tlb_fix_spurious_fault(vma, address) do { } while (0) 774 775 #define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) 776 777 #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 778 extern int pmdp_set_access_flags(struct vm_area_struct *vma, 779 unsigned long address, pmd_t *pmdp, 780 pmd_t entry, int dirty); 781 782 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG 783 extern int pmdp_test_and_clear_young(struct vm_area_struct *vma, 784 unsigned long addr, pmd_t *pmdp); 785 786 #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 787 extern int pmdp_clear_flush_young(struct vm_area_struct *vma, 788 unsigned long address, pmd_t *pmdp); 789 790 791 #define __HAVE_ARCH_PMDP_SPLITTING_FLUSH 792 extern void pmdp_splitting_flush(struct vm_area_struct *vma, 793 unsigned long addr, pmd_t *pmdp); 794 795 #define __HAVE_ARCH_PMD_WRITE 796 static inline int pmd_write(pmd_t pmd) 797 { 798 return pmd_flags(pmd) & _PAGE_RW; 799 } 800 801 #define __HAVE_ARCH_PMDP_GET_AND_CLEAR 802 static inline pmd_t pmdp_get_and_clear(struct mm_struct *mm, unsigned long addr, 803 pmd_t *pmdp) 804 { 805 pmd_t pmd = native_pmdp_get_and_clear(pmdp); 806 pmd_update(mm, addr, pmdp); 807 return pmd; 808 } 809 810 #define __HAVE_ARCH_PMDP_SET_WRPROTECT 811 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 812 unsigned long addr, pmd_t *pmdp) 813 { 814 clear_bit(_PAGE_BIT_RW, (unsigned long *)pmdp); 815 pmd_update(mm, addr, pmdp); 816 } 817 818 /* 819 * clone_pgd_range(pgd_t *dst, pgd_t *src, int count); 820 * 821 * dst - pointer to pgd range anwhere on a pgd page 822 * src - "" 823 * count - the number of pgds to copy. 824 * 825 * dst and src can be on the same page, but the range must not overlap, 826 * and must not cross a page boundary. 827 */ 828 static inline void clone_pgd_range(pgd_t *dst, pgd_t *src, int count) 829 { 830 memcpy(dst, src, count * sizeof(pgd_t)); 831 } 832 833 #define PTE_SHIFT ilog2(PTRS_PER_PTE) 834 static inline int page_level_shift(enum pg_level level) 835 { 836 return (PAGE_SHIFT - PTE_SHIFT) + level * PTE_SHIFT; 837 } 838 static inline unsigned long page_level_size(enum pg_level level) 839 { 840 return 1UL << page_level_shift(level); 841 } 842 static inline unsigned long page_level_mask(enum pg_level level) 843 { 844 return ~(page_level_size(level) - 1); 845 } 846 847 /* 848 * The x86 doesn't have any external MMU info: the kernel page 849 * tables contain all the necessary information. 850 */ 851 static inline void update_mmu_cache(struct vm_area_struct *vma, 852 unsigned long addr, pte_t *ptep) 853 { 854 } 855 static inline void update_mmu_cache_pmd(struct vm_area_struct *vma, 856 unsigned long addr, pmd_t *pmd) 857 { 858 } 859 860 static inline pte_t pte_swp_mksoft_dirty(pte_t pte) 861 { 862 VM_BUG_ON(pte_present(pte)); 863 return pte_set_flags(pte, _PAGE_SWP_SOFT_DIRTY); 864 } 865 866 static inline int pte_swp_soft_dirty(pte_t pte) 867 { 868 VM_BUG_ON(pte_present(pte)); 869 return pte_flags(pte) & _PAGE_SWP_SOFT_DIRTY; 870 } 871 872 static inline pte_t pte_swp_clear_soft_dirty(pte_t pte) 873 { 874 VM_BUG_ON(pte_present(pte)); 875 return pte_clear_flags(pte, _PAGE_SWP_SOFT_DIRTY); 876 } 877 878 #include <asm-generic/pgtable.h> 879 #endif /* __ASSEMBLY__ */ 880 881 #endif /* _ASM_X86_PGTABLE_H */ 882