1 /* 2 * S390 version 3 * Copyright IBM Corp. 1999, 2000 4 * Author(s): Hartmut Penner (hp@de.ibm.com) 5 * Ulrich Weigand (weigand@de.ibm.com) 6 * Martin Schwidefsky (schwidefsky@de.ibm.com) 7 * 8 * Derived from "include/asm-i386/pgtable.h" 9 */ 10 11 #ifndef _ASM_S390_PGTABLE_H 12 #define _ASM_S390_PGTABLE_H 13 14 /* 15 * The Linux memory management assumes a three-level page table setup. 16 * For s390 64 bit we use up to four of the five levels the hardware 17 * provides (region first tables are not used). 18 * 19 * The "pgd_xxx()" functions are trivial for a folded two-level 20 * setup: the pgd is never bad, and a pmd always exists (as it's folded 21 * into the pgd entry) 22 * 23 * This file contains the functions and defines necessary to modify and use 24 * the S390 page table tree. 25 */ 26 #ifndef __ASSEMBLY__ 27 #include <linux/sched.h> 28 #include <linux/mm_types.h> 29 #include <linux/page-flags.h> 30 #include <linux/radix-tree.h> 31 #include <linux/atomic.h> 32 #include <asm/bug.h> 33 #include <asm/page.h> 34 35 extern pgd_t swapper_pg_dir[]; 36 extern void paging_init(void); 37 extern void vmem_map_init(void); 38 pmd_t *vmem_pmd_alloc(void); 39 pte_t *vmem_pte_alloc(void); 40 41 enum { 42 PG_DIRECT_MAP_4K = 0, 43 PG_DIRECT_MAP_1M, 44 PG_DIRECT_MAP_2G, 45 PG_DIRECT_MAP_MAX 46 }; 47 48 extern atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX]; 49 50 static inline void update_page_count(int level, long count) 51 { 52 if (IS_ENABLED(CONFIG_PROC_FS)) 53 atomic_long_add(count, &direct_pages_count[level]); 54 } 55 56 struct seq_file; 57 void arch_report_meminfo(struct seq_file *m); 58 59 /* 60 * The S390 doesn't have any external MMU info: the kernel page 61 * tables contain all the necessary information. 62 */ 63 #define update_mmu_cache(vma, address, ptep) do { } while (0) 64 #define update_mmu_cache_pmd(vma, address, ptep) do { } while (0) 65 66 /* 67 * ZERO_PAGE is a global shared page that is always zero; used 68 * for zero-mapped memory areas etc.. 69 */ 70 71 extern unsigned long empty_zero_page; 72 extern unsigned long zero_page_mask; 73 74 #define ZERO_PAGE(vaddr) \ 75 (virt_to_page((void *)(empty_zero_page + \ 76 (((unsigned long)(vaddr)) &zero_page_mask)))) 77 #define __HAVE_COLOR_ZERO_PAGE 78 79 /* TODO: s390 cannot support io_remap_pfn_range... */ 80 #endif /* !__ASSEMBLY__ */ 81 82 /* 83 * PMD_SHIFT determines the size of the area a second-level page 84 * table can map 85 * PGDIR_SHIFT determines what a third-level page table entry can map 86 */ 87 #define PMD_SHIFT 20 88 #define PUD_SHIFT 31 89 #define PGDIR_SHIFT 42 90 91 #define PMD_SIZE (1UL << PMD_SHIFT) 92 #define PMD_MASK (~(PMD_SIZE-1)) 93 #define PUD_SIZE (1UL << PUD_SHIFT) 94 #define PUD_MASK (~(PUD_SIZE-1)) 95 #define PGDIR_SIZE (1UL << PGDIR_SHIFT) 96 #define PGDIR_MASK (~(PGDIR_SIZE-1)) 97 98 /* 99 * entries per page directory level: the S390 is two-level, so 100 * we don't really have any PMD directory physically. 101 * for S390 segment-table entries are combined to one PGD 102 * that leads to 1024 pte per pgd 103 */ 104 #define PTRS_PER_PTE 256 105 #define PTRS_PER_PMD 2048 106 #define PTRS_PER_PUD 2048 107 #define PTRS_PER_PGD 2048 108 109 #define FIRST_USER_ADDRESS 0UL 110 111 #define pte_ERROR(e) \ 112 printk("%s:%d: bad pte %p.\n", __FILE__, __LINE__, (void *) pte_val(e)) 113 #define pmd_ERROR(e) \ 114 printk("%s:%d: bad pmd %p.\n", __FILE__, __LINE__, (void *) pmd_val(e)) 115 #define pud_ERROR(e) \ 116 printk("%s:%d: bad pud %p.\n", __FILE__, __LINE__, (void *) pud_val(e)) 117 #define pgd_ERROR(e) \ 118 printk("%s:%d: bad pgd %p.\n", __FILE__, __LINE__, (void *) pgd_val(e)) 119 120 #ifndef __ASSEMBLY__ 121 /* 122 * The vmalloc and module area will always be on the topmost area of the 123 * kernel mapping. We reserve 128GB (64bit) for vmalloc and modules. 124 * On 64 bit kernels we have a 2GB area at the top of the vmalloc area where 125 * modules will reside. That makes sure that inter module branches always 126 * happen without trampolines and in addition the placement within a 2GB frame 127 * is branch prediction unit friendly. 128 */ 129 extern unsigned long VMALLOC_START; 130 extern unsigned long VMALLOC_END; 131 extern struct page *vmemmap; 132 133 #define VMEM_MAX_PHYS ((unsigned long) vmemmap) 134 135 extern unsigned long MODULES_VADDR; 136 extern unsigned long MODULES_END; 137 #define MODULES_VADDR MODULES_VADDR 138 #define MODULES_END MODULES_END 139 #define MODULES_LEN (1UL << 31) 140 141 static inline int is_module_addr(void *addr) 142 { 143 BUILD_BUG_ON(MODULES_LEN > (1UL << 31)); 144 if (addr < (void *)MODULES_VADDR) 145 return 0; 146 if (addr > (void *)MODULES_END) 147 return 0; 148 return 1; 149 } 150 151 /* 152 * A 64 bit pagetable entry of S390 has following format: 153 * | PFRA |0IPC| OS | 154 * 0000000000111111111122222222223333333333444444444455555555556666 155 * 0123456789012345678901234567890123456789012345678901234567890123 156 * 157 * I Page-Invalid Bit: Page is not available for address-translation 158 * P Page-Protection Bit: Store access not possible for page 159 * C Change-bit override: HW is not required to set change bit 160 * 161 * A 64 bit segmenttable entry of S390 has following format: 162 * | P-table origin | TT 163 * 0000000000111111111122222222223333333333444444444455555555556666 164 * 0123456789012345678901234567890123456789012345678901234567890123 165 * 166 * I Segment-Invalid Bit: Segment is not available for address-translation 167 * C Common-Segment Bit: Segment is not private (PoP 3-30) 168 * P Page-Protection Bit: Store access not possible for page 169 * TT Type 00 170 * 171 * A 64 bit region table entry of S390 has following format: 172 * | S-table origin | TF TTTL 173 * 0000000000111111111122222222223333333333444444444455555555556666 174 * 0123456789012345678901234567890123456789012345678901234567890123 175 * 176 * I Segment-Invalid Bit: Segment is not available for address-translation 177 * TT Type 01 178 * TF 179 * TL Table length 180 * 181 * The 64 bit regiontable origin of S390 has following format: 182 * | region table origon | DTTL 183 * 0000000000111111111122222222223333333333444444444455555555556666 184 * 0123456789012345678901234567890123456789012345678901234567890123 185 * 186 * X Space-Switch event: 187 * G Segment-Invalid Bit: 188 * P Private-Space Bit: 189 * S Storage-Alteration: 190 * R Real space 191 * TL Table-Length: 192 * 193 * A storage key has the following format: 194 * | ACC |F|R|C|0| 195 * 0 3 4 5 6 7 196 * ACC: access key 197 * F : fetch protection bit 198 * R : referenced bit 199 * C : changed bit 200 */ 201 202 /* Hardware bits in the page table entry */ 203 #define _PAGE_PROTECT 0x200 /* HW read-only bit */ 204 #define _PAGE_INVALID 0x400 /* HW invalid bit */ 205 #define _PAGE_LARGE 0x800 /* Bit to mark a large pte */ 206 207 /* Software bits in the page table entry */ 208 #define _PAGE_PRESENT 0x001 /* SW pte present bit */ 209 #define _PAGE_YOUNG 0x004 /* SW pte young bit */ 210 #define _PAGE_DIRTY 0x008 /* SW pte dirty bit */ 211 #define _PAGE_READ 0x010 /* SW pte read bit */ 212 #define _PAGE_WRITE 0x020 /* SW pte write bit */ 213 #define _PAGE_SPECIAL 0x040 /* SW associated with special page */ 214 #define _PAGE_UNUSED 0x080 /* SW bit for pgste usage state */ 215 #define __HAVE_ARCH_PTE_SPECIAL 216 217 #ifdef CONFIG_MEM_SOFT_DIRTY 218 #define _PAGE_SOFT_DIRTY 0x002 /* SW pte soft dirty bit */ 219 #else 220 #define _PAGE_SOFT_DIRTY 0x000 221 #endif 222 223 /* Set of bits not changed in pte_modify */ 224 #define _PAGE_CHG_MASK (PAGE_MASK | _PAGE_SPECIAL | _PAGE_DIRTY | \ 225 _PAGE_YOUNG | _PAGE_SOFT_DIRTY) 226 227 /* 228 * handle_pte_fault uses pte_present and pte_none to find out the pte type 229 * WITHOUT holding the page table lock. The _PAGE_PRESENT bit is used to 230 * distinguish present from not-present ptes. It is changed only with the page 231 * table lock held. 232 * 233 * The following table gives the different possible bit combinations for 234 * the pte hardware and software bits in the last 12 bits of a pte 235 * (. unassigned bit, x don't care, t swap type): 236 * 237 * 842100000000 238 * 000084210000 239 * 000000008421 240 * .IR.uswrdy.p 241 * empty .10.00000000 242 * swap .11..ttttt.0 243 * prot-none, clean, old .11.xx0000.1 244 * prot-none, clean, young .11.xx0001.1 245 * prot-none, dirty, old .10.xx0010.1 246 * prot-none, dirty, young .10.xx0011.1 247 * read-only, clean, old .11.xx0100.1 248 * read-only, clean, young .01.xx0101.1 249 * read-only, dirty, old .11.xx0110.1 250 * read-only, dirty, young .01.xx0111.1 251 * read-write, clean, old .11.xx1100.1 252 * read-write, clean, young .01.xx1101.1 253 * read-write, dirty, old .10.xx1110.1 254 * read-write, dirty, young .00.xx1111.1 255 * HW-bits: R read-only, I invalid 256 * SW-bits: p present, y young, d dirty, r read, w write, s special, 257 * u unused, l large 258 * 259 * pte_none is true for the bit pattern .10.00000000, pte == 0x400 260 * pte_swap is true for the bit pattern .11..ooooo.0, (pte & 0x201) == 0x200 261 * pte_present is true for the bit pattern .xx.xxxxxx.1, (pte & 0x001) == 0x001 262 */ 263 264 /* Bits in the segment/region table address-space-control-element */ 265 #define _ASCE_ORIGIN ~0xfffUL/* segment table origin */ 266 #define _ASCE_PRIVATE_SPACE 0x100 /* private space control */ 267 #define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */ 268 #define _ASCE_SPACE_SWITCH 0x40 /* space switch event */ 269 #define _ASCE_REAL_SPACE 0x20 /* real space control */ 270 #define _ASCE_TYPE_MASK 0x0c /* asce table type mask */ 271 #define _ASCE_TYPE_REGION1 0x0c /* region first table type */ 272 #define _ASCE_TYPE_REGION2 0x08 /* region second table type */ 273 #define _ASCE_TYPE_REGION3 0x04 /* region third table type */ 274 #define _ASCE_TYPE_SEGMENT 0x00 /* segment table type */ 275 #define _ASCE_TABLE_LENGTH 0x03 /* region table length */ 276 277 /* Bits in the region table entry */ 278 #define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */ 279 #define _REGION_ENTRY_PROTECT 0x200 /* region protection bit */ 280 #define _REGION_ENTRY_INVALID 0x20 /* invalid region table entry */ 281 #define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */ 282 #define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */ 283 #define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */ 284 #define _REGION_ENTRY_TYPE_R3 0x04 /* region third table type */ 285 #define _REGION_ENTRY_LENGTH 0x03 /* region third length */ 286 287 #define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH) 288 #define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID) 289 #define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH) 290 #define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID) 291 #define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH) 292 #define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID) 293 294 #define _REGION3_ENTRY_ORIGIN_LARGE ~0x7fffffffUL /* large page address */ 295 #define _REGION3_ENTRY_ORIGIN ~0x7ffUL/* region third table origin */ 296 297 #define _REGION3_ENTRY_DIRTY 0x2000 /* SW region dirty bit */ 298 #define _REGION3_ENTRY_YOUNG 0x1000 /* SW region young bit */ 299 #define _REGION3_ENTRY_LARGE 0x0400 /* RTTE-format control, large page */ 300 #define _REGION3_ENTRY_READ 0x0002 /* SW region read bit */ 301 #define _REGION3_ENTRY_WRITE 0x0001 /* SW region write bit */ 302 303 #ifdef CONFIG_MEM_SOFT_DIRTY 304 #define _REGION3_ENTRY_SOFT_DIRTY 0x4000 /* SW region soft dirty bit */ 305 #else 306 #define _REGION3_ENTRY_SOFT_DIRTY 0x0000 /* SW region soft dirty bit */ 307 #endif 308 309 #define _REGION_ENTRY_BITS 0xfffffffffffff227UL 310 #define _REGION_ENTRY_BITS_LARGE 0xffffffff8000fe27UL 311 312 /* Bits in the segment table entry */ 313 #define _SEGMENT_ENTRY_BITS 0xfffffffffffffe33UL 314 #define _SEGMENT_ENTRY_BITS_LARGE 0xfffffffffff0ff33UL 315 #define _SEGMENT_ENTRY_ORIGIN_LARGE ~0xfffffUL /* large page address */ 316 #define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* segment table origin */ 317 #define _SEGMENT_ENTRY_PROTECT 0x200 /* page protection bit */ 318 #define _SEGMENT_ENTRY_INVALID 0x20 /* invalid segment table entry */ 319 320 #define _SEGMENT_ENTRY (0) 321 #define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INVALID) 322 323 #define _SEGMENT_ENTRY_DIRTY 0x2000 /* SW segment dirty bit */ 324 #define _SEGMENT_ENTRY_YOUNG 0x1000 /* SW segment young bit */ 325 #define _SEGMENT_ENTRY_LARGE 0x0400 /* STE-format control, large page */ 326 #define _SEGMENT_ENTRY_READ 0x0002 /* SW segment read bit */ 327 #define _SEGMENT_ENTRY_WRITE 0x0001 /* SW segment write bit */ 328 329 #ifdef CONFIG_MEM_SOFT_DIRTY 330 #define _SEGMENT_ENTRY_SOFT_DIRTY 0x4000 /* SW segment soft dirty bit */ 331 #else 332 #define _SEGMENT_ENTRY_SOFT_DIRTY 0x0000 /* SW segment soft dirty bit */ 333 #endif 334 335 /* 336 * Segment table and region3 table entry encoding 337 * (R = read-only, I = invalid, y = young bit): 338 * dy..R...I...rw 339 * prot-none, clean, old 00..1...1...00 340 * prot-none, clean, young 01..1...1...00 341 * prot-none, dirty, old 10..1...1...00 342 * prot-none, dirty, young 11..1...1...00 343 * read-only, clean, old 00..1...1...10 344 * read-only, clean, young 01..1...0...10 345 * read-only, dirty, old 10..1...1...10 346 * read-only, dirty, young 11..1...0...10 347 * read-write, clean, old 00..1...1...11 348 * read-write, clean, young 01..1...0...11 349 * read-write, dirty, old 10..0...1...11 350 * read-write, dirty, young 11..0...0...11 351 * The segment table origin is used to distinguish empty (origin==0) from 352 * read-write, old segment table entries (origin!=0) 353 * HW-bits: R read-only, I invalid 354 * SW-bits: y young, d dirty, r read, w write 355 */ 356 357 /* Page status table bits for virtualization */ 358 #define PGSTE_ACC_BITS 0xf000000000000000UL 359 #define PGSTE_FP_BIT 0x0800000000000000UL 360 #define PGSTE_PCL_BIT 0x0080000000000000UL 361 #define PGSTE_HR_BIT 0x0040000000000000UL 362 #define PGSTE_HC_BIT 0x0020000000000000UL 363 #define PGSTE_GR_BIT 0x0004000000000000UL 364 #define PGSTE_GC_BIT 0x0002000000000000UL 365 #define PGSTE_UC_BIT 0x0000800000000000UL /* user dirty (migration) */ 366 #define PGSTE_IN_BIT 0x0000400000000000UL /* IPTE notify bit */ 367 368 /* Guest Page State used for virtualization */ 369 #define _PGSTE_GPS_ZERO 0x0000000080000000UL 370 #define _PGSTE_GPS_USAGE_MASK 0x0000000003000000UL 371 #define _PGSTE_GPS_USAGE_STABLE 0x0000000000000000UL 372 #define _PGSTE_GPS_USAGE_UNUSED 0x0000000001000000UL 373 374 /* 375 * A user page table pointer has the space-switch-event bit, the 376 * private-space-control bit and the storage-alteration-event-control 377 * bit set. A kernel page table pointer doesn't need them. 378 */ 379 #define _ASCE_USER_BITS (_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \ 380 _ASCE_ALT_EVENT) 381 382 /* 383 * Page protection definitions. 384 */ 385 #define PAGE_NONE __pgprot(_PAGE_PRESENT | _PAGE_INVALID) 386 #define PAGE_READ __pgprot(_PAGE_PRESENT | _PAGE_READ | \ 387 _PAGE_INVALID | _PAGE_PROTECT) 388 #define PAGE_WRITE __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ 389 _PAGE_INVALID | _PAGE_PROTECT) 390 391 #define PAGE_SHARED __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ 392 _PAGE_YOUNG | _PAGE_DIRTY) 393 #define PAGE_KERNEL __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE | \ 394 _PAGE_YOUNG | _PAGE_DIRTY) 395 #define PAGE_KERNEL_RO __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_YOUNG | \ 396 _PAGE_PROTECT) 397 398 /* 399 * On s390 the page table entry has an invalid bit and a read-only bit. 400 * Read permission implies execute permission and write permission 401 * implies read permission. 402 */ 403 /*xwr*/ 404 #define __P000 PAGE_NONE 405 #define __P001 PAGE_READ 406 #define __P010 PAGE_READ 407 #define __P011 PAGE_READ 408 #define __P100 PAGE_READ 409 #define __P101 PAGE_READ 410 #define __P110 PAGE_READ 411 #define __P111 PAGE_READ 412 413 #define __S000 PAGE_NONE 414 #define __S001 PAGE_READ 415 #define __S010 PAGE_WRITE 416 #define __S011 PAGE_WRITE 417 #define __S100 PAGE_READ 418 #define __S101 PAGE_READ 419 #define __S110 PAGE_WRITE 420 #define __S111 PAGE_WRITE 421 422 /* 423 * Segment entry (large page) protection definitions. 424 */ 425 #define SEGMENT_NONE __pgprot(_SEGMENT_ENTRY_INVALID | \ 426 _SEGMENT_ENTRY_PROTECT) 427 #define SEGMENT_READ __pgprot(_SEGMENT_ENTRY_PROTECT | \ 428 _SEGMENT_ENTRY_READ) 429 #define SEGMENT_WRITE __pgprot(_SEGMENT_ENTRY_READ | \ 430 _SEGMENT_ENTRY_WRITE) 431 #define SEGMENT_KERNEL __pgprot(_SEGMENT_ENTRY | \ 432 _SEGMENT_ENTRY_LARGE | \ 433 _SEGMENT_ENTRY_READ | \ 434 _SEGMENT_ENTRY_WRITE | \ 435 _SEGMENT_ENTRY_YOUNG | \ 436 _SEGMENT_ENTRY_DIRTY) 437 #define SEGMENT_KERNEL_RO __pgprot(_SEGMENT_ENTRY | \ 438 _SEGMENT_ENTRY_LARGE | \ 439 _SEGMENT_ENTRY_READ | \ 440 _SEGMENT_ENTRY_YOUNG | \ 441 _SEGMENT_ENTRY_PROTECT) 442 443 /* 444 * Region3 entry (large page) protection definitions. 445 */ 446 447 #define REGION3_KERNEL __pgprot(_REGION_ENTRY_TYPE_R3 | \ 448 _REGION3_ENTRY_LARGE | \ 449 _REGION3_ENTRY_READ | \ 450 _REGION3_ENTRY_WRITE | \ 451 _REGION3_ENTRY_YOUNG | \ 452 _REGION3_ENTRY_DIRTY) 453 #define REGION3_KERNEL_RO __pgprot(_REGION_ENTRY_TYPE_R3 | \ 454 _REGION3_ENTRY_LARGE | \ 455 _REGION3_ENTRY_READ | \ 456 _REGION3_ENTRY_YOUNG | \ 457 _REGION_ENTRY_PROTECT) 458 459 static inline int mm_has_pgste(struct mm_struct *mm) 460 { 461 #ifdef CONFIG_PGSTE 462 if (unlikely(mm->context.has_pgste)) 463 return 1; 464 #endif 465 return 0; 466 } 467 468 static inline int mm_alloc_pgste(struct mm_struct *mm) 469 { 470 #ifdef CONFIG_PGSTE 471 if (unlikely(mm->context.alloc_pgste)) 472 return 1; 473 #endif 474 return 0; 475 } 476 477 /* 478 * In the case that a guest uses storage keys 479 * faults should no longer be backed by zero pages 480 */ 481 #define mm_forbids_zeropage mm_use_skey 482 static inline int mm_use_skey(struct mm_struct *mm) 483 { 484 #ifdef CONFIG_PGSTE 485 if (mm->context.use_skey) 486 return 1; 487 #endif 488 return 0; 489 } 490 491 static inline void csp(unsigned int *ptr, unsigned int old, unsigned int new) 492 { 493 register unsigned long reg2 asm("2") = old; 494 register unsigned long reg3 asm("3") = new; 495 unsigned long address = (unsigned long)ptr | 1; 496 497 asm volatile( 498 " csp %0,%3" 499 : "+d" (reg2), "+m" (*ptr) 500 : "d" (reg3), "d" (address) 501 : "cc"); 502 } 503 504 static inline void cspg(unsigned long *ptr, unsigned long old, unsigned long new) 505 { 506 register unsigned long reg2 asm("2") = old; 507 register unsigned long reg3 asm("3") = new; 508 unsigned long address = (unsigned long)ptr | 1; 509 510 asm volatile( 511 " .insn rre,0xb98a0000,%0,%3" 512 : "+d" (reg2), "+m" (*ptr) 513 : "d" (reg3), "d" (address) 514 : "cc"); 515 } 516 517 #define CRDTE_DTT_PAGE 0x00UL 518 #define CRDTE_DTT_SEGMENT 0x10UL 519 #define CRDTE_DTT_REGION3 0x14UL 520 #define CRDTE_DTT_REGION2 0x18UL 521 #define CRDTE_DTT_REGION1 0x1cUL 522 523 static inline void crdte(unsigned long old, unsigned long new, 524 unsigned long table, unsigned long dtt, 525 unsigned long address, unsigned long asce) 526 { 527 register unsigned long reg2 asm("2") = old; 528 register unsigned long reg3 asm("3") = new; 529 register unsigned long reg4 asm("4") = table | dtt; 530 register unsigned long reg5 asm("5") = address; 531 532 asm volatile(".insn rrf,0xb98f0000,%0,%2,%4,0" 533 : "+d" (reg2) 534 : "d" (reg3), "d" (reg4), "d" (reg5), "a" (asce) 535 : "memory", "cc"); 536 } 537 538 /* 539 * pgd/pmd/pte query functions 540 */ 541 static inline int pgd_present(pgd_t pgd) 542 { 543 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2) 544 return 1; 545 return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL; 546 } 547 548 static inline int pgd_none(pgd_t pgd) 549 { 550 if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2) 551 return 0; 552 return (pgd_val(pgd) & _REGION_ENTRY_INVALID) != 0UL; 553 } 554 555 static inline int pgd_bad(pgd_t pgd) 556 { 557 /* 558 * With dynamic page table levels the pgd can be a region table 559 * entry or a segment table entry. Check for the bit that are 560 * invalid for either table entry. 561 */ 562 unsigned long mask = 563 ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INVALID & 564 ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH; 565 return (pgd_val(pgd) & mask) != 0; 566 } 567 568 static inline int pud_present(pud_t pud) 569 { 570 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) 571 return 1; 572 return (pud_val(pud) & _REGION_ENTRY_ORIGIN) != 0UL; 573 } 574 575 static inline int pud_none(pud_t pud) 576 { 577 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) 578 return 0; 579 return pud_val(pud) == _REGION3_ENTRY_EMPTY; 580 } 581 582 static inline int pud_large(pud_t pud) 583 { 584 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) != _REGION_ENTRY_TYPE_R3) 585 return 0; 586 return !!(pud_val(pud) & _REGION3_ENTRY_LARGE); 587 } 588 589 static inline unsigned long pud_pfn(pud_t pud) 590 { 591 unsigned long origin_mask; 592 593 origin_mask = _REGION3_ENTRY_ORIGIN; 594 if (pud_large(pud)) 595 origin_mask = _REGION3_ENTRY_ORIGIN_LARGE; 596 return (pud_val(pud) & origin_mask) >> PAGE_SHIFT; 597 } 598 599 static inline int pmd_large(pmd_t pmd) 600 { 601 return (pmd_val(pmd) & _SEGMENT_ENTRY_LARGE) != 0; 602 } 603 604 static inline int pmd_bad(pmd_t pmd) 605 { 606 if (pmd_large(pmd)) 607 return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS_LARGE) != 0; 608 return (pmd_val(pmd) & ~_SEGMENT_ENTRY_BITS) != 0; 609 } 610 611 static inline int pud_bad(pud_t pud) 612 { 613 if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3) 614 return pmd_bad(__pmd(pud_val(pud))); 615 if (pud_large(pud)) 616 return (pud_val(pud) & ~_REGION_ENTRY_BITS_LARGE) != 0; 617 return (pud_val(pud) & ~_REGION_ENTRY_BITS) != 0; 618 } 619 620 static inline int pmd_present(pmd_t pmd) 621 { 622 return pmd_val(pmd) != _SEGMENT_ENTRY_INVALID; 623 } 624 625 static inline int pmd_none(pmd_t pmd) 626 { 627 return pmd_val(pmd) == _SEGMENT_ENTRY_INVALID; 628 } 629 630 static inline unsigned long pmd_pfn(pmd_t pmd) 631 { 632 unsigned long origin_mask; 633 634 origin_mask = _SEGMENT_ENTRY_ORIGIN; 635 if (pmd_large(pmd)) 636 origin_mask = _SEGMENT_ENTRY_ORIGIN_LARGE; 637 return (pmd_val(pmd) & origin_mask) >> PAGE_SHIFT; 638 } 639 640 #define __HAVE_ARCH_PMD_WRITE 641 static inline int pmd_write(pmd_t pmd) 642 { 643 return (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) != 0; 644 } 645 646 static inline int pmd_dirty(pmd_t pmd) 647 { 648 int dirty = 1; 649 if (pmd_large(pmd)) 650 dirty = (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) != 0; 651 return dirty; 652 } 653 654 static inline int pmd_young(pmd_t pmd) 655 { 656 int young = 1; 657 if (pmd_large(pmd)) 658 young = (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) != 0; 659 return young; 660 } 661 662 static inline int pte_present(pte_t pte) 663 { 664 /* Bit pattern: (pte & 0x001) == 0x001 */ 665 return (pte_val(pte) & _PAGE_PRESENT) != 0; 666 } 667 668 static inline int pte_none(pte_t pte) 669 { 670 /* Bit pattern: pte == 0x400 */ 671 return pte_val(pte) == _PAGE_INVALID; 672 } 673 674 static inline int pte_swap(pte_t pte) 675 { 676 /* Bit pattern: (pte & 0x201) == 0x200 */ 677 return (pte_val(pte) & (_PAGE_PROTECT | _PAGE_PRESENT)) 678 == _PAGE_PROTECT; 679 } 680 681 static inline int pte_special(pte_t pte) 682 { 683 return (pte_val(pte) & _PAGE_SPECIAL); 684 } 685 686 #define __HAVE_ARCH_PTE_SAME 687 static inline int pte_same(pte_t a, pte_t b) 688 { 689 return pte_val(a) == pte_val(b); 690 } 691 692 #ifdef CONFIG_NUMA_BALANCING 693 static inline int pte_protnone(pte_t pte) 694 { 695 return pte_present(pte) && !(pte_val(pte) & _PAGE_READ); 696 } 697 698 static inline int pmd_protnone(pmd_t pmd) 699 { 700 /* pmd_large(pmd) implies pmd_present(pmd) */ 701 return pmd_large(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_READ); 702 } 703 #endif 704 705 static inline int pte_soft_dirty(pte_t pte) 706 { 707 return pte_val(pte) & _PAGE_SOFT_DIRTY; 708 } 709 #define pte_swp_soft_dirty pte_soft_dirty 710 711 static inline pte_t pte_mksoft_dirty(pte_t pte) 712 { 713 pte_val(pte) |= _PAGE_SOFT_DIRTY; 714 return pte; 715 } 716 #define pte_swp_mksoft_dirty pte_mksoft_dirty 717 718 static inline pte_t pte_clear_soft_dirty(pte_t pte) 719 { 720 pte_val(pte) &= ~_PAGE_SOFT_DIRTY; 721 return pte; 722 } 723 #define pte_swp_clear_soft_dirty pte_clear_soft_dirty 724 725 static inline int pmd_soft_dirty(pmd_t pmd) 726 { 727 return pmd_val(pmd) & _SEGMENT_ENTRY_SOFT_DIRTY; 728 } 729 730 static inline pmd_t pmd_mksoft_dirty(pmd_t pmd) 731 { 732 pmd_val(pmd) |= _SEGMENT_ENTRY_SOFT_DIRTY; 733 return pmd; 734 } 735 736 static inline pmd_t pmd_clear_soft_dirty(pmd_t pmd) 737 { 738 pmd_val(pmd) &= ~_SEGMENT_ENTRY_SOFT_DIRTY; 739 return pmd; 740 } 741 742 /* 743 * query functions pte_write/pte_dirty/pte_young only work if 744 * pte_present() is true. Undefined behaviour if not.. 745 */ 746 static inline int pte_write(pte_t pte) 747 { 748 return (pte_val(pte) & _PAGE_WRITE) != 0; 749 } 750 751 static inline int pte_dirty(pte_t pte) 752 { 753 return (pte_val(pte) & _PAGE_DIRTY) != 0; 754 } 755 756 static inline int pte_young(pte_t pte) 757 { 758 return (pte_val(pte) & _PAGE_YOUNG) != 0; 759 } 760 761 #define __HAVE_ARCH_PTE_UNUSED 762 static inline int pte_unused(pte_t pte) 763 { 764 return pte_val(pte) & _PAGE_UNUSED; 765 } 766 767 /* 768 * pgd/pmd/pte modification functions 769 */ 770 771 static inline void pgd_clear(pgd_t *pgd) 772 { 773 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) 774 pgd_val(*pgd) = _REGION2_ENTRY_EMPTY; 775 } 776 777 static inline void pud_clear(pud_t *pud) 778 { 779 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) 780 pud_val(*pud) = _REGION3_ENTRY_EMPTY; 781 } 782 783 static inline void pmd_clear(pmd_t *pmdp) 784 { 785 pmd_val(*pmdp) = _SEGMENT_ENTRY_INVALID; 786 } 787 788 static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) 789 { 790 pte_val(*ptep) = _PAGE_INVALID; 791 } 792 793 /* 794 * The following pte modification functions only work if 795 * pte_present() is true. Undefined behaviour if not.. 796 */ 797 static inline pte_t pte_modify(pte_t pte, pgprot_t newprot) 798 { 799 pte_val(pte) &= _PAGE_CHG_MASK; 800 pte_val(pte) |= pgprot_val(newprot); 801 /* 802 * newprot for PAGE_NONE, PAGE_READ and PAGE_WRITE has the 803 * invalid bit set, clear it again for readable, young pages 804 */ 805 if ((pte_val(pte) & _PAGE_YOUNG) && (pte_val(pte) & _PAGE_READ)) 806 pte_val(pte) &= ~_PAGE_INVALID; 807 /* 808 * newprot for PAGE_READ and PAGE_WRITE has the page protection 809 * bit set, clear it again for writable, dirty pages 810 */ 811 if ((pte_val(pte) & _PAGE_DIRTY) && (pte_val(pte) & _PAGE_WRITE)) 812 pte_val(pte) &= ~_PAGE_PROTECT; 813 return pte; 814 } 815 816 static inline pte_t pte_wrprotect(pte_t pte) 817 { 818 pte_val(pte) &= ~_PAGE_WRITE; 819 pte_val(pte) |= _PAGE_PROTECT; 820 return pte; 821 } 822 823 static inline pte_t pte_mkwrite(pte_t pte) 824 { 825 pte_val(pte) |= _PAGE_WRITE; 826 if (pte_val(pte) & _PAGE_DIRTY) 827 pte_val(pte) &= ~_PAGE_PROTECT; 828 return pte; 829 } 830 831 static inline pte_t pte_mkclean(pte_t pte) 832 { 833 pte_val(pte) &= ~_PAGE_DIRTY; 834 pte_val(pte) |= _PAGE_PROTECT; 835 return pte; 836 } 837 838 static inline pte_t pte_mkdirty(pte_t pte) 839 { 840 pte_val(pte) |= _PAGE_DIRTY | _PAGE_SOFT_DIRTY; 841 if (pte_val(pte) & _PAGE_WRITE) 842 pte_val(pte) &= ~_PAGE_PROTECT; 843 return pte; 844 } 845 846 static inline pte_t pte_mkold(pte_t pte) 847 { 848 pte_val(pte) &= ~_PAGE_YOUNG; 849 pte_val(pte) |= _PAGE_INVALID; 850 return pte; 851 } 852 853 static inline pte_t pte_mkyoung(pte_t pte) 854 { 855 pte_val(pte) |= _PAGE_YOUNG; 856 if (pte_val(pte) & _PAGE_READ) 857 pte_val(pte) &= ~_PAGE_INVALID; 858 return pte; 859 } 860 861 static inline pte_t pte_mkspecial(pte_t pte) 862 { 863 pte_val(pte) |= _PAGE_SPECIAL; 864 return pte; 865 } 866 867 #ifdef CONFIG_HUGETLB_PAGE 868 static inline pte_t pte_mkhuge(pte_t pte) 869 { 870 pte_val(pte) |= _PAGE_LARGE; 871 return pte; 872 } 873 #endif 874 875 static inline void __ptep_ipte(unsigned long address, pte_t *ptep) 876 { 877 unsigned long pto = (unsigned long) ptep; 878 879 /* Invalidation + global TLB flush for the pte */ 880 asm volatile( 881 " ipte %2,%3" 882 : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address)); 883 } 884 885 static inline void __ptep_ipte_local(unsigned long address, pte_t *ptep) 886 { 887 unsigned long pto = (unsigned long) ptep; 888 889 /* Invalidation + local TLB flush for the pte */ 890 asm volatile( 891 " .insn rrf,0xb2210000,%2,%3,0,1" 892 : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address)); 893 } 894 895 static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep) 896 { 897 unsigned long pto = (unsigned long) ptep; 898 899 /* Invalidate a range of ptes + global TLB flush of the ptes */ 900 do { 901 asm volatile( 902 " .insn rrf,0xb2210000,%2,%0,%1,0" 903 : "+a" (address), "+a" (nr) : "a" (pto) : "memory"); 904 } while (nr != 255); 905 } 906 907 /* 908 * This is hard to understand. ptep_get_and_clear and ptep_clear_flush 909 * both clear the TLB for the unmapped pte. The reason is that 910 * ptep_get_and_clear is used in common code (e.g. change_pte_range) 911 * to modify an active pte. The sequence is 912 * 1) ptep_get_and_clear 913 * 2) set_pte_at 914 * 3) flush_tlb_range 915 * On s390 the tlb needs to get flushed with the modification of the pte 916 * if the pte is active. The only way how this can be implemented is to 917 * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range 918 * is a nop. 919 */ 920 pte_t ptep_xchg_direct(struct mm_struct *, unsigned long, pte_t *, pte_t); 921 pte_t ptep_xchg_lazy(struct mm_struct *, unsigned long, pte_t *, pte_t); 922 923 #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG 924 static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, 925 unsigned long addr, pte_t *ptep) 926 { 927 pte_t pte = *ptep; 928 929 pte = ptep_xchg_direct(vma->vm_mm, addr, ptep, pte_mkold(pte)); 930 return pte_young(pte); 931 } 932 933 #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH 934 static inline int ptep_clear_flush_young(struct vm_area_struct *vma, 935 unsigned long address, pte_t *ptep) 936 { 937 return ptep_test_and_clear_young(vma, address, ptep); 938 } 939 940 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR 941 static inline pte_t ptep_get_and_clear(struct mm_struct *mm, 942 unsigned long addr, pte_t *ptep) 943 { 944 return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); 945 } 946 947 #define __HAVE_ARCH_PTEP_MODIFY_PROT_TRANSACTION 948 pte_t ptep_modify_prot_start(struct mm_struct *, unsigned long, pte_t *); 949 void ptep_modify_prot_commit(struct mm_struct *, unsigned long, pte_t *, pte_t); 950 951 #define __HAVE_ARCH_PTEP_CLEAR_FLUSH 952 static inline pte_t ptep_clear_flush(struct vm_area_struct *vma, 953 unsigned long addr, pte_t *ptep) 954 { 955 return ptep_xchg_direct(vma->vm_mm, addr, ptep, __pte(_PAGE_INVALID)); 956 } 957 958 /* 959 * The batched pte unmap code uses ptep_get_and_clear_full to clear the 960 * ptes. Here an optimization is possible. tlb_gather_mmu flushes all 961 * tlbs of an mm if it can guarantee that the ptes of the mm_struct 962 * cannot be accessed while the batched unmap is running. In this case 963 * full==1 and a simple pte_clear is enough. See tlb.h. 964 */ 965 #define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL 966 static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, 967 unsigned long addr, 968 pte_t *ptep, int full) 969 { 970 if (full) { 971 pte_t pte = *ptep; 972 *ptep = __pte(_PAGE_INVALID); 973 return pte; 974 } 975 return ptep_xchg_lazy(mm, addr, ptep, __pte(_PAGE_INVALID)); 976 } 977 978 #define __HAVE_ARCH_PTEP_SET_WRPROTECT 979 static inline void ptep_set_wrprotect(struct mm_struct *mm, 980 unsigned long addr, pte_t *ptep) 981 { 982 pte_t pte = *ptep; 983 984 if (pte_write(pte)) 985 ptep_xchg_lazy(mm, addr, ptep, pte_wrprotect(pte)); 986 } 987 988 #define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS 989 static inline int ptep_set_access_flags(struct vm_area_struct *vma, 990 unsigned long addr, pte_t *ptep, 991 pte_t entry, int dirty) 992 { 993 if (pte_same(*ptep, entry)) 994 return 0; 995 ptep_xchg_direct(vma->vm_mm, addr, ptep, entry); 996 return 1; 997 } 998 999 /* 1000 * Additional functions to handle KVM guest page tables 1001 */ 1002 void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, 1003 pte_t *ptep, pte_t entry); 1004 void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 1005 void ptep_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 1006 void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, 1007 pte_t *ptep , int reset); 1008 void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep); 1009 1010 bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long address); 1011 int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, 1012 unsigned char key, bool nq); 1013 unsigned char get_guest_storage_key(struct mm_struct *mm, unsigned long addr); 1014 1015 /* 1016 * Certain architectures need to do special things when PTEs 1017 * within a page table are directly modified. Thus, the following 1018 * hook is made available. 1019 */ 1020 static inline void set_pte_at(struct mm_struct *mm, unsigned long addr, 1021 pte_t *ptep, pte_t entry) 1022 { 1023 if (mm_has_pgste(mm)) 1024 ptep_set_pte_at(mm, addr, ptep, entry); 1025 else 1026 *ptep = entry; 1027 } 1028 1029 /* 1030 * Conversion functions: convert a page and protection to a page entry, 1031 * and a page entry and page directory to the page they refer to. 1032 */ 1033 static inline pte_t mk_pte_phys(unsigned long physpage, pgprot_t pgprot) 1034 { 1035 pte_t __pte; 1036 pte_val(__pte) = physpage + pgprot_val(pgprot); 1037 return pte_mkyoung(__pte); 1038 } 1039 1040 static inline pte_t mk_pte(struct page *page, pgprot_t pgprot) 1041 { 1042 unsigned long physpage = page_to_phys(page); 1043 pte_t __pte = mk_pte_phys(physpage, pgprot); 1044 1045 if (pte_write(__pte) && PageDirty(page)) 1046 __pte = pte_mkdirty(__pte); 1047 return __pte; 1048 } 1049 1050 #define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1)) 1051 #define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1)) 1052 #define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1)) 1053 #define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1)) 1054 1055 #define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) 1056 #define pgd_offset_k(address) pgd_offset(&init_mm, address) 1057 1058 #define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN) 1059 #define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN) 1060 #define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) 1061 1062 static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address) 1063 { 1064 pud_t *pud = (pud_t *) pgd; 1065 if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2) 1066 pud = (pud_t *) pgd_deref(*pgd); 1067 return pud + pud_index(address); 1068 } 1069 1070 static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address) 1071 { 1072 pmd_t *pmd = (pmd_t *) pud; 1073 if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3) 1074 pmd = (pmd_t *) pud_deref(*pud); 1075 return pmd + pmd_index(address); 1076 } 1077 1078 #define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot)) 1079 #define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT) 1080 #define pte_page(x) pfn_to_page(pte_pfn(x)) 1081 1082 #define pmd_page(pmd) pfn_to_page(pmd_pfn(pmd)) 1083 #define pud_page(pud) pfn_to_page(pud_pfn(pud)) 1084 1085 /* Find an entry in the lowest level page table.. */ 1086 #define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr)) 1087 #define pte_offset_kernel(pmd, address) pte_offset(pmd,address) 1088 #define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address) 1089 #define pte_unmap(pte) do { } while (0) 1090 1091 static inline pmd_t pmd_wrprotect(pmd_t pmd) 1092 { 1093 pmd_val(pmd) &= ~_SEGMENT_ENTRY_WRITE; 1094 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT; 1095 return pmd; 1096 } 1097 1098 static inline pmd_t pmd_mkwrite(pmd_t pmd) 1099 { 1100 pmd_val(pmd) |= _SEGMENT_ENTRY_WRITE; 1101 if (pmd_large(pmd) && !(pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY)) 1102 return pmd; 1103 pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT; 1104 return pmd; 1105 } 1106 1107 static inline pmd_t pmd_mkclean(pmd_t pmd) 1108 { 1109 if (pmd_large(pmd)) { 1110 pmd_val(pmd) &= ~_SEGMENT_ENTRY_DIRTY; 1111 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT; 1112 } 1113 return pmd; 1114 } 1115 1116 static inline pmd_t pmd_mkdirty(pmd_t pmd) 1117 { 1118 if (pmd_large(pmd)) { 1119 pmd_val(pmd) |= _SEGMENT_ENTRY_DIRTY | 1120 _SEGMENT_ENTRY_SOFT_DIRTY; 1121 if (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) 1122 pmd_val(pmd) &= ~_SEGMENT_ENTRY_PROTECT; 1123 } 1124 return pmd; 1125 } 1126 1127 static inline pud_t pud_wrprotect(pud_t pud) 1128 { 1129 pud_val(pud) &= ~_REGION3_ENTRY_WRITE; 1130 pud_val(pud) |= _REGION_ENTRY_PROTECT; 1131 return pud; 1132 } 1133 1134 static inline pud_t pud_mkwrite(pud_t pud) 1135 { 1136 pud_val(pud) |= _REGION3_ENTRY_WRITE; 1137 if (pud_large(pud) && !(pud_val(pud) & _REGION3_ENTRY_DIRTY)) 1138 return pud; 1139 pud_val(pud) &= ~_REGION_ENTRY_PROTECT; 1140 return pud; 1141 } 1142 1143 static inline pud_t pud_mkclean(pud_t pud) 1144 { 1145 if (pud_large(pud)) { 1146 pud_val(pud) &= ~_REGION3_ENTRY_DIRTY; 1147 pud_val(pud) |= _REGION_ENTRY_PROTECT; 1148 } 1149 return pud; 1150 } 1151 1152 static inline pud_t pud_mkdirty(pud_t pud) 1153 { 1154 if (pud_large(pud)) { 1155 pud_val(pud) |= _REGION3_ENTRY_DIRTY | 1156 _REGION3_ENTRY_SOFT_DIRTY; 1157 if (pud_val(pud) & _REGION3_ENTRY_WRITE) 1158 pud_val(pud) &= ~_REGION_ENTRY_PROTECT; 1159 } 1160 return pud; 1161 } 1162 1163 #if defined(CONFIG_TRANSPARENT_HUGEPAGE) || defined(CONFIG_HUGETLB_PAGE) 1164 static inline unsigned long massage_pgprot_pmd(pgprot_t pgprot) 1165 { 1166 /* 1167 * pgprot is PAGE_NONE, PAGE_READ, or PAGE_WRITE (see __Pxxx / __Sxxx) 1168 * Convert to segment table entry format. 1169 */ 1170 if (pgprot_val(pgprot) == pgprot_val(PAGE_NONE)) 1171 return pgprot_val(SEGMENT_NONE); 1172 if (pgprot_val(pgprot) == pgprot_val(PAGE_READ)) 1173 return pgprot_val(SEGMENT_READ); 1174 return pgprot_val(SEGMENT_WRITE); 1175 } 1176 1177 static inline pmd_t pmd_mkyoung(pmd_t pmd) 1178 { 1179 if (pmd_large(pmd)) { 1180 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG; 1181 if (pmd_val(pmd) & _SEGMENT_ENTRY_READ) 1182 pmd_val(pmd) &= ~_SEGMENT_ENTRY_INVALID; 1183 } 1184 return pmd; 1185 } 1186 1187 static inline pmd_t pmd_mkold(pmd_t pmd) 1188 { 1189 if (pmd_large(pmd)) { 1190 pmd_val(pmd) &= ~_SEGMENT_ENTRY_YOUNG; 1191 pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID; 1192 } 1193 return pmd; 1194 } 1195 1196 static inline pmd_t pmd_modify(pmd_t pmd, pgprot_t newprot) 1197 { 1198 if (pmd_large(pmd)) { 1199 pmd_val(pmd) &= _SEGMENT_ENTRY_ORIGIN_LARGE | 1200 _SEGMENT_ENTRY_DIRTY | _SEGMENT_ENTRY_YOUNG | 1201 _SEGMENT_ENTRY_LARGE | _SEGMENT_ENTRY_SOFT_DIRTY; 1202 pmd_val(pmd) |= massage_pgprot_pmd(newprot); 1203 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY)) 1204 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT; 1205 if (!(pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG)) 1206 pmd_val(pmd) |= _SEGMENT_ENTRY_INVALID; 1207 return pmd; 1208 } 1209 pmd_val(pmd) &= _SEGMENT_ENTRY_ORIGIN; 1210 pmd_val(pmd) |= massage_pgprot_pmd(newprot); 1211 return pmd; 1212 } 1213 1214 static inline pmd_t mk_pmd_phys(unsigned long physpage, pgprot_t pgprot) 1215 { 1216 pmd_t __pmd; 1217 pmd_val(__pmd) = physpage + massage_pgprot_pmd(pgprot); 1218 return __pmd; 1219 } 1220 1221 #endif /* CONFIG_TRANSPARENT_HUGEPAGE || CONFIG_HUGETLB_PAGE */ 1222 1223 static inline void __pmdp_csp(pmd_t *pmdp) 1224 { 1225 csp((unsigned int *)pmdp + 1, pmd_val(*pmdp), 1226 pmd_val(*pmdp) | _SEGMENT_ENTRY_INVALID); 1227 } 1228 1229 static inline void __pmdp_idte(unsigned long address, pmd_t *pmdp) 1230 { 1231 unsigned long sto; 1232 1233 sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t); 1234 asm volatile( 1235 " .insn rrf,0xb98e0000,%2,%3,0,0" 1236 : "=m" (*pmdp) 1237 : "m" (*pmdp), "a" (sto), "a" ((address & HPAGE_MASK)) 1238 : "cc" ); 1239 } 1240 1241 static inline void __pudp_idte(unsigned long address, pud_t *pudp) 1242 { 1243 unsigned long r3o; 1244 1245 r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t); 1246 r3o |= _ASCE_TYPE_REGION3; 1247 asm volatile( 1248 " .insn rrf,0xb98e0000,%2,%3,0,0" 1249 : "=m" (*pudp) 1250 : "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK)) 1251 : "cc"); 1252 } 1253 1254 static inline void __pmdp_idte_local(unsigned long address, pmd_t *pmdp) 1255 { 1256 unsigned long sto; 1257 1258 sto = (unsigned long) pmdp - pmd_index(address) * sizeof(pmd_t); 1259 asm volatile( 1260 " .insn rrf,0xb98e0000,%2,%3,0,1" 1261 : "=m" (*pmdp) 1262 : "m" (*pmdp), "a" (sto), "a" ((address & HPAGE_MASK)) 1263 : "cc" ); 1264 } 1265 1266 static inline void __pudp_idte_local(unsigned long address, pud_t *pudp) 1267 { 1268 unsigned long r3o; 1269 1270 r3o = (unsigned long) pudp - pud_index(address) * sizeof(pud_t); 1271 r3o |= _ASCE_TYPE_REGION3; 1272 asm volatile( 1273 " .insn rrf,0xb98e0000,%2,%3,0,1" 1274 : "=m" (*pudp) 1275 : "m" (*pudp), "a" (r3o), "a" ((address & PUD_MASK)) 1276 : "cc"); 1277 } 1278 1279 pmd_t pmdp_xchg_direct(struct mm_struct *, unsigned long, pmd_t *, pmd_t); 1280 pmd_t pmdp_xchg_lazy(struct mm_struct *, unsigned long, pmd_t *, pmd_t); 1281 pud_t pudp_xchg_direct(struct mm_struct *, unsigned long, pud_t *, pud_t); 1282 1283 #ifdef CONFIG_TRANSPARENT_HUGEPAGE 1284 1285 #define __HAVE_ARCH_PGTABLE_DEPOSIT 1286 void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, 1287 pgtable_t pgtable); 1288 1289 #define __HAVE_ARCH_PGTABLE_WITHDRAW 1290 pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp); 1291 1292 #define __HAVE_ARCH_PMDP_SET_ACCESS_FLAGS 1293 static inline int pmdp_set_access_flags(struct vm_area_struct *vma, 1294 unsigned long addr, pmd_t *pmdp, 1295 pmd_t entry, int dirty) 1296 { 1297 VM_BUG_ON(addr & ~HPAGE_MASK); 1298 1299 entry = pmd_mkyoung(entry); 1300 if (dirty) 1301 entry = pmd_mkdirty(entry); 1302 if (pmd_val(*pmdp) == pmd_val(entry)) 1303 return 0; 1304 pmdp_xchg_direct(vma->vm_mm, addr, pmdp, entry); 1305 return 1; 1306 } 1307 1308 #define __HAVE_ARCH_PMDP_TEST_AND_CLEAR_YOUNG 1309 static inline int pmdp_test_and_clear_young(struct vm_area_struct *vma, 1310 unsigned long addr, pmd_t *pmdp) 1311 { 1312 pmd_t pmd = *pmdp; 1313 1314 pmd = pmdp_xchg_direct(vma->vm_mm, addr, pmdp, pmd_mkold(pmd)); 1315 return pmd_young(pmd); 1316 } 1317 1318 #define __HAVE_ARCH_PMDP_CLEAR_YOUNG_FLUSH 1319 static inline int pmdp_clear_flush_young(struct vm_area_struct *vma, 1320 unsigned long addr, pmd_t *pmdp) 1321 { 1322 VM_BUG_ON(addr & ~HPAGE_MASK); 1323 return pmdp_test_and_clear_young(vma, addr, pmdp); 1324 } 1325 1326 static inline void set_pmd_at(struct mm_struct *mm, unsigned long addr, 1327 pmd_t *pmdp, pmd_t entry) 1328 { 1329 *pmdp = entry; 1330 } 1331 1332 static inline pmd_t pmd_mkhuge(pmd_t pmd) 1333 { 1334 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE; 1335 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG; 1336 pmd_val(pmd) |= _SEGMENT_ENTRY_PROTECT; 1337 return pmd; 1338 } 1339 1340 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR 1341 static inline pmd_t pmdp_huge_get_and_clear(struct mm_struct *mm, 1342 unsigned long addr, pmd_t *pmdp) 1343 { 1344 return pmdp_xchg_direct(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID)); 1345 } 1346 1347 #define __HAVE_ARCH_PMDP_HUGE_GET_AND_CLEAR_FULL 1348 static inline pmd_t pmdp_huge_get_and_clear_full(struct mm_struct *mm, 1349 unsigned long addr, 1350 pmd_t *pmdp, int full) 1351 { 1352 if (full) { 1353 pmd_t pmd = *pmdp; 1354 *pmdp = __pmd(_SEGMENT_ENTRY_INVALID); 1355 return pmd; 1356 } 1357 return pmdp_xchg_lazy(mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID)); 1358 } 1359 1360 #define __HAVE_ARCH_PMDP_HUGE_CLEAR_FLUSH 1361 static inline pmd_t pmdp_huge_clear_flush(struct vm_area_struct *vma, 1362 unsigned long addr, pmd_t *pmdp) 1363 { 1364 return pmdp_huge_get_and_clear(vma->vm_mm, addr, pmdp); 1365 } 1366 1367 #define __HAVE_ARCH_PMDP_INVALIDATE 1368 static inline void pmdp_invalidate(struct vm_area_struct *vma, 1369 unsigned long addr, pmd_t *pmdp) 1370 { 1371 pmdp_xchg_direct(vma->vm_mm, addr, pmdp, __pmd(_SEGMENT_ENTRY_INVALID)); 1372 } 1373 1374 #define __HAVE_ARCH_PMDP_SET_WRPROTECT 1375 static inline void pmdp_set_wrprotect(struct mm_struct *mm, 1376 unsigned long addr, pmd_t *pmdp) 1377 { 1378 pmd_t pmd = *pmdp; 1379 1380 if (pmd_write(pmd)) 1381 pmd = pmdp_xchg_lazy(mm, addr, pmdp, pmd_wrprotect(pmd)); 1382 } 1383 1384 static inline pmd_t pmdp_collapse_flush(struct vm_area_struct *vma, 1385 unsigned long address, 1386 pmd_t *pmdp) 1387 { 1388 return pmdp_huge_get_and_clear(vma->vm_mm, address, pmdp); 1389 } 1390 #define pmdp_collapse_flush pmdp_collapse_flush 1391 1392 #define pfn_pmd(pfn, pgprot) mk_pmd_phys(__pa((pfn) << PAGE_SHIFT), (pgprot)) 1393 #define mk_pmd(page, pgprot) pfn_pmd(page_to_pfn(page), (pgprot)) 1394 1395 static inline int pmd_trans_huge(pmd_t pmd) 1396 { 1397 return pmd_val(pmd) & _SEGMENT_ENTRY_LARGE; 1398 } 1399 1400 #define has_transparent_hugepage has_transparent_hugepage 1401 static inline int has_transparent_hugepage(void) 1402 { 1403 return MACHINE_HAS_HPAGE ? 1 : 0; 1404 } 1405 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ 1406 1407 /* 1408 * 64 bit swap entry format: 1409 * A page-table entry has some bits we have to treat in a special way. 1410 * Bits 52 and bit 55 have to be zero, otherwise a specification 1411 * exception will occur instead of a page translation exception. The 1412 * specification exception has the bad habit not to store necessary 1413 * information in the lowcore. 1414 * Bits 54 and 63 are used to indicate the page type. 1415 * A swap pte is indicated by bit pattern (pte & 0x201) == 0x200 1416 * This leaves the bits 0-51 and bits 56-62 to store type and offset. 1417 * We use the 5 bits from 57-61 for the type and the 52 bits from 0-51 1418 * for the offset. 1419 * | offset |01100|type |00| 1420 * |0000000000111111111122222222223333333333444444444455|55555|55566|66| 1421 * |0123456789012345678901234567890123456789012345678901|23456|78901|23| 1422 */ 1423 1424 #define __SWP_OFFSET_MASK ((1UL << 52) - 1) 1425 #define __SWP_OFFSET_SHIFT 12 1426 #define __SWP_TYPE_MASK ((1UL << 5) - 1) 1427 #define __SWP_TYPE_SHIFT 2 1428 1429 static inline pte_t mk_swap_pte(unsigned long type, unsigned long offset) 1430 { 1431 pte_t pte; 1432 1433 pte_val(pte) = _PAGE_INVALID | _PAGE_PROTECT; 1434 pte_val(pte) |= (offset & __SWP_OFFSET_MASK) << __SWP_OFFSET_SHIFT; 1435 pte_val(pte) |= (type & __SWP_TYPE_MASK) << __SWP_TYPE_SHIFT; 1436 return pte; 1437 } 1438 1439 static inline unsigned long __swp_type(swp_entry_t entry) 1440 { 1441 return (entry.val >> __SWP_TYPE_SHIFT) & __SWP_TYPE_MASK; 1442 } 1443 1444 static inline unsigned long __swp_offset(swp_entry_t entry) 1445 { 1446 return (entry.val >> __SWP_OFFSET_SHIFT) & __SWP_OFFSET_MASK; 1447 } 1448 1449 static inline swp_entry_t __swp_entry(unsigned long type, unsigned long offset) 1450 { 1451 return (swp_entry_t) { pte_val(mk_swap_pte(type, offset)) }; 1452 } 1453 1454 #define __pte_to_swp_entry(pte) ((swp_entry_t) { pte_val(pte) }) 1455 #define __swp_entry_to_pte(x) ((pte_t) { (x).val }) 1456 1457 #endif /* !__ASSEMBLY__ */ 1458 1459 #define kern_addr_valid(addr) (1) 1460 1461 extern int vmem_add_mapping(unsigned long start, unsigned long size); 1462 extern int vmem_remove_mapping(unsigned long start, unsigned long size); 1463 extern int s390_enable_sie(void); 1464 extern int s390_enable_skey(void); 1465 extern void s390_reset_cmma(struct mm_struct *mm); 1466 1467 /* s390 has a private copy of get unmapped area to deal with cache synonyms */ 1468 #define HAVE_ARCH_UNMAPPED_AREA 1469 #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN 1470 1471 /* 1472 * No page table caches to initialise 1473 */ 1474 static inline void pgtable_cache_init(void) { } 1475 static inline void check_pgt_cache(void) { } 1476 1477 #include <asm-generic/pgtable.h> 1478 1479 #endif /* _S390_PAGE_H */ 1480