1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  * Page table handling routines for radix page table.
4  *
5  * Copyright 2015-2016, Aneesh Kumar K.V, IBM Corporation.
6  */
7 
8 #define pr_fmt(fmt) "radix-mmu: " fmt
9 
10 #include <linux/io.h>
11 #include <linux/kernel.h>
12 #include <linux/sched/mm.h>
13 #include <linux/memblock.h>
14 #include <linux/of.h>
15 #include <linux/of_fdt.h>
16 #include <linux/mm.h>
17 #include <linux/hugetlb.h>
18 #include <linux/string_helpers.h>
19 #include <linux/memory.h>
20 
21 #include <asm/pgalloc.h>
22 #include <asm/mmu_context.h>
23 #include <asm/dma.h>
24 #include <asm/machdep.h>
25 #include <asm/mmu.h>
26 #include <asm/firmware.h>
27 #include <asm/powernv.h>
28 #include <asm/sections.h>
29 #include <asm/smp.h>
30 #include <asm/trace.h>
31 #include <asm/uaccess.h>
32 #include <asm/ultravisor.h>
33 
34 #include <trace/events/thp.h>
35 
36 unsigned int mmu_base_pid;
37 unsigned long radix_mem_block_size __ro_after_init;
38 
39 static __ref void *early_alloc_pgtable(unsigned long size, int nid,
40 			unsigned long region_start, unsigned long region_end)
41 {
42 	phys_addr_t min_addr = MEMBLOCK_LOW_LIMIT;
43 	phys_addr_t max_addr = MEMBLOCK_ALLOC_ANYWHERE;
44 	void *ptr;
45 
46 	if (region_start)
47 		min_addr = region_start;
48 	if (region_end)
49 		max_addr = region_end;
50 
51 	ptr = memblock_alloc_try_nid(size, size, min_addr, max_addr, nid);
52 
53 	if (!ptr)
54 		panic("%s: Failed to allocate %lu bytes align=0x%lx nid=%d from=%pa max_addr=%pa\n",
55 		      __func__, size, size, nid, &min_addr, &max_addr);
56 
57 	return ptr;
58 }
59 
60 /*
61  * When allocating pud or pmd pointers, we allocate a complete page
62  * of PAGE_SIZE rather than PUD_TABLE_SIZE or PMD_TABLE_SIZE. This
63  * is to ensure that the page obtained from the memblock allocator
64  * can be completely used as page table page and can be freed
65  * correctly when the page table entries are removed.
66  */
67 static int early_map_kernel_page(unsigned long ea, unsigned long pa,
68 			  pgprot_t flags,
69 			  unsigned int map_page_size,
70 			  int nid,
71 			  unsigned long region_start, unsigned long region_end)
72 {
73 	unsigned long pfn = pa >> PAGE_SHIFT;
74 	pgd_t *pgdp;
75 	p4d_t *p4dp;
76 	pud_t *pudp;
77 	pmd_t *pmdp;
78 	pte_t *ptep;
79 
80 	pgdp = pgd_offset_k(ea);
81 	p4dp = p4d_offset(pgdp, ea);
82 	if (p4d_none(*p4dp)) {
83 		pudp = early_alloc_pgtable(PAGE_SIZE, nid,
84 					   region_start, region_end);
85 		p4d_populate(&init_mm, p4dp, pudp);
86 	}
87 	pudp = pud_offset(p4dp, ea);
88 	if (map_page_size == PUD_SIZE) {
89 		ptep = (pte_t *)pudp;
90 		goto set_the_pte;
91 	}
92 	if (pud_none(*pudp)) {
93 		pmdp = early_alloc_pgtable(PAGE_SIZE, nid, region_start,
94 					   region_end);
95 		pud_populate(&init_mm, pudp, pmdp);
96 	}
97 	pmdp = pmd_offset(pudp, ea);
98 	if (map_page_size == PMD_SIZE) {
99 		ptep = pmdp_ptep(pmdp);
100 		goto set_the_pte;
101 	}
102 	if (!pmd_present(*pmdp)) {
103 		ptep = early_alloc_pgtable(PAGE_SIZE, nid,
104 						region_start, region_end);
105 		pmd_populate_kernel(&init_mm, pmdp, ptep);
106 	}
107 	ptep = pte_offset_kernel(pmdp, ea);
108 
109 set_the_pte:
110 	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
111 	asm volatile("ptesync": : :"memory");
112 	return 0;
113 }
114 
115 /*
116  * nid, region_start, and region_end are hints to try to place the page
117  * table memory in the same node or region.
118  */
119 static int __map_kernel_page(unsigned long ea, unsigned long pa,
120 			  pgprot_t flags,
121 			  unsigned int map_page_size,
122 			  int nid,
123 			  unsigned long region_start, unsigned long region_end)
124 {
125 	unsigned long pfn = pa >> PAGE_SHIFT;
126 	pgd_t *pgdp;
127 	p4d_t *p4dp;
128 	pud_t *pudp;
129 	pmd_t *pmdp;
130 	pte_t *ptep;
131 	/*
132 	 * Make sure task size is correct as per the max adddr
133 	 */
134 	BUILD_BUG_ON(TASK_SIZE_USER64 > RADIX_PGTABLE_RANGE);
135 
136 #ifdef CONFIG_PPC_64K_PAGES
137 	BUILD_BUG_ON(RADIX_KERN_MAP_SIZE != (1UL << MAX_EA_BITS_PER_CONTEXT));
138 #endif
139 
140 	if (unlikely(!slab_is_available()))
141 		return early_map_kernel_page(ea, pa, flags, map_page_size,
142 						nid, region_start, region_end);
143 
144 	/*
145 	 * Should make page table allocation functions be able to take a
146 	 * node, so we can place kernel page tables on the right nodes after
147 	 * boot.
148 	 */
149 	pgdp = pgd_offset_k(ea);
150 	p4dp = p4d_offset(pgdp, ea);
151 	pudp = pud_alloc(&init_mm, p4dp, ea);
152 	if (!pudp)
153 		return -ENOMEM;
154 	if (map_page_size == PUD_SIZE) {
155 		ptep = (pte_t *)pudp;
156 		goto set_the_pte;
157 	}
158 	pmdp = pmd_alloc(&init_mm, pudp, ea);
159 	if (!pmdp)
160 		return -ENOMEM;
161 	if (map_page_size == PMD_SIZE) {
162 		ptep = pmdp_ptep(pmdp);
163 		goto set_the_pte;
164 	}
165 	ptep = pte_alloc_kernel(pmdp, ea);
166 	if (!ptep)
167 		return -ENOMEM;
168 
169 set_the_pte:
170 	set_pte_at(&init_mm, ea, ptep, pfn_pte(pfn, flags));
171 	asm volatile("ptesync": : :"memory");
172 	return 0;
173 }
174 
175 int radix__map_kernel_page(unsigned long ea, unsigned long pa,
176 			  pgprot_t flags,
177 			  unsigned int map_page_size)
178 {
179 	return __map_kernel_page(ea, pa, flags, map_page_size, -1, 0, 0);
180 }
181 
182 #ifdef CONFIG_STRICT_KERNEL_RWX
183 static void radix__change_memory_range(unsigned long start, unsigned long end,
184 				       unsigned long clear)
185 {
186 	unsigned long idx;
187 	pgd_t *pgdp;
188 	p4d_t *p4dp;
189 	pud_t *pudp;
190 	pmd_t *pmdp;
191 	pte_t *ptep;
192 
193 	start = ALIGN_DOWN(start, PAGE_SIZE);
194 	end = PAGE_ALIGN(end); // aligns up
195 
196 	pr_debug("Changing flags on range %lx-%lx removing 0x%lx\n",
197 		 start, end, clear);
198 
199 	for (idx = start; idx < end; idx += PAGE_SIZE) {
200 		pgdp = pgd_offset_k(idx);
201 		p4dp = p4d_offset(pgdp, idx);
202 		pudp = pud_alloc(&init_mm, p4dp, idx);
203 		if (!pudp)
204 			continue;
205 		if (pud_is_leaf(*pudp)) {
206 			ptep = (pte_t *)pudp;
207 			goto update_the_pte;
208 		}
209 		pmdp = pmd_alloc(&init_mm, pudp, idx);
210 		if (!pmdp)
211 			continue;
212 		if (pmd_is_leaf(*pmdp)) {
213 			ptep = pmdp_ptep(pmdp);
214 			goto update_the_pte;
215 		}
216 		ptep = pte_alloc_kernel(pmdp, idx);
217 		if (!ptep)
218 			continue;
219 update_the_pte:
220 		radix__pte_update(&init_mm, idx, ptep, clear, 0, 0);
221 	}
222 
223 	radix__flush_tlb_kernel_range(start, end);
224 }
225 
226 void radix__mark_rodata_ro(void)
227 {
228 	unsigned long start, end;
229 
230 	start = (unsigned long)_stext;
231 	end = (unsigned long)__init_begin;
232 
233 	radix__change_memory_range(start, end, _PAGE_WRITE);
234 }
235 
236 void radix__mark_initmem_nx(void)
237 {
238 	unsigned long start = (unsigned long)__init_begin;
239 	unsigned long end = (unsigned long)__init_end;
240 
241 	radix__change_memory_range(start, end, _PAGE_EXEC);
242 }
243 #endif /* CONFIG_STRICT_KERNEL_RWX */
244 
245 static inline void __meminit
246 print_mapping(unsigned long start, unsigned long end, unsigned long size, bool exec)
247 {
248 	char buf[10];
249 
250 	if (end <= start)
251 		return;
252 
253 	string_get_size(size, 1, STRING_UNITS_2, buf, sizeof(buf));
254 
255 	pr_info("Mapped 0x%016lx-0x%016lx with %s pages%s\n", start, end, buf,
256 		exec ? " (exec)" : "");
257 }
258 
259 static unsigned long next_boundary(unsigned long addr, unsigned long end)
260 {
261 #ifdef CONFIG_STRICT_KERNEL_RWX
262 	if (addr < __pa_symbol(__init_begin))
263 		return __pa_symbol(__init_begin);
264 #endif
265 	return end;
266 }
267 
268 static int __meminit create_physical_mapping(unsigned long start,
269 					     unsigned long end,
270 					     unsigned long max_mapping_size,
271 					     int nid, pgprot_t _prot)
272 {
273 	unsigned long vaddr, addr, mapping_size = 0;
274 	bool prev_exec, exec = false;
275 	pgprot_t prot;
276 	int psize;
277 
278 	start = ALIGN(start, PAGE_SIZE);
279 	end   = ALIGN_DOWN(end, PAGE_SIZE);
280 	for (addr = start; addr < end; addr += mapping_size) {
281 		unsigned long gap, previous_size;
282 		int rc;
283 
284 		gap = next_boundary(addr, end) - addr;
285 		if (gap > max_mapping_size)
286 			gap = max_mapping_size;
287 		previous_size = mapping_size;
288 		prev_exec = exec;
289 
290 		if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE &&
291 		    mmu_psize_defs[MMU_PAGE_1G].shift) {
292 			mapping_size = PUD_SIZE;
293 			psize = MMU_PAGE_1G;
294 		} else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE &&
295 			   mmu_psize_defs[MMU_PAGE_2M].shift) {
296 			mapping_size = PMD_SIZE;
297 			psize = MMU_PAGE_2M;
298 		} else {
299 			mapping_size = PAGE_SIZE;
300 			psize = mmu_virtual_psize;
301 		}
302 
303 		vaddr = (unsigned long)__va(addr);
304 
305 		if (overlaps_kernel_text(vaddr, vaddr + mapping_size) ||
306 		    overlaps_interrupt_vector_text(vaddr, vaddr + mapping_size)) {
307 			prot = PAGE_KERNEL_X;
308 			exec = true;
309 		} else {
310 			prot = _prot;
311 			exec = false;
312 		}
313 
314 		if (mapping_size != previous_size || exec != prev_exec) {
315 			print_mapping(start, addr, previous_size, prev_exec);
316 			start = addr;
317 		}
318 
319 		rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end);
320 		if (rc)
321 			return rc;
322 
323 		update_page_count(psize, 1);
324 	}
325 
326 	print_mapping(start, addr, mapping_size, exec);
327 	return 0;
328 }
329 
330 static void __init radix_init_pgtable(void)
331 {
332 	unsigned long rts_field;
333 	phys_addr_t start, end;
334 	u64 i;
335 
336 	/* We don't support slb for radix */
337 	slb_set_size(0);
338 
339 	/*
340 	 * Create the linear mapping
341 	 */
342 	for_each_mem_range(i, &start, &end) {
343 		/*
344 		 * The memblock allocator  is up at this point, so the
345 		 * page tables will be allocated within the range. No
346 		 * need or a node (which we don't have yet).
347 		 */
348 
349 		if (end >= RADIX_VMALLOC_START) {
350 			pr_warn("Outside the supported range\n");
351 			continue;
352 		}
353 
354 		WARN_ON(create_physical_mapping(start, end,
355 						radix_mem_block_size,
356 						-1, PAGE_KERNEL));
357 	}
358 
359 	if (!cpu_has_feature(CPU_FTR_HVMODE) &&
360 			cpu_has_feature(CPU_FTR_P9_RADIX_PREFETCH_BUG)) {
361 		/*
362 		 * Older versions of KVM on these machines prefer if the
363 		 * guest only uses the low 19 PID bits.
364 		 */
365 		mmu_pid_bits = 19;
366 	}
367 	mmu_base_pid = 1;
368 
369 	/*
370 	 * Allocate Partition table and process table for the
371 	 * host.
372 	 */
373 	BUG_ON(PRTB_SIZE_SHIFT > 36);
374 	process_tb = early_alloc_pgtable(1UL << PRTB_SIZE_SHIFT, -1, 0, 0);
375 	/*
376 	 * Fill in the process table.
377 	 */
378 	rts_field = radix__get_tree_size();
379 	process_tb->prtb0 = cpu_to_be64(rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE);
380 
381 	/*
382 	 * The init_mm context is given the first available (non-zero) PID,
383 	 * which is the "guard PID" and contains no page table. PIDR should
384 	 * never be set to zero because that duplicates the kernel address
385 	 * space at the 0x0... offset (quadrant 0)!
386 	 *
387 	 * An arbitrary PID that may later be allocated by the PID allocator
388 	 * for userspace processes must not be used either, because that
389 	 * would cause stale user mappings for that PID on CPUs outside of
390 	 * the TLB invalidation scheme (because it won't be in mm_cpumask).
391 	 *
392 	 * So permanently carve out one PID for the purpose of a guard PID.
393 	 */
394 	init_mm.context.id = mmu_base_pid;
395 	mmu_base_pid++;
396 }
397 
398 static void __init radix_init_partition_table(void)
399 {
400 	unsigned long rts_field, dw0, dw1;
401 
402 	mmu_partition_table_init();
403 	rts_field = radix__get_tree_size();
404 	dw0 = rts_field | __pa(init_mm.pgd) | RADIX_PGD_INDEX_SIZE | PATB_HR;
405 	dw1 = __pa(process_tb) | (PRTB_SIZE_SHIFT - 12) | PATB_GR;
406 	mmu_partition_table_set_entry(0, dw0, dw1, false);
407 
408 	pr_info("Initializing Radix MMU\n");
409 }
410 
411 static int __init get_idx_from_shift(unsigned int shift)
412 {
413 	int idx = -1;
414 
415 	switch (shift) {
416 	case 0xc:
417 		idx = MMU_PAGE_4K;
418 		break;
419 	case 0x10:
420 		idx = MMU_PAGE_64K;
421 		break;
422 	case 0x15:
423 		idx = MMU_PAGE_2M;
424 		break;
425 	case 0x1e:
426 		idx = MMU_PAGE_1G;
427 		break;
428 	}
429 	return idx;
430 }
431 
432 static int __init radix_dt_scan_page_sizes(unsigned long node,
433 					   const char *uname, int depth,
434 					   void *data)
435 {
436 	int size = 0;
437 	int shift, idx;
438 	unsigned int ap;
439 	const __be32 *prop;
440 	const char *type = of_get_flat_dt_prop(node, "device_type", NULL);
441 
442 	/* We are scanning "cpu" nodes only */
443 	if (type == NULL || strcmp(type, "cpu") != 0)
444 		return 0;
445 
446 	/* Grab page size encodings */
447 	prop = of_get_flat_dt_prop(node, "ibm,processor-radix-AP-encodings", &size);
448 	if (!prop)
449 		return 0;
450 
451 	pr_info("Page sizes from device-tree:\n");
452 	for (; size >= 4; size -= 4, ++prop) {
453 
454 		struct mmu_psize_def *def;
455 
456 		/* top 3 bit is AP encoding */
457 		shift = be32_to_cpu(prop[0]) & ~(0xe << 28);
458 		ap = be32_to_cpu(prop[0]) >> 29;
459 		pr_info("Page size shift = %d AP=0x%x\n", shift, ap);
460 
461 		idx = get_idx_from_shift(shift);
462 		if (idx < 0)
463 			continue;
464 
465 		def = &mmu_psize_defs[idx];
466 		def->shift = shift;
467 		def->ap  = ap;
468 		def->h_rpt_pgsize = psize_to_rpti_pgsize(idx);
469 	}
470 
471 	/* needed ? */
472 	cur_cpu_spec->mmu_features &= ~MMU_FTR_NO_SLBIE_B;
473 	return 1;
474 }
475 
476 #ifdef CONFIG_MEMORY_HOTPLUG
477 static int __init probe_memory_block_size(unsigned long node, const char *uname, int
478 					  depth, void *data)
479 {
480 	unsigned long *mem_block_size = (unsigned long *)data;
481 	const __be32 *prop;
482 	int len;
483 
484 	if (depth != 1)
485 		return 0;
486 
487 	if (strcmp(uname, "ibm,dynamic-reconfiguration-memory"))
488 		return 0;
489 
490 	prop = of_get_flat_dt_prop(node, "ibm,lmb-size", &len);
491 
492 	if (!prop || len < dt_root_size_cells * sizeof(__be32))
493 		/*
494 		 * Nothing in the device tree
495 		 */
496 		*mem_block_size = MIN_MEMORY_BLOCK_SIZE;
497 	else
498 		*mem_block_size = of_read_number(prop, dt_root_size_cells);
499 	return 1;
500 }
501 
502 static unsigned long __init radix_memory_block_size(void)
503 {
504 	unsigned long mem_block_size = MIN_MEMORY_BLOCK_SIZE;
505 
506 	/*
507 	 * OPAL firmware feature is set by now. Hence we are ok
508 	 * to test OPAL feature.
509 	 */
510 	if (firmware_has_feature(FW_FEATURE_OPAL))
511 		mem_block_size = 1UL * 1024 * 1024 * 1024;
512 	else
513 		of_scan_flat_dt(probe_memory_block_size, &mem_block_size);
514 
515 	return mem_block_size;
516 }
517 
518 #else   /* CONFIG_MEMORY_HOTPLUG */
519 
520 static unsigned long __init radix_memory_block_size(void)
521 {
522 	return 1UL * 1024 * 1024 * 1024;
523 }
524 
525 #endif /* CONFIG_MEMORY_HOTPLUG */
526 
527 
528 void __init radix__early_init_devtree(void)
529 {
530 	int rc;
531 
532 	/*
533 	 * Try to find the available page sizes in the device-tree
534 	 */
535 	rc = of_scan_flat_dt(radix_dt_scan_page_sizes, NULL);
536 	if (!rc) {
537 		/*
538 		 * No page size details found in device tree.
539 		 * Let's assume we have page 4k and 64k support
540 		 */
541 		mmu_psize_defs[MMU_PAGE_4K].shift = 12;
542 		mmu_psize_defs[MMU_PAGE_4K].ap = 0x0;
543 		mmu_psize_defs[MMU_PAGE_4K].h_rpt_pgsize =
544 			psize_to_rpti_pgsize(MMU_PAGE_4K);
545 
546 		mmu_psize_defs[MMU_PAGE_64K].shift = 16;
547 		mmu_psize_defs[MMU_PAGE_64K].ap = 0x5;
548 		mmu_psize_defs[MMU_PAGE_64K].h_rpt_pgsize =
549 			psize_to_rpti_pgsize(MMU_PAGE_64K);
550 	}
551 
552 	/*
553 	 * Max mapping size used when mapping pages. We don't use
554 	 * ppc_md.memory_block_size() here because this get called
555 	 * early and we don't have machine probe called yet. Also
556 	 * the pseries implementation only check for ibm,lmb-size.
557 	 * All hypervisor supporting radix do expose that device
558 	 * tree node.
559 	 */
560 	radix_mem_block_size = radix_memory_block_size();
561 	return;
562 }
563 
564 void __init radix__early_init_mmu(void)
565 {
566 	unsigned long lpcr;
567 
568 #ifdef CONFIG_PPC_64S_HASH_MMU
569 #ifdef CONFIG_PPC_64K_PAGES
570 	/* PAGE_SIZE mappings */
571 	mmu_virtual_psize = MMU_PAGE_64K;
572 #else
573 	mmu_virtual_psize = MMU_PAGE_4K;
574 #endif
575 
576 #ifdef CONFIG_SPARSEMEM_VMEMMAP
577 	/* vmemmap mapping */
578 	if (mmu_psize_defs[MMU_PAGE_2M].shift) {
579 		/*
580 		 * map vmemmap using 2M if available
581 		 */
582 		mmu_vmemmap_psize = MMU_PAGE_2M;
583 	} else
584 		mmu_vmemmap_psize = mmu_virtual_psize;
585 #endif
586 #endif
587 	/*
588 	 * initialize page table size
589 	 */
590 	__pte_index_size = RADIX_PTE_INDEX_SIZE;
591 	__pmd_index_size = RADIX_PMD_INDEX_SIZE;
592 	__pud_index_size = RADIX_PUD_INDEX_SIZE;
593 	__pgd_index_size = RADIX_PGD_INDEX_SIZE;
594 	__pud_cache_index = RADIX_PUD_INDEX_SIZE;
595 	__pte_table_size = RADIX_PTE_TABLE_SIZE;
596 	__pmd_table_size = RADIX_PMD_TABLE_SIZE;
597 	__pud_table_size = RADIX_PUD_TABLE_SIZE;
598 	__pgd_table_size = RADIX_PGD_TABLE_SIZE;
599 
600 	__pmd_val_bits = RADIX_PMD_VAL_BITS;
601 	__pud_val_bits = RADIX_PUD_VAL_BITS;
602 	__pgd_val_bits = RADIX_PGD_VAL_BITS;
603 
604 	__kernel_virt_start = RADIX_KERN_VIRT_START;
605 	__vmalloc_start = RADIX_VMALLOC_START;
606 	__vmalloc_end = RADIX_VMALLOC_END;
607 	__kernel_io_start = RADIX_KERN_IO_START;
608 	__kernel_io_end = RADIX_KERN_IO_END;
609 	vmemmap = (struct page *)RADIX_VMEMMAP_START;
610 	ioremap_bot = IOREMAP_BASE;
611 
612 #ifdef CONFIG_PCI
613 	pci_io_base = ISA_IO_BASE;
614 #endif
615 	__pte_frag_nr = RADIX_PTE_FRAG_NR;
616 	__pte_frag_size_shift = RADIX_PTE_FRAG_SIZE_SHIFT;
617 	__pmd_frag_nr = RADIX_PMD_FRAG_NR;
618 	__pmd_frag_size_shift = RADIX_PMD_FRAG_SIZE_SHIFT;
619 
620 	radix_init_pgtable();
621 
622 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
623 		lpcr = mfspr(SPRN_LPCR);
624 		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
625 		radix_init_partition_table();
626 	} else {
627 		radix_init_pseries();
628 	}
629 
630 	memblock_set_current_limit(MEMBLOCK_ALLOC_ANYWHERE);
631 
632 	/* Switch to the guard PID before turning on MMU */
633 	radix__switch_mmu_context(NULL, &init_mm);
634 	tlbiel_all();
635 }
636 
637 void radix__early_init_mmu_secondary(void)
638 {
639 	unsigned long lpcr;
640 	/*
641 	 * update partition table control register and UPRT
642 	 */
643 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
644 		lpcr = mfspr(SPRN_LPCR);
645 		mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR);
646 
647 		set_ptcr_when_no_uv(__pa(partition_tb) |
648 				    (PATB_SIZE_SHIFT - 12));
649 	}
650 
651 	radix__switch_mmu_context(NULL, &init_mm);
652 	tlbiel_all();
653 
654 	/* Make sure userspace can't change the AMR */
655 	mtspr(SPRN_UAMOR, 0);
656 }
657 
658 /* Called during kexec sequence with MMU off */
659 notrace void radix__mmu_cleanup_all(void)
660 {
661 	unsigned long lpcr;
662 
663 	if (!firmware_has_feature(FW_FEATURE_LPAR)) {
664 		lpcr = mfspr(SPRN_LPCR);
665 		mtspr(SPRN_LPCR, lpcr & ~LPCR_UPRT);
666 		set_ptcr_when_no_uv(0);
667 		powernv_set_nmmu_ptcr(0);
668 		radix__flush_tlb_all();
669 	}
670 }
671 
672 #ifdef CONFIG_MEMORY_HOTPLUG
673 static void free_pte_table(pte_t *pte_start, pmd_t *pmd)
674 {
675 	pte_t *pte;
676 	int i;
677 
678 	for (i = 0; i < PTRS_PER_PTE; i++) {
679 		pte = pte_start + i;
680 		if (!pte_none(*pte))
681 			return;
682 	}
683 
684 	pte_free_kernel(&init_mm, pte_start);
685 	pmd_clear(pmd);
686 }
687 
688 static void free_pmd_table(pmd_t *pmd_start, pud_t *pud)
689 {
690 	pmd_t *pmd;
691 	int i;
692 
693 	for (i = 0; i < PTRS_PER_PMD; i++) {
694 		pmd = pmd_start + i;
695 		if (!pmd_none(*pmd))
696 			return;
697 	}
698 
699 	pmd_free(&init_mm, pmd_start);
700 	pud_clear(pud);
701 }
702 
703 static void free_pud_table(pud_t *pud_start, p4d_t *p4d)
704 {
705 	pud_t *pud;
706 	int i;
707 
708 	for (i = 0; i < PTRS_PER_PUD; i++) {
709 		pud = pud_start + i;
710 		if (!pud_none(*pud))
711 			return;
712 	}
713 
714 	pud_free(&init_mm, pud_start);
715 	p4d_clear(p4d);
716 }
717 
718 static void remove_pte_table(pte_t *pte_start, unsigned long addr,
719 			     unsigned long end)
720 {
721 	unsigned long next;
722 	pte_t *pte;
723 
724 	pte = pte_start + pte_index(addr);
725 	for (; addr < end; addr = next, pte++) {
726 		next = (addr + PAGE_SIZE) & PAGE_MASK;
727 		if (next > end)
728 			next = end;
729 
730 		if (!pte_present(*pte))
731 			continue;
732 
733 		if (!PAGE_ALIGNED(addr) || !PAGE_ALIGNED(next)) {
734 			/*
735 			 * The vmemmap_free() and remove_section_mapping()
736 			 * codepaths call us with aligned addresses.
737 			 */
738 			WARN_ONCE(1, "%s: unaligned range\n", __func__);
739 			continue;
740 		}
741 
742 		pte_clear(&init_mm, addr, pte);
743 	}
744 }
745 
746 static void __meminit remove_pmd_table(pmd_t *pmd_start, unsigned long addr,
747 			     unsigned long end)
748 {
749 	unsigned long next;
750 	pte_t *pte_base;
751 	pmd_t *pmd;
752 
753 	pmd = pmd_start + pmd_index(addr);
754 	for (; addr < end; addr = next, pmd++) {
755 		next = pmd_addr_end(addr, end);
756 
757 		if (!pmd_present(*pmd))
758 			continue;
759 
760 		if (pmd_is_leaf(*pmd)) {
761 			if (!IS_ALIGNED(addr, PMD_SIZE) ||
762 			    !IS_ALIGNED(next, PMD_SIZE)) {
763 				WARN_ONCE(1, "%s: unaligned range\n", __func__);
764 				continue;
765 			}
766 			pte_clear(&init_mm, addr, (pte_t *)pmd);
767 			continue;
768 		}
769 
770 		pte_base = (pte_t *)pmd_page_vaddr(*pmd);
771 		remove_pte_table(pte_base, addr, next);
772 		free_pte_table(pte_base, pmd);
773 	}
774 }
775 
776 static void __meminit remove_pud_table(pud_t *pud_start, unsigned long addr,
777 			     unsigned long end)
778 {
779 	unsigned long next;
780 	pmd_t *pmd_base;
781 	pud_t *pud;
782 
783 	pud = pud_start + pud_index(addr);
784 	for (; addr < end; addr = next, pud++) {
785 		next = pud_addr_end(addr, end);
786 
787 		if (!pud_present(*pud))
788 			continue;
789 
790 		if (pud_is_leaf(*pud)) {
791 			if (!IS_ALIGNED(addr, PUD_SIZE) ||
792 			    !IS_ALIGNED(next, PUD_SIZE)) {
793 				WARN_ONCE(1, "%s: unaligned range\n", __func__);
794 				continue;
795 			}
796 			pte_clear(&init_mm, addr, (pte_t *)pud);
797 			continue;
798 		}
799 
800 		pmd_base = pud_pgtable(*pud);
801 		remove_pmd_table(pmd_base, addr, next);
802 		free_pmd_table(pmd_base, pud);
803 	}
804 }
805 
806 static void __meminit remove_pagetable(unsigned long start, unsigned long end)
807 {
808 	unsigned long addr, next;
809 	pud_t *pud_base;
810 	pgd_t *pgd;
811 	p4d_t *p4d;
812 
813 	spin_lock(&init_mm.page_table_lock);
814 
815 	for (addr = start; addr < end; addr = next) {
816 		next = pgd_addr_end(addr, end);
817 
818 		pgd = pgd_offset_k(addr);
819 		p4d = p4d_offset(pgd, addr);
820 		if (!p4d_present(*p4d))
821 			continue;
822 
823 		if (p4d_is_leaf(*p4d)) {
824 			if (!IS_ALIGNED(addr, P4D_SIZE) ||
825 			    !IS_ALIGNED(next, P4D_SIZE)) {
826 				WARN_ONCE(1, "%s: unaligned range\n", __func__);
827 				continue;
828 			}
829 
830 			pte_clear(&init_mm, addr, (pte_t *)pgd);
831 			continue;
832 		}
833 
834 		pud_base = p4d_pgtable(*p4d);
835 		remove_pud_table(pud_base, addr, next);
836 		free_pud_table(pud_base, p4d);
837 	}
838 
839 	spin_unlock(&init_mm.page_table_lock);
840 	radix__flush_tlb_kernel_range(start, end);
841 }
842 
843 int __meminit radix__create_section_mapping(unsigned long start,
844 					    unsigned long end, int nid,
845 					    pgprot_t prot)
846 {
847 	if (end >= RADIX_VMALLOC_START) {
848 		pr_warn("Outside the supported range\n");
849 		return -1;
850 	}
851 
852 	return create_physical_mapping(__pa(start), __pa(end),
853 				       radix_mem_block_size, nid, prot);
854 }
855 
856 int __meminit radix__remove_section_mapping(unsigned long start, unsigned long end)
857 {
858 	remove_pagetable(start, end);
859 	return 0;
860 }
861 #endif /* CONFIG_MEMORY_HOTPLUG */
862 
863 #ifdef CONFIG_SPARSEMEM_VMEMMAP
864 static int __map_kernel_page_nid(unsigned long ea, unsigned long pa,
865 				 pgprot_t flags, unsigned int map_page_size,
866 				 int nid)
867 {
868 	return __map_kernel_page(ea, pa, flags, map_page_size, nid, 0, 0);
869 }
870 
871 int __meminit radix__vmemmap_create_mapping(unsigned long start,
872 				      unsigned long page_size,
873 				      unsigned long phys)
874 {
875 	/* Create a PTE encoding */
876 	unsigned long flags = _PAGE_PRESENT | _PAGE_ACCESSED | _PAGE_KERNEL_RW;
877 	int nid = early_pfn_to_nid(phys >> PAGE_SHIFT);
878 	int ret;
879 
880 	if ((start + page_size) >= RADIX_VMEMMAP_END) {
881 		pr_warn("Outside the supported range\n");
882 		return -1;
883 	}
884 
885 	ret = __map_kernel_page_nid(start, phys, __pgprot(flags), page_size, nid);
886 	BUG_ON(ret);
887 
888 	return 0;
889 }
890 
891 #ifdef CONFIG_MEMORY_HOTPLUG
892 void __meminit radix__vmemmap_remove_mapping(unsigned long start, unsigned long page_size)
893 {
894 	remove_pagetable(start, start + page_size);
895 }
896 #endif
897 #endif
898 
899 #ifdef CONFIG_DEBUG_PAGEALLOC
900 void radix__kernel_map_pages(struct page *page, int numpages, int enable)
901 {
902 	pr_warn_once("DEBUG_PAGEALLOC not supported in radix mode\n");
903 }
904 #endif
905 
906 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
907 
908 unsigned long radix__pmd_hugepage_update(struct mm_struct *mm, unsigned long addr,
909 				  pmd_t *pmdp, unsigned long clr,
910 				  unsigned long set)
911 {
912 	unsigned long old;
913 
914 #ifdef CONFIG_DEBUG_VM
915 	WARN_ON(!radix__pmd_trans_huge(*pmdp) && !pmd_devmap(*pmdp));
916 	assert_spin_locked(pmd_lockptr(mm, pmdp));
917 #endif
918 
919 	old = radix__pte_update(mm, addr, (pte_t *)pmdp, clr, set, 1);
920 	trace_hugepage_update(addr, old, clr, set);
921 
922 	return old;
923 }
924 
925 pmd_t radix__pmdp_collapse_flush(struct vm_area_struct *vma, unsigned long address,
926 			pmd_t *pmdp)
927 
928 {
929 	pmd_t pmd;
930 
931 	VM_BUG_ON(address & ~HPAGE_PMD_MASK);
932 	VM_BUG_ON(radix__pmd_trans_huge(*pmdp));
933 	VM_BUG_ON(pmd_devmap(*pmdp));
934 	/*
935 	 * khugepaged calls this for normal pmd
936 	 */
937 	pmd = *pmdp;
938 	pmd_clear(pmdp);
939 
940 	/*
941 	 * pmdp collapse_flush need to ensure that there are no parallel gup
942 	 * walk after this call. This is needed so that we can have stable
943 	 * page ref count when collapsing a page. We don't allow a collapse page
944 	 * if we have gup taken on the page. We can ensure that by sending IPI
945 	 * because gup walk happens with IRQ disabled.
946 	 */
947 	serialize_against_pte_lookup(vma->vm_mm);
948 
949 	radix__flush_tlb_collapsed_pmd(vma->vm_mm, address);
950 
951 	return pmd;
952 }
953 
954 /*
955  * For us pgtable_t is pte_t *. Inorder to save the deposisted
956  * page table, we consider the allocated page table as a list
957  * head. On withdraw we need to make sure we zero out the used
958  * list_head memory area.
959  */
960 void radix__pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp,
961 				 pgtable_t pgtable)
962 {
963 	struct list_head *lh = (struct list_head *) pgtable;
964 
965 	assert_spin_locked(pmd_lockptr(mm, pmdp));
966 
967 	/* FIFO */
968 	if (!pmd_huge_pte(mm, pmdp))
969 		INIT_LIST_HEAD(lh);
970 	else
971 		list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp));
972 	pmd_huge_pte(mm, pmdp) = pgtable;
973 }
974 
975 pgtable_t radix__pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp)
976 {
977 	pte_t *ptep;
978 	pgtable_t pgtable;
979 	struct list_head *lh;
980 
981 	assert_spin_locked(pmd_lockptr(mm, pmdp));
982 
983 	/* FIFO */
984 	pgtable = pmd_huge_pte(mm, pmdp);
985 	lh = (struct list_head *) pgtable;
986 	if (list_empty(lh))
987 		pmd_huge_pte(mm, pmdp) = NULL;
988 	else {
989 		pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next;
990 		list_del(lh);
991 	}
992 	ptep = (pte_t *) pgtable;
993 	*ptep = __pte(0);
994 	ptep++;
995 	*ptep = __pte(0);
996 	return pgtable;
997 }
998 
999 pmd_t radix__pmdp_huge_get_and_clear(struct mm_struct *mm,
1000 				     unsigned long addr, pmd_t *pmdp)
1001 {
1002 	pmd_t old_pmd;
1003 	unsigned long old;
1004 
1005 	old = radix__pmd_hugepage_update(mm, addr, pmdp, ~0UL, 0);
1006 	old_pmd = __pmd(old);
1007 	return old_pmd;
1008 }
1009 
1010 #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1011 
1012 void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep,
1013 				  pte_t entry, unsigned long address, int psize)
1014 {
1015 	struct mm_struct *mm = vma->vm_mm;
1016 	unsigned long set = pte_val(entry) & (_PAGE_DIRTY | _PAGE_ACCESSED |
1017 					      _PAGE_RW | _PAGE_EXEC);
1018 
1019 	unsigned long change = pte_val(entry) ^ pte_val(*ptep);
1020 	/*
1021 	 * To avoid NMMU hang while relaxing access, we need mark
1022 	 * the pte invalid in between.
1023 	 */
1024 	if ((change & _PAGE_RW) && atomic_read(&mm->context.copros) > 0) {
1025 		unsigned long old_pte, new_pte;
1026 
1027 		old_pte = __radix_pte_update(ptep, _PAGE_PRESENT, _PAGE_INVALID);
1028 		/*
1029 		 * new value of pte
1030 		 */
1031 		new_pte = old_pte | set;
1032 		radix__flush_tlb_page_psize(mm, address, psize);
1033 		__radix_pte_update(ptep, _PAGE_INVALID, new_pte);
1034 	} else {
1035 		__radix_pte_update(ptep, 0, set);
1036 		/*
1037 		 * Book3S does not require a TLB flush when relaxing access
1038 		 * restrictions when the address space is not attached to a
1039 		 * NMMU, because the core MMU will reload the pte after taking
1040 		 * an access fault, which is defined by the architecture.
1041 		 */
1042 	}
1043 	/* See ptesync comment in radix__set_pte_at */
1044 }
1045 
1046 void radix__ptep_modify_prot_commit(struct vm_area_struct *vma,
1047 				    unsigned long addr, pte_t *ptep,
1048 				    pte_t old_pte, pte_t pte)
1049 {
1050 	struct mm_struct *mm = vma->vm_mm;
1051 
1052 	/*
1053 	 * To avoid NMMU hang while relaxing access we need to flush the tlb before
1054 	 * we set the new value. We need to do this only for radix, because hash
1055 	 * translation does flush when updating the linux pte.
1056 	 */
1057 	if (is_pte_rw_upgrade(pte_val(old_pte), pte_val(pte)) &&
1058 	    (atomic_read(&mm->context.copros) > 0))
1059 		radix__flush_tlb_page(vma, addr);
1060 
1061 	set_pte_at(mm, addr, ptep, pte);
1062 }
1063 
1064 int pud_set_huge(pud_t *pud, phys_addr_t addr, pgprot_t prot)
1065 {
1066 	pte_t *ptep = (pte_t *)pud;
1067 	pte_t new_pud = pfn_pte(__phys_to_pfn(addr), prot);
1068 
1069 	if (!radix_enabled())
1070 		return 0;
1071 
1072 	set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pud);
1073 
1074 	return 1;
1075 }
1076 
1077 int pud_clear_huge(pud_t *pud)
1078 {
1079 	if (pud_is_leaf(*pud)) {
1080 		pud_clear(pud);
1081 		return 1;
1082 	}
1083 
1084 	return 0;
1085 }
1086 
1087 int pud_free_pmd_page(pud_t *pud, unsigned long addr)
1088 {
1089 	pmd_t *pmd;
1090 	int i;
1091 
1092 	pmd = pud_pgtable(*pud);
1093 	pud_clear(pud);
1094 
1095 	flush_tlb_kernel_range(addr, addr + PUD_SIZE);
1096 
1097 	for (i = 0; i < PTRS_PER_PMD; i++) {
1098 		if (!pmd_none(pmd[i])) {
1099 			pte_t *pte;
1100 			pte = (pte_t *)pmd_page_vaddr(pmd[i]);
1101 
1102 			pte_free_kernel(&init_mm, pte);
1103 		}
1104 	}
1105 
1106 	pmd_free(&init_mm, pmd);
1107 
1108 	return 1;
1109 }
1110 
1111 int pmd_set_huge(pmd_t *pmd, phys_addr_t addr, pgprot_t prot)
1112 {
1113 	pte_t *ptep = (pte_t *)pmd;
1114 	pte_t new_pmd = pfn_pte(__phys_to_pfn(addr), prot);
1115 
1116 	if (!radix_enabled())
1117 		return 0;
1118 
1119 	set_pte_at(&init_mm, 0 /* radix unused */, ptep, new_pmd);
1120 
1121 	return 1;
1122 }
1123 
1124 int pmd_clear_huge(pmd_t *pmd)
1125 {
1126 	if (pmd_is_leaf(*pmd)) {
1127 		pmd_clear(pmd);
1128 		return 1;
1129 	}
1130 
1131 	return 0;
1132 }
1133 
1134 int pmd_free_pte_page(pmd_t *pmd, unsigned long addr)
1135 {
1136 	pte_t *pte;
1137 
1138 	pte = (pte_t *)pmd_page_vaddr(*pmd);
1139 	pmd_clear(pmd);
1140 
1141 	flush_tlb_kernel_range(addr, addr + PMD_SIZE);
1142 
1143 	pte_free_kernel(&init_mm, pte);
1144 
1145 	return 1;
1146 }
1147