1 // SPDX-License-Identifier: GPL-2.0-or-later
6  * memory contents. The most of the code implementation has been adapted
33 #include <asm/fadump-internal.h>
58 #define RESERVED_RNGS_SZ	16384 /* 16K - 128 entries */
63 reserved_mrange_info = { "reserved", rngs, RESERVED_RNGS_SZ, 0, RESERVED_RNGS_CNT, true };
71  * fadump_cma_init() - Initialize CMA area from a fadump reserved memory
73  * This function initializes CMA area from fadump reserved memory.
74  * The total size of fadump reserved memory covers for boot memory size
76  * Initialize only the area equivalent to boot memory size for CMA use.
77  * The remaining portion of fadump reserved memory will be not given
78  * to CMA and pages for those will stay reserved. boot memory size is
80  * But for some reason even if it fails we still have the memory reservation
102 		pr_err("Failed to init cma area for firmware-assisted dump,%d\n", rc);  in fadump_cma_init()
104 		 * Though the CMA init has failed we still have memory  in fadump_cma_init()
105 		 * reservation with us. The reserved memory will be  in fadump_cma_init()
113 	 *  If CMA activation fails, keep the pages reserved, instead of  in fadump_cma_init()
122 		"bytes of memory reserved for firmware-assisted dump\n",  in fadump_cma_init()
155  * If fadump is registered, check if the memory provided
156  * falls within boot memory area and reserved memory area.
189  * Returns true, if there are no holes in memory area between d_start to d_end,
202 			/* Memory hole from d_start to start */  in is_fadump_mem_area_contiguous()
219  * Returns true, if there are no holes in boot memory area,
241  * Returns true, if there are no holes in reserved memory area,
258 	pr_debug("Support for firmware-assisted dump (fadump): %s\n",  in fadump_show_config()
271 	pr_debug("    Boot memory size   : %lx\n", fw_dump.boot_memory_size);  in fadump_show_config()
272 	pr_debug("    Boot memory top    : %llx\n", fw_dump.boot_mem_top);  in fadump_show_config()
273 	pr_debug("Boot memory regions cnt: %llx\n", fw_dump.boot_mem_regs_cnt);  in fadump_show_config()
283  * Function to find the largest memory size we need to reserve during early
284  * boot process. This will be the size of the memory that is required for a
287  * This function has been taken from phyp-assisted dump feature implementation.
291  * TODO: Come up with better approach to find out more accurate memory size
306 	 * memory at a predefined offset.  in fadump_calculate_reserve_size()
314 			pr_info("Using 'crashkernel=' parameter for memory reservation.\n");  in fadump_calculate_reserve_size()
319 		 * Adjust if the boot memory size specified is above  in fadump_calculate_reserve_size()
325 			pr_info("Adjusted boot memory size to %luMB\n",  in fadump_calculate_reserve_size()
332 		 * 'fadump_reserve_mem=' is being used to reserve memory  in fadump_calculate_reserve_size()
333 		 * for firmware-assisted dump.  in fadump_calculate_reserve_size()
344 	/* Truncate to memory_limit. We don't want to over reserve the memory.*/  in fadump_calculate_reserve_size()
348 	bootmem_min = fw_dump.ops->fadump_get_bootmem_min();  in fadump_calculate_reserve_size()
353  * Calculate the total memory size required to be reserved for
354  * firmware-assisted dump registration.
363 	 * Account for pagesize alignment of boot memory area destination address.  in get_fadump_area_size()
364 	 * This faciliates in mmap reading of first kernel's memory.  in get_fadump_area_size()
371 	/* Program headers for crash memory regions. */  in get_fadump_area_size()
372 	size += sizeof(struct elf_phdr) * (memblock_num_regions(memory) + 2);  in get_fadump_area_size()
377 	size += (fw_dump.ops->fadump_get_metadata_size ?  in get_fadump_area_size()
378 		 fw_dump.ops->fadump_get_metadata_size() : 0);  in get_fadump_area_size()
392 	pr_debug("Added boot memory range[%d] [%#016lx-%#016lx)\n",  in add_boot_mem_region()
401  * Honour that by splitting a memory range into multiple regions.
421 		msize -= rsize;  in add_boot_mem_regions()
442 		size = reg_end - reg_start;  in fadump_get_boot_mem_regions()
443 		hole_size += (reg_start - last_end);  in fadump_get_boot_mem_regions()
446 			size = (mem_size - cur_size);  in fadump_get_boot_mem_regions()
451 		mem_size -= size;  in fadump_get_boot_mem_regions()
465  * Returns true, if the given range overlaps with reserved memory ranges
466  * starting at idx. Also, updates idx to index of overlapping memory range
467  * with the given memory range.
493  * Locate a suitable memory area to reserve memory for FADump. While at it,
494  * lookup reserved-ranges & avoid overlap with them, as they are used by F/W.
512 		while ((mend > base) && ((mend - base) >= size)) {  in fadump_locate_reserve_mem()
536 		pr_info("Firmware-Assisted Dump is not supported on this hardware\n");  in fadump_reserve_mem()
541 	 * Initialize boot memory size  in fadump_reserve_mem()
556 		bootmem_min = fw_dump.ops->fadump_get_bootmem_min();  in fadump_reserve_mem()
558 			pr_err("Can't enable fadump with boot memory size (0x%lx) less than 0x%llx\n",  in fadump_reserve_mem()
564 			pr_err("Too many holes in boot memory area to enable fadump\n");  in fadump_reserve_mem()
570 	 * Calculate the memory boundary.  in fadump_reserve_mem()
571 	 * If memory_limit is less than actual memory boundary then reserve  in fadump_reserve_mem()
572 	 * the memory for fadump beyond the memory_limit and adjust the  in fadump_reserve_mem()
582 		printk(KERN_INFO "Adjusted memory_limit for firmware-assisted"  in fadump_reserve_mem()
594 		pr_info("Firmware-assisted dump is active.\n");  in fadump_reserve_mem()
605 		 * If last boot has crashed then reserve all the memory  in fadump_reserve_mem()
606 		 * above boot memory size so that we don't touch it until  in fadump_reserve_mem()
607 		 * dump is written to disk by userspace tool. This memory  in fadump_reserve_mem()
617 		 * Reserve memory at an offset closer to bottom of the RAM to  in fadump_reserve_mem()
618 		 * minimize the impact of memory hot-remove operation.  in fadump_reserve_mem()
623 			pr_err("Failed to find memory chunk for reservation!\n");  in fadump_reserve_mem()
632 		if (fw_dump.ops->fadump_setup_metadata &&  in fadump_reserve_mem()
633 		    (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0))  in fadump_reserve_mem()
637 			pr_err("Failed to reserve memory!\n");  in fadump_reserve_mem()
641 		pr_info("Reserved %lldMB of memory at %#016llx (System RAM: %lldMB)\n",  in fadump_reserve_mem()
690 	unsigned int ncpus = num_online_cpus() - 1;  in crash_fadump()
696 	 * old_cpu == -1 means this is the first CPU which has come here,  in crash_fadump()
699 	 * old_cpu != -1 means some other CPU has already on it's way  in crash_fadump()
703 	old_cpu = cmpxchg(&crashing_cpu, -1, this_cpu);  in crash_fadump()
705 	if (old_cpu != -1) {  in crash_fadump()
710 		 * is in force. If we race with fadump un-registration this  in crash_fadump()
721 	fdh->crashing_cpu = crashing_cpu;  in crash_fadump()
725 		fdh->regs = *regs;  in crash_fadump()
727 		ppc_save_regs(&fdh->regs);  in crash_fadump()
729 	fdh->cpu_mask = *cpu_online_mask;  in crash_fadump()
735 	if (TRAP(&(fdh->regs)) == INTERRUPT_SYSTEM_RESET) {  in crash_fadump()
737 		while ((atomic_read(&cpus_in_fadump) < ncpus) && (--msecs > 0))  in crash_fadump()
741 	fw_dump.ops->fadump_trigger(fdh, str);  in crash_fadump()
768 	if (phdr->p_type == PT_NOTE) {  in fadump_update_elfcore_header()
769 		phdr->p_paddr	= __pa(fw_dump.cpu_notes_buf_vaddr);  in fadump_update_elfcore_header()
770 		phdr->p_offset	= phdr->p_paddr;  in fadump_update_elfcore_header()
771 		phdr->p_filesz	= fw_dump.cpu_notes_buf_size;  in fadump_update_elfcore_header()
772 		phdr->p_memsz = fw_dump.cpu_notes_buf_size;  in fadump_update_elfcore_header()
796 	free_reserved_area((void *)vaddr, (void *)(vaddr + size), -1, NULL);  in fadump_free_buffer()
809 		return -ENOMEM;  in fadump_setup_cpu_notes_buf()
831 	if (mrange_info->is_static) {  in fadump_free_mem_ranges()
832 		mrange_info->mem_range_cnt = 0;  in fadump_free_mem_ranges()
836 	kfree(mrange_info->mem_ranges);  in fadump_free_mem_ranges()
838 	       (sizeof(struct fadump_mrange_info) - RNG_NAME_SZ));  in fadump_free_mem_ranges()
850 	new_size = mrange_info->mem_ranges_sz + PAGE_SIZE;  in fadump_alloc_mem_ranges()
851 	pr_debug("Allocating %llu bytes of memory for %s memory ranges\n",  in fadump_alloc_mem_ranges()
852 		 new_size, mrange_info->name);  in fadump_alloc_mem_ranges()
854 	new_array = krealloc(mrange_info->mem_ranges, new_size, GFP_KERNEL);  in fadump_alloc_mem_ranges()
856 		pr_err("Insufficient memory for setting up %s memory ranges\n",  in fadump_alloc_mem_ranges()
857 		       mrange_info->name);  in fadump_alloc_mem_ranges()
859 		return -ENOMEM;  in fadump_alloc_mem_ranges()
862 	mrange_info->mem_ranges = new_array;  in fadump_alloc_mem_ranges()
863 	mrange_info->mem_ranges_sz = new_size;  in fadump_alloc_mem_ranges()
864 	mrange_info->max_mem_ranges = (new_size /  in fadump_alloc_mem_ranges()
871 	struct fadump_memory_range *mem_ranges = mrange_info->mem_ranges;  in fadump_add_mem_range()
879 	 * Fold adjacent memory ranges to bring down the memory ranges/  in fadump_add_mem_range()
882 	if (mrange_info->mem_range_cnt) {  in fadump_add_mem_range()
883 		start = mem_ranges[mrange_info->mem_range_cnt - 1].base;  in fadump_add_mem_range()
884 		size  = mem_ranges[mrange_info->mem_range_cnt - 1].size;  in fadump_add_mem_range()
887 		 * Boot memory area needs separate PT_LOAD segment(s) as it  in fadump_add_mem_range()
889 		 * So, fold only if the region is not boot memory area.  in fadump_add_mem_range()
896 		if (mrange_info->mem_range_cnt == mrange_info->max_mem_ranges) {  in fadump_add_mem_range()
899 			if (mrange_info->is_static) {  in fadump_add_mem_range()
900 				pr_err("Reached array size limit for %s memory ranges\n",  in fadump_add_mem_range()
901 				       mrange_info->name);  in fadump_add_mem_range()
902 				return -ENOSPC;  in fadump_add_mem_range()
910 			mem_ranges = mrange_info->mem_ranges;  in fadump_add_mem_range()
914 		mem_ranges[mrange_info->mem_range_cnt].base = start;  in fadump_add_mem_range()
915 		mrange_info->mem_range_cnt++;  in fadump_add_mem_range()
918 	mem_ranges[mrange_info->mem_range_cnt - 1].size = (end - start);  in fadump_add_mem_range()
919 	pr_debug("%s_memory_range[%d] [%#016llx-%#016llx], %#llx bytes\n",  in fadump_add_mem_range()
920 		 mrange_info->name, (mrange_info->mem_range_cnt - 1),  in fadump_add_mem_range()
921 		 start, end - 1, (end - start));  in fadump_add_mem_range()
961 	memcpy(elf->e_ident, ELFMAG, SELFMAG);  in fadump_init_elfcore_header()
962 	elf->e_ident[EI_CLASS] = ELF_CLASS;  in fadump_init_elfcore_header()
963 	elf->e_ident[EI_DATA] = ELF_DATA;  in fadump_init_elfcore_header()
964 	elf->e_ident[EI_VERSION] = EV_CURRENT;  in fadump_init_elfcore_header()
965 	elf->e_ident[EI_OSABI] = ELF_OSABI;  in fadump_init_elfcore_header()
966 	memset(elf->e_ident+EI_PAD, 0, EI_NIDENT-EI_PAD);  in fadump_init_elfcore_header()
967 	elf->e_type = ET_CORE;  in fadump_init_elfcore_header()
968 	elf->e_machine = ELF_ARCH;  in fadump_init_elfcore_header()
969 	elf->e_version = EV_CURRENT;  in fadump_init_elfcore_header()
970 	elf->e_entry = 0;  in fadump_init_elfcore_header()
971 	elf->e_phoff = sizeof(struct elfhdr);  in fadump_init_elfcore_header()
972 	elf->e_shoff = 0;  in fadump_init_elfcore_header()
975 		elf->e_flags = 2;  in fadump_init_elfcore_header()
977 		elf->e_flags = 1;  in fadump_init_elfcore_header()
979 		elf->e_flags = 0;  in fadump_init_elfcore_header()
981 	elf->e_ehsize = sizeof(struct elfhdr);  in fadump_init_elfcore_header()
982 	elf->e_phentsize = sizeof(struct elf_phdr);  in fadump_init_elfcore_header()
983 	elf->e_phnum = 0;  in fadump_init_elfcore_header()
984 	elf->e_shentsize = 0;  in fadump_init_elfcore_header()
985 	elf->e_shnum = 0;  in fadump_init_elfcore_header()
986 	elf->e_shstrndx = 0;  in fadump_init_elfcore_header()
992  * Traverse through memblock structure and setup crash memory ranges. These
1000 	pr_debug("Setup crash memory ranges.\n");  in fadump_setup_crash_memory_ranges()
1004 	 * Boot memory region(s) registered with firmware are moved to  in fadump_setup_crash_memory_ranges()
1006 	 * header(s) for this memory chunk(s) with the correct offset.  in fadump_setup_crash_memory_ranges()
1018 		 * skip the memory chunk that is already added  in fadump_setup_crash_memory_ranges()
1028 		/* add this range excluding the reserved dump area. */  in fadump_setup_crash_memory_ranges()
1038  * If the given physical address falls within the boot memory region then
1039  * return the relocated address that points to the dump region reserved
1040  * for saving initial boot memory contents.
1053 		hole_size += (rstart - rlast);  in fadump_relocate()
1056 			raddr += fw_dump.boot_mem_dest_addr - hole_size;  in fadump_relocate()
1087 	phdr->p_type = PT_NOTE;  in fadump_create_elfcore_headers()
1088 	phdr->p_flags = 0;  in fadump_create_elfcore_headers()
1089 	phdr->p_vaddr = 0;  in fadump_create_elfcore_headers()
1090 	phdr->p_align = 0;  in fadump_create_elfcore_headers()
1092 	phdr->p_offset = 0;  in fadump_create_elfcore_headers()
1093 	phdr->p_paddr = 0;  in fadump_create_elfcore_headers()
1094 	phdr->p_filesz = 0;  in fadump_create_elfcore_headers()
1095 	phdr->p_memsz = 0;  in fadump_create_elfcore_headers()
1097 	(elf->e_phnum)++;  in fadump_create_elfcore_headers()
1102 	phdr->p_type	= PT_NOTE;  in fadump_create_elfcore_headers()
1103 	phdr->p_flags	= 0;  in fadump_create_elfcore_headers()
1104 	phdr->p_vaddr	= 0;  in fadump_create_elfcore_headers()
1105 	phdr->p_align	= 0;  in fadump_create_elfcore_headers()
1107 	phdr->p_paddr	= fadump_relocate(paddr_vmcoreinfo_note());  in fadump_create_elfcore_headers()
1108 	phdr->p_offset	= phdr->p_paddr;  in fadump_create_elfcore_headers()
1109 	phdr->p_memsz	= phdr->p_filesz = VMCOREINFO_NOTE_SIZE;  in fadump_create_elfcore_headers()
1112 	(elf->e_phnum)++;  in fadump_create_elfcore_headers()
1128 		phdr->p_type	= PT_LOAD;  in fadump_create_elfcore_headers()
1129 		phdr->p_flags	= PF_R|PF_W|PF_X;  in fadump_create_elfcore_headers()
1130 		phdr->p_offset	= mbase;  in fadump_create_elfcore_headers()
1134 			 * The entire real memory region will be moved by  in fadump_create_elfcore_headers()
1138 			phdr->p_offset = fw_dump.boot_mem_dest_addr + offset;  in fadump_create_elfcore_headers()
1139 			if (j < (fw_dump.boot_mem_regs_cnt - 1)) {  in fadump_create_elfcore_headers()
1145 		phdr->p_paddr = mbase;  in fadump_create_elfcore_headers()
1146 		phdr->p_vaddr = (unsigned long)__va(mbase);  in fadump_create_elfcore_headers()
1147 		phdr->p_filesz = msize;  in fadump_create_elfcore_headers()
1148 		phdr->p_memsz = msize;  in fadump_create_elfcore_headers()
1149 		phdr->p_align = 0;  in fadump_create_elfcore_headers()
1152 		(elf->e_phnum)++;  in fadump_create_elfcore_headers()
1168 	fdh->magic_number = FADUMP_CRASH_INFO_MAGIC;  in init_fadump_header()
1169 	fdh->elfcorehdr_addr = addr;  in init_fadump_header()
1171 	fdh->crashing_cpu = FADUMP_CPU_UNKNOWN;  in init_fadump_header()
1176 	fdh->cpu_mask = *cpu_possible_mask;  in init_fadump_header()
1188 	 * If no memory is reserved then we can not register for firmware-  in register_fadump()
1192 		return -ENODEV;  in register_fadump()
1208 	pr_debug("Registering for firmware-assisted kernel dump...\n");  in register_fadump()
1209 	return fw_dump.ops->fadump_register(&fw_dump);  in register_fadump()
1219 		pr_debug("Invalidating firmware-assisted dump registration\n");  in fadump_cleanup()
1220 		fw_dump.ops->fadump_invalidate(&fw_dump);  in fadump_cleanup()
1222 		/* Un-register Firmware-assisted dump if it was registered. */  in fadump_cleanup()
1223 		fw_dump.ops->fadump_unregister(&fw_dump);  in fadump_cleanup()
1227 	if (fw_dump.ops->fadump_cleanup)  in fadump_cleanup()
1228 		fw_dump.ops->fadump_cleanup(&fw_dump);  in fadump_cleanup()
1237 	pr_info("freeing reserved memory (0x%llx - 0x%llx)\n",  in fadump_free_reserved_memory()
1251  * Skip memory holes and free memory that was actually reserved.
1278  * Sort the mem ranges in-place and merge adjacent ranges
1279  * to minimize the memory ranges count.
1290 	/* Sort the memory ranges */  in sort_and_merge_mem_ranges()
1291 	mem_ranges = mrange_info->mem_ranges;  in sort_and_merge_mem_ranges()
1292 	for (i = 0; i < mrange_info->mem_range_cnt; i++) {  in sort_and_merge_mem_ranges()
1294 		for (j = (i + 1); j < mrange_info->mem_range_cnt; j++) {  in sort_and_merge_mem_ranges()
1302 	/* Merge adjacent reserved ranges */  in sort_and_merge_mem_ranges()
1304 	for (i = 1; i < mrange_info->mem_range_cnt; i++) {  in sort_and_merge_mem_ranges()
1305 		base = mem_ranges[i-1].base;  in sort_and_merge_mem_ranges()
1306 		size = mem_ranges[i-1].size;  in sort_and_merge_mem_ranges()
1317 	mrange_info->mem_range_cnt = idx + 1;  in sort_and_merge_mem_ranges()
1321  * Scan reserved-ranges to consider them while reserving/releasing
1322  * memory for FADump.
1327 	int len, ret = -1;  in early_init_dt_scan_reserved_ranges()
1330 	/* reserved-ranges already scanned */  in early_init_dt_scan_reserved_ranges()
1334 	prop = of_get_flat_dt_prop(node, "reserved-ranges", &len);  in early_init_dt_scan_reserved_ranges()
1339 	 * Each reserved range is an (address,size) pair, 2 cells each,  in early_init_dt_scan_reserved_ranges()
1352 				pr_warn("some reserved ranges are ignored!\n");  in early_init_dt_scan_reserved_ranges()
1358 	/* Compact reserved ranges */  in early_init_dt_scan_reserved_ranges()
1363  * Release the memory that was reserved during early boot to preserve the
1364  * crash'ed kernel's memory contents except reserved dump area (permanent
1365  * reservation) and reserved ranges used by F/W. The released memory will
1377 	 * If reserved ranges array limit is hit, overwrite the last reserved  in fadump_release_memory()
1378 	 * memory range with reserved dump area to ensure it is excluded from  in fadump_release_memory()
1379 	 * the memory being released (reused for next FADump registration).  in fadump_release_memory()
1383 		reserved_mrange_info.mem_range_cnt--;  in fadump_release_memory()
1389 	/* Get the reserved ranges list in order first. */  in fadump_release_memory()
1392 	/* Exclude reserved ranges and release remaining memory */  in fadump_release_memory()
1426 	 * memory structure for FADump re-registration.  in fadump_invalidate_release_mem()
1428 	if (fw_dump.ops->fadump_setup_metadata &&  in fadump_invalidate_release_mem()
1429 	    (fw_dump.ops->fadump_setup_metadata(&fw_dump) < 0))  in fadump_invalidate_release_mem()
1431 	fw_dump.ops->fadump_init_mem_struct(&fw_dump);  in fadump_invalidate_release_mem()
1438 	int input = -1;  in release_mem_store()
1441 		return -EPERM;  in release_mem_store()
1444 		return -EINVAL;  in release_mem_store()
1449 		 * memory, hence it will not be valid anymore.  in release_mem_store()
1457 		return -EINVAL;  in release_mem_store()
1461 /* Release the reserved memory and disable the FADump */
1497 	int input = -1;  in registered_store()
1500 		return -EPERM;  in registered_store()
1503 		return -EINVAL;  in registered_store()
1513 		/* Un-register Firmware-assisted dump */  in registered_store()
1514 		pr_debug("Un-register firmware-assisted dump\n");  in registered_store()
1515 		fw_dump.ops->fadump_unregister(&fw_dump);  in registered_store()
1519 			/* Un-register Firmware-assisted dump */  in registered_store()
1520 			fw_dump.ops->fadump_unregister(&fw_dump);  in registered_store()
1522 		/* Register Firmware-assisted dump */  in registered_store()
1526 		ret = -EINVAL;  in registered_store()
1541 	fw_dump.ops->fadump_region_show(&fw_dump, m);  in fadump_region_show()
1594 	 *      - fadump_enabled -> fadump/enabled  in fadump_init_files()
1595 	 *      - fadump_registered -> fadump/registered  in fadump_init_files()
1596 	 *      - fadump_release_mem -> fadump/release_mem  in fadump_init_files()
1627  * Prepare for firmware-assisted dump.
1647 		 * and release memory before proceeding for re-registration.  in setup_fadump()
1649 		if (fw_dump.ops->fadump_process(&fw_dump) < 0)  in setup_fadump()
1652 	/* Initialize the kernel dump memory structure and register with f/w */  in setup_fadump()
1654 		fw_dump.ops->fadump_init_mem_struct(&fw_dump);  in setup_fadump()
1689  * preserve crash data. The subsequent memory preserving kernel boot
1696 		 * If last boot has crashed then reserve all the memory  in fadump_reserve_mem()
1697 		 * above boot memory to preserve crash data.  in fadump_reserve_mem()
1702 		pr_debug("FADump-aware kernel..\n");  in fadump_reserve_mem()
1714 		msize  = mend - mstart;  in fadump_reserve_crash_area()
1720 			msize -= (base - mstart);  in fadump_reserve_crash_area()
1724 		pr_info("Reserving %lluMB of memory at %#016llx for preserving crash data",  in fadump_reserve_crash_area()