1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6#include <asm/asm-offsets.h> 7#include <asm/asm.h> 8#include <linux/init.h> 9#include <linux/linkage.h> 10#include <asm/thread_info.h> 11#include <asm/page.h> 12#include <asm/pgtable.h> 13#include <asm/csr.h> 14#include <asm/hwcap.h> 15#include <asm/image.h> 16#include "efi-header.S" 17 18#ifdef CONFIG_XIP_KERNEL 19.macro XIP_FIXUP_OFFSET reg 20 REG_L t0, _xip_fixup 21 add \reg, \reg, t0 22.endm 23_xip_fixup: .dword CONFIG_PHYS_RAM_BASE - CONFIG_XIP_PHYS_ADDR - XIP_OFFSET 24#else 25.macro XIP_FIXUP_OFFSET reg 26.endm 27#endif /* CONFIG_XIP_KERNEL */ 28 29__HEAD 30ENTRY(_start) 31 /* 32 * Image header expected by Linux boot-loaders. The image header data 33 * structure is described in asm/image.h. 34 * Do not modify it without modifying the structure and all bootloaders 35 * that expects this header format!! 36 */ 37#ifdef CONFIG_EFI 38 /* 39 * This instruction decodes to "MZ" ASCII required by UEFI. 40 */ 41 c.li s4,-13 42 j _start_kernel 43#else 44 /* jump to start kernel */ 45 j _start_kernel 46 /* reserved */ 47 .word 0 48#endif 49 .balign 8 50#ifdef CONFIG_RISCV_M_MODE 51 /* Image load offset (0MB) from start of RAM for M-mode */ 52 .dword 0 53#else 54#if __riscv_xlen == 64 55 /* Image load offset(2MB) from start of RAM */ 56 .dword 0x200000 57#else 58 /* Image load offset(4MB) from start of RAM */ 59 .dword 0x400000 60#endif 61#endif 62 /* Effective size of kernel image */ 63 .dword _end - _start 64 .dword __HEAD_FLAGS 65 .word RISCV_HEADER_VERSION 66 .word 0 67 .dword 0 68 .ascii RISCV_IMAGE_MAGIC 69 .balign 4 70 .ascii RISCV_IMAGE_MAGIC2 71#ifdef CONFIG_EFI 72 .word pe_head_start - _start 73pe_head_start: 74 75 __EFI_PE_HEADER 76#else 77 .word 0 78#endif 79 80.align 2 81#ifdef CONFIG_MMU 82relocate: 83 /* Relocate return address */ 84 la a1, kernel_map 85 XIP_FIXUP_OFFSET a1 86 REG_L a1, KERNEL_MAP_VIRT_ADDR(a1) 87 la a2, _start 88 sub a1, a1, a2 89 add ra, ra, a1 90 91 /* Point stvec to virtual address of intruction after satp write */ 92 la a2, 1f 93 add a2, a2, a1 94 csrw CSR_TVEC, a2 95 96 /* Compute satp for kernel page tables, but don't load it yet */ 97 srl a2, a0, PAGE_SHIFT 98 li a1, SATP_MODE 99 or a2, a2, a1 100 101 /* 102 * Load trampoline page directory, which will cause us to trap to 103 * stvec if VA != PA, or simply fall through if VA == PA. We need a 104 * full fence here because setup_vm() just wrote these PTEs and we need 105 * to ensure the new translations are in use. 106 */ 107 la a0, trampoline_pg_dir 108 XIP_FIXUP_OFFSET a0 109 srl a0, a0, PAGE_SHIFT 110 or a0, a0, a1 111 sfence.vma 112 csrw CSR_SATP, a0 113.align 2 1141: 115 /* Set trap vector to spin forever to help debug */ 116 la a0, .Lsecondary_park 117 csrw CSR_TVEC, a0 118 119 /* Reload the global pointer */ 120.option push 121.option norelax 122 la gp, __global_pointer$ 123.option pop 124 125 /* 126 * Switch to kernel page tables. A full fence is necessary in order to 127 * avoid using the trampoline translations, which are only correct for 128 * the first superpage. Fetching the fence is guarnteed to work 129 * because that first superpage is translated the same way. 130 */ 131 csrw CSR_SATP, a2 132 sfence.vma 133 134 ret 135#endif /* CONFIG_MMU */ 136#ifdef CONFIG_SMP 137 .global secondary_start_sbi 138secondary_start_sbi: 139 /* Mask all interrupts */ 140 csrw CSR_IE, zero 141 csrw CSR_IP, zero 142 143 /* Load the global pointer */ 144 .option push 145 .option norelax 146 la gp, __global_pointer$ 147 .option pop 148 149 /* 150 * Disable FPU to detect illegal usage of 151 * floating point in kernel space 152 */ 153 li t0, SR_FS 154 csrc CSR_STATUS, t0 155 156 /* Set trap vector to spin forever to help debug */ 157 la a3, .Lsecondary_park 158 csrw CSR_TVEC, a3 159 160 slli a3, a0, LGREG 161 la a4, __cpu_up_stack_pointer 162 XIP_FIXUP_OFFSET a4 163 la a5, __cpu_up_task_pointer 164 XIP_FIXUP_OFFSET a5 165 add a4, a3, a4 166 add a5, a3, a5 167 REG_L sp, (a4) 168 REG_L tp, (a5) 169 170 .global secondary_start_common 171secondary_start_common: 172 173#ifdef CONFIG_MMU 174 /* Enable virtual memory and relocate to virtual address */ 175 la a0, swapper_pg_dir 176 XIP_FIXUP_OFFSET a0 177 call relocate 178#endif 179 call setup_trap_vector 180 tail smp_callin 181#endif /* CONFIG_SMP */ 182 183.align 2 184setup_trap_vector: 185 /* Set trap vector to exception handler */ 186 la a0, handle_exception 187 csrw CSR_TVEC, a0 188 189 /* 190 * Set sup0 scratch register to 0, indicating to exception vector that 191 * we are presently executing in kernel. 192 */ 193 csrw CSR_SCRATCH, zero 194 ret 195 196.align 2 197.Lsecondary_park: 198 /* We lack SMP support or have too many harts, so park this hart */ 199 wfi 200 j .Lsecondary_park 201 202END(_start) 203 204ENTRY(_start_kernel) 205 /* Mask all interrupts */ 206 csrw CSR_IE, zero 207 csrw CSR_IP, zero 208 209#ifdef CONFIG_RISCV_M_MODE 210 /* flush the instruction cache */ 211 fence.i 212 213 /* Reset all registers except ra, a0, a1 */ 214 call reset_regs 215 216 /* 217 * Setup a PMP to permit access to all of memory. Some machines may 218 * not implement PMPs, so we set up a quick trap handler to just skip 219 * touching the PMPs on any trap. 220 */ 221 la a0, pmp_done 222 csrw CSR_TVEC, a0 223 224 li a0, -1 225 csrw CSR_PMPADDR0, a0 226 li a0, (PMP_A_NAPOT | PMP_R | PMP_W | PMP_X) 227 csrw CSR_PMPCFG0, a0 228.align 2 229pmp_done: 230 231 /* 232 * The hartid in a0 is expected later on, and we have no firmware 233 * to hand it to us. 234 */ 235 csrr a0, CSR_MHARTID 236#endif /* CONFIG_RISCV_M_MODE */ 237 238 /* Load the global pointer */ 239.option push 240.option norelax 241 la gp, __global_pointer$ 242.option pop 243 244 /* 245 * Disable FPU to detect illegal usage of 246 * floating point in kernel space 247 */ 248 li t0, SR_FS 249 csrc CSR_STATUS, t0 250 251#ifdef CONFIG_SMP 252 li t0, CONFIG_NR_CPUS 253 blt a0, t0, .Lgood_cores 254 tail .Lsecondary_park 255.Lgood_cores: 256#endif 257 258#ifndef CONFIG_XIP_KERNEL 259 /* Pick one hart to run the main boot sequence */ 260 la a3, hart_lottery 261 li a2, 1 262 amoadd.w a3, a2, (a3) 263 bnez a3, .Lsecondary_start 264 265#else 266 /* hart_lottery in flash contains a magic number */ 267 la a3, hart_lottery 268 mv a2, a3 269 XIP_FIXUP_OFFSET a2 270 lw t1, (a3) 271 amoswap.w t0, t1, (a2) 272 /* first time here if hart_lottery in RAM is not set */ 273 beq t0, t1, .Lsecondary_start 274 275 la sp, _end + THREAD_SIZE 276 XIP_FIXUP_OFFSET sp 277 mv s0, a0 278 call __copy_data 279 280 /* Restore a0 copy */ 281 mv a0, s0 282#endif 283 284#ifndef CONFIG_XIP_KERNEL 285 /* Clear BSS for flat non-ELF images */ 286 la a3, __bss_start 287 la a4, __bss_stop 288 ble a4, a3, clear_bss_done 289clear_bss: 290 REG_S zero, (a3) 291 add a3, a3, RISCV_SZPTR 292 blt a3, a4, clear_bss 293clear_bss_done: 294#endif 295 /* Save hart ID and DTB physical address */ 296 mv s0, a0 297 mv s1, a1 298 299 la a2, boot_cpu_hartid 300 XIP_FIXUP_OFFSET a2 301 REG_S a0, (a2) 302 303 /* Initialize page tables and relocate to virtual addresses */ 304 la tp, init_task 305 la sp, init_thread_union + THREAD_SIZE 306 XIP_FIXUP_OFFSET sp 307#ifdef CONFIG_BUILTIN_DTB 308 la a0, __dtb_start 309#else 310 mv a0, s1 311#endif /* CONFIG_BUILTIN_DTB */ 312 call setup_vm 313#ifdef CONFIG_MMU 314 la a0, early_pg_dir 315 XIP_FIXUP_OFFSET a0 316 call relocate 317#endif /* CONFIG_MMU */ 318 319 call setup_trap_vector 320 /* Restore C environment */ 321 la tp, init_task 322 sw zero, TASK_TI_CPU(tp) 323 la sp, init_thread_union + THREAD_SIZE 324 325#ifdef CONFIG_KASAN 326 call kasan_early_init 327#endif 328 /* Start the kernel */ 329 call soc_early_init 330 tail start_kernel 331 332.Lsecondary_start: 333#ifdef CONFIG_SMP 334 /* Set trap vector to spin forever to help debug */ 335 la a3, .Lsecondary_park 336 csrw CSR_TVEC, a3 337 338 slli a3, a0, LGREG 339 la a1, __cpu_up_stack_pointer 340 XIP_FIXUP_OFFSET a1 341 la a2, __cpu_up_task_pointer 342 XIP_FIXUP_OFFSET a2 343 add a1, a3, a1 344 add a2, a3, a2 345 346 /* 347 * This hart didn't win the lottery, so we wait for the winning hart to 348 * get far enough along the boot process that it should continue. 349 */ 350.Lwait_for_cpu_up: 351 /* FIXME: We should WFI to save some energy here. */ 352 REG_L sp, (a1) 353 REG_L tp, (a2) 354 beqz sp, .Lwait_for_cpu_up 355 beqz tp, .Lwait_for_cpu_up 356 fence 357 358 tail secondary_start_common 359#endif 360 361END(_start_kernel) 362 363#ifdef CONFIG_RISCV_M_MODE 364ENTRY(reset_regs) 365 li sp, 0 366 li gp, 0 367 li tp, 0 368 li t0, 0 369 li t1, 0 370 li t2, 0 371 li s0, 0 372 li s1, 0 373 li a2, 0 374 li a3, 0 375 li a4, 0 376 li a5, 0 377 li a6, 0 378 li a7, 0 379 li s2, 0 380 li s3, 0 381 li s4, 0 382 li s5, 0 383 li s6, 0 384 li s7, 0 385 li s8, 0 386 li s9, 0 387 li s10, 0 388 li s11, 0 389 li t3, 0 390 li t4, 0 391 li t5, 0 392 li t6, 0 393 csrw CSR_SCRATCH, 0 394 395#ifdef CONFIG_FPU 396 csrr t0, CSR_MISA 397 andi t0, t0, (COMPAT_HWCAP_ISA_F | COMPAT_HWCAP_ISA_D) 398 beqz t0, .Lreset_regs_done 399 400 li t1, SR_FS 401 csrs CSR_STATUS, t1 402 fmv.s.x f0, zero 403 fmv.s.x f1, zero 404 fmv.s.x f2, zero 405 fmv.s.x f3, zero 406 fmv.s.x f4, zero 407 fmv.s.x f5, zero 408 fmv.s.x f6, zero 409 fmv.s.x f7, zero 410 fmv.s.x f8, zero 411 fmv.s.x f9, zero 412 fmv.s.x f10, zero 413 fmv.s.x f11, zero 414 fmv.s.x f12, zero 415 fmv.s.x f13, zero 416 fmv.s.x f14, zero 417 fmv.s.x f15, zero 418 fmv.s.x f16, zero 419 fmv.s.x f17, zero 420 fmv.s.x f18, zero 421 fmv.s.x f19, zero 422 fmv.s.x f20, zero 423 fmv.s.x f21, zero 424 fmv.s.x f22, zero 425 fmv.s.x f23, zero 426 fmv.s.x f24, zero 427 fmv.s.x f25, zero 428 fmv.s.x f26, zero 429 fmv.s.x f27, zero 430 fmv.s.x f28, zero 431 fmv.s.x f29, zero 432 fmv.s.x f30, zero 433 fmv.s.x f31, zero 434 csrw fcsr, 0 435 /* note that the caller must clear SR_FS */ 436#endif /* CONFIG_FPU */ 437.Lreset_regs_done: 438 ret 439END(reset_regs) 440#endif /* CONFIG_RISCV_M_MODE */ 441 442__PAGE_ALIGNED_BSS 443 /* Empty zero page */ 444 .balign PAGE_SIZE 445