1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Copyright (C) 2012 ARM Ltd. 4 */ 5 6#include <linux/linkage.h> 7#include <linux/const.h> 8#include <asm/assembler.h> 9#include <asm/page.h> 10#include <asm/cpufeature.h> 11#include <asm/alternative.h> 12 13/* 14 * Copy a page from src to dest (both are page aligned) 15 * 16 * Parameters: 17 * x0 - dest 18 * x1 - src 19 */ 20SYM_FUNC_START(copy_page) 21alternative_if ARM64_HAS_NO_HW_PREFETCH 22 // Prefetch three cache lines ahead. 23 prfm pldl1strm, [x1, #128] 24 prfm pldl1strm, [x1, #256] 25 prfm pldl1strm, [x1, #384] 26alternative_else_nop_endif 27 28 ldp x2, x3, [x1] 29 ldp x4, x5, [x1, #16] 30 ldp x6, x7, [x1, #32] 31 ldp x8, x9, [x1, #48] 32 ldp x10, x11, [x1, #64] 33 ldp x12, x13, [x1, #80] 34 ldp x14, x15, [x1, #96] 35 ldp x16, x17, [x1, #112] 36 37 mov x18, #(PAGE_SIZE - 128) 38 add x1, x1, #128 391: 40 subs x18, x18, #128 41 42alternative_if ARM64_HAS_NO_HW_PREFETCH 43 prfm pldl1strm, [x1, #384] 44alternative_else_nop_endif 45 46 stnp x2, x3, [x0] 47 ldp x2, x3, [x1] 48 stnp x4, x5, [x0, #16] 49 ldp x4, x5, [x1, #16] 50 stnp x6, x7, [x0, #32] 51 ldp x6, x7, [x1, #32] 52 stnp x8, x9, [x0, #48] 53 ldp x8, x9, [x1, #48] 54 stnp x10, x11, [x0, #64] 55 ldp x10, x11, [x1, #64] 56 stnp x12, x13, [x0, #80] 57 ldp x12, x13, [x1, #80] 58 stnp x14, x15, [x0, #96] 59 ldp x14, x15, [x1, #96] 60 stnp x16, x17, [x0, #112] 61 ldp x16, x17, [x1, #112] 62 63 add x0, x0, #128 64 add x1, x1, #128 65 66 b.gt 1b 67 68 stnp x2, x3, [x0] 69 stnp x4, x5, [x0, #16] 70 stnp x6, x7, [x0, #32] 71 stnp x8, x9, [x0, #48] 72 stnp x10, x11, [x0, #64] 73 stnp x12, x13, [x0, #80] 74 stnp x14, x15, [x0, #96] 75 stnp x16, x17, [x0, #112] 76 77 ret 78SYM_FUNC_END(copy_page) 79EXPORT_SYMBOL(copy_page) 80