1caab277bSThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 2f27bb139SMarc Zyngier/* 3f27bb139SMarc Zyngier * Copyright (C) 2012 ARM Ltd. 4f27bb139SMarc Zyngier */ 5f27bb139SMarc Zyngier 6f27bb139SMarc Zyngier#include <linux/linkage.h> 7f27bb139SMarc Zyngier#include <linux/const.h> 8f27bb139SMarc Zyngier#include <asm/assembler.h> 9f27bb139SMarc Zyngier#include <asm/page.h> 1060e0a09dSAndrew Pinski#include <asm/cpufeature.h> 1160e0a09dSAndrew Pinski#include <asm/alternative.h> 12f27bb139SMarc Zyngier 13f27bb139SMarc Zyngier/* 14f27bb139SMarc Zyngier * Copy a page from src to dest (both are page aligned) 15f27bb139SMarc Zyngier * 16f27bb139SMarc Zyngier * Parameters: 17f27bb139SMarc Zyngier * x0 - dest 18f27bb139SMarc Zyngier * x1 - src 19f27bb139SMarc Zyngier */ 20f27bb139SMarc ZyngierENTRY(copy_page) 216ba3b554SMark Rutlandalternative_if ARM64_HAS_NO_HW_PREFETCH 22288be97cSArd Biesheuvel // Prefetch three cache lines ahead. 2360e0a09dSAndrew Pinski prfm pldl1strm, [x1, #128] 2460e0a09dSAndrew Pinski prfm pldl1strm, [x1, #256] 25288be97cSArd Biesheuvel prfm pldl1strm, [x1, #384] 266ba3b554SMark Rutlandalternative_else_nop_endif 2760e0a09dSAndrew Pinski 28223e23e8SWill Deacon ldp x2, x3, [x1] 29f27bb139SMarc Zyngier ldp x4, x5, [x1, #16] 30f27bb139SMarc Zyngier ldp x6, x7, [x1, #32] 31f27bb139SMarc Zyngier ldp x8, x9, [x1, #48] 32223e23e8SWill Deacon ldp x10, x11, [x1, #64] 33223e23e8SWill Deacon ldp x12, x13, [x1, #80] 34223e23e8SWill Deacon ldp x14, x15, [x1, #96] 35223e23e8SWill Deacon ldp x16, x17, [x1, #112] 36223e23e8SWill Deacon 37223e23e8SWill Deacon mov x18, #(PAGE_SIZE - 128) 38223e23e8SWill Deacon add x1, x1, #128 39223e23e8SWill Deacon1: 40223e23e8SWill Deacon subs x18, x18, #128 41223e23e8SWill Deacon 426ba3b554SMark Rutlandalternative_if ARM64_HAS_NO_HW_PREFETCH 4360e0a09dSAndrew Pinski prfm pldl1strm, [x1, #384] 446ba3b554SMark Rutlandalternative_else_nop_endif 4560e0a09dSAndrew Pinski 46223e23e8SWill Deacon stnp x2, x3, [x0] 47223e23e8SWill Deacon ldp x2, x3, [x1] 48223e23e8SWill Deacon stnp x4, x5, [x0, #16] 49223e23e8SWill Deacon ldp x4, x5, [x1, #16] 50223e23e8SWill Deacon stnp x6, x7, [x0, #32] 51223e23e8SWill Deacon ldp x6, x7, [x1, #32] 52223e23e8SWill Deacon stnp x8, x9, [x0, #48] 53223e23e8SWill Deacon ldp x8, x9, [x1, #48] 54223e23e8SWill Deacon stnp x10, x11, [x0, #64] 55223e23e8SWill Deacon ldp x10, x11, [x1, #64] 56223e23e8SWill Deacon stnp x12, x13, [x0, #80] 57223e23e8SWill Deacon ldp x12, x13, [x1, #80] 58223e23e8SWill Deacon stnp x14, x15, [x0, #96] 59223e23e8SWill Deacon ldp x14, x15, [x1, #96] 60223e23e8SWill Deacon stnp x16, x17, [x0, #112] 61223e23e8SWill Deacon ldp x16, x17, [x1, #112] 62223e23e8SWill Deacon 63223e23e8SWill Deacon add x0, x0, #128 64223e23e8SWill Deacon add x1, x1, #128 65223e23e8SWill Deacon 66223e23e8SWill Deacon b.gt 1b 67223e23e8SWill Deacon 68f27bb139SMarc Zyngier stnp x2, x3, [x0] 69f27bb139SMarc Zyngier stnp x4, x5, [x0, #16] 70f27bb139SMarc Zyngier stnp x6, x7, [x0, #32] 71f27bb139SMarc Zyngier stnp x8, x9, [x0, #48] 72223e23e8SWill Deacon stnp x10, x11, [x0, #64] 73223e23e8SWill Deacon stnp x12, x13, [x0, #80] 74223e23e8SWill Deacon stnp x14, x15, [x0, #96] 75223e23e8SWill Deacon stnp x16, x17, [x0, #112] 76223e23e8SWill Deacon 77f27bb139SMarc Zyngier ret 78f27bb139SMarc ZyngierENDPROC(copy_page) 7950fdecb2SMark RutlandEXPORT_SYMBOL(copy_page) 80