1caab277bSThomas Gleixner/* SPDX-License-Identifier: GPL-2.0-only */ 2f27bb139SMarc Zyngier/* 3f27bb139SMarc Zyngier * Copyright (C) 2012 ARM Ltd. 4f27bb139SMarc Zyngier */ 5f27bb139SMarc Zyngier 6f27bb139SMarc Zyngier#include <linux/linkage.h> 7f27bb139SMarc Zyngier#include <linux/const.h> 8f27bb139SMarc Zyngier#include <asm/assembler.h> 9f27bb139SMarc Zyngier#include <asm/page.h> 1060e0a09dSAndrew Pinski#include <asm/cpufeature.h> 1160e0a09dSAndrew Pinski#include <asm/alternative.h> 12f27bb139SMarc Zyngier 13f27bb139SMarc Zyngier/* 14f27bb139SMarc Zyngier * Copy a page from src to dest (both are page aligned) 15f27bb139SMarc Zyngier * 16f27bb139SMarc Zyngier * Parameters: 17f27bb139SMarc Zyngier * x0 - dest 18f27bb139SMarc Zyngier * x1 - src 19f27bb139SMarc Zyngier */ 20*0f61f6beSMark RutlandSYM_FUNC_START(__pi_copy_page) 216ba3b554SMark Rutlandalternative_if ARM64_HAS_NO_HW_PREFETCH 22288be97cSArd Biesheuvel // Prefetch three cache lines ahead. 2360e0a09dSAndrew Pinski prfm pldl1strm, [x1, #128] 2460e0a09dSAndrew Pinski prfm pldl1strm, [x1, #256] 25288be97cSArd Biesheuvel prfm pldl1strm, [x1, #384] 266ba3b554SMark Rutlandalternative_else_nop_endif 2760e0a09dSAndrew Pinski 28223e23e8SWill Deacon ldp x2, x3, [x1] 29f27bb139SMarc Zyngier ldp x4, x5, [x1, #16] 30f27bb139SMarc Zyngier ldp x6, x7, [x1, #32] 31f27bb139SMarc Zyngier ldp x8, x9, [x1, #48] 32223e23e8SWill Deacon ldp x10, x11, [x1, #64] 33223e23e8SWill Deacon ldp x12, x13, [x1, #80] 34223e23e8SWill Deacon ldp x14, x15, [x1, #96] 35223e23e8SWill Deacon ldp x16, x17, [x1, #112] 36223e23e8SWill Deacon 377f153ccbSArd Biesheuvel add x0, x0, #256 38223e23e8SWill Deacon add x1, x1, #128 39223e23e8SWill Deacon1: 407f153ccbSArd Biesheuvel tst x0, #(PAGE_SIZE - 1) 41223e23e8SWill Deacon 426ba3b554SMark Rutlandalternative_if ARM64_HAS_NO_HW_PREFETCH 4360e0a09dSAndrew Pinski prfm pldl1strm, [x1, #384] 446ba3b554SMark Rutlandalternative_else_nop_endif 4560e0a09dSAndrew Pinski 467f153ccbSArd Biesheuvel stnp x2, x3, [x0, #-256] 47223e23e8SWill Deacon ldp x2, x3, [x1] 487f153ccbSArd Biesheuvel stnp x4, x5, [x0, #16 - 256] 49223e23e8SWill Deacon ldp x4, x5, [x1, #16] 507f153ccbSArd Biesheuvel stnp x6, x7, [x0, #32 - 256] 51223e23e8SWill Deacon ldp x6, x7, [x1, #32] 527f153ccbSArd Biesheuvel stnp x8, x9, [x0, #48 - 256] 53223e23e8SWill Deacon ldp x8, x9, [x1, #48] 547f153ccbSArd Biesheuvel stnp x10, x11, [x0, #64 - 256] 55223e23e8SWill Deacon ldp x10, x11, [x1, #64] 567f153ccbSArd Biesheuvel stnp x12, x13, [x0, #80 - 256] 57223e23e8SWill Deacon ldp x12, x13, [x1, #80] 587f153ccbSArd Biesheuvel stnp x14, x15, [x0, #96 - 256] 59223e23e8SWill Deacon ldp x14, x15, [x1, #96] 607f153ccbSArd Biesheuvel stnp x16, x17, [x0, #112 - 256] 61223e23e8SWill Deacon ldp x16, x17, [x1, #112] 62223e23e8SWill Deacon 63223e23e8SWill Deacon add x0, x0, #128 64223e23e8SWill Deacon add x1, x1, #128 65223e23e8SWill Deacon 667f153ccbSArd Biesheuvel b.ne 1b 67223e23e8SWill Deacon 687f153ccbSArd Biesheuvel stnp x2, x3, [x0, #-256] 697f153ccbSArd Biesheuvel stnp x4, x5, [x0, #16 - 256] 707f153ccbSArd Biesheuvel stnp x6, x7, [x0, #32 - 256] 717f153ccbSArd Biesheuvel stnp x8, x9, [x0, #48 - 256] 727f153ccbSArd Biesheuvel stnp x10, x11, [x0, #64 - 256] 737f153ccbSArd Biesheuvel stnp x12, x13, [x0, #80 - 256] 747f153ccbSArd Biesheuvel stnp x14, x15, [x0, #96 - 256] 757f153ccbSArd Biesheuvel stnp x16, x17, [x0, #112 - 256] 76223e23e8SWill Deacon 77f27bb139SMarc Zyngier ret 78*0f61f6beSMark RutlandSYM_FUNC_END(__pi_copy_page) 79*0f61f6beSMark RutlandSYM_FUNC_ALIAS(copy_page, __pi_copy_page) 8050fdecb2SMark RutlandEXPORT_SYMBOL(copy_page) 81