1/* 2 * Copyright (C) 2012 ARM Ltd. 3 * 4 * This program is free software; you can redistribute it and/or modify 5 * it under the terms of the GNU General Public License version 2 as 6 * published by the Free Software Foundation. 7 * 8 * This program is distributed in the hope that it will be useful, 9 * but WITHOUT ANY WARRANTY; without even the implied warranty of 10 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 11 * GNU General Public License for more details. 12 * 13 * You should have received a copy of the GNU General Public License 14 * along with this program. If not, see <http://www.gnu.org/licenses/>. 15 */ 16 17#include <linux/linkage.h> 18#include <linux/const.h> 19#include <asm/assembler.h> 20#include <asm/page.h> 21#include <asm/cpufeature.h> 22#include <asm/alternative.h> 23 24/* 25 * Copy a page from src to dest (both are page aligned) 26 * 27 * Parameters: 28 * x0 - dest 29 * x1 - src 30 */ 31ENTRY(copy_page) 32alternative_if ARM64_HAS_NO_HW_PREFETCH 33 // Prefetch three cache lines ahead. 34 prfm pldl1strm, [x1, #128] 35 prfm pldl1strm, [x1, #256] 36 prfm pldl1strm, [x1, #384] 37alternative_else_nop_endif 38 39 ldp x2, x3, [x1] 40 ldp x4, x5, [x1, #16] 41 ldp x6, x7, [x1, #32] 42 ldp x8, x9, [x1, #48] 43 ldp x10, x11, [x1, #64] 44 ldp x12, x13, [x1, #80] 45 ldp x14, x15, [x1, #96] 46 ldp x16, x17, [x1, #112] 47 48 mov x18, #(PAGE_SIZE - 128) 49 add x1, x1, #128 501: 51 subs x18, x18, #128 52 53alternative_if ARM64_HAS_NO_HW_PREFETCH 54 prfm pldl1strm, [x1, #384] 55alternative_else_nop_endif 56 57 stnp x2, x3, [x0] 58 ldp x2, x3, [x1] 59 stnp x4, x5, [x0, #16] 60 ldp x4, x5, [x1, #16] 61 stnp x6, x7, [x0, #32] 62 ldp x6, x7, [x1, #32] 63 stnp x8, x9, [x0, #48] 64 ldp x8, x9, [x1, #48] 65 stnp x10, x11, [x0, #64] 66 ldp x10, x11, [x1, #64] 67 stnp x12, x13, [x0, #80] 68 ldp x12, x13, [x1, #80] 69 stnp x14, x15, [x0, #96] 70 ldp x14, x15, [x1, #96] 71 stnp x16, x17, [x0, #112] 72 ldp x16, x17, [x1, #112] 73 74 add x0, x0, #128 75 add x1, x1, #128 76 77 b.gt 1b 78 79 stnp x2, x3, [x0] 80 stnp x4, x5, [x0, #16] 81 stnp x6, x7, [x0, #32] 82 stnp x8, x9, [x0, #48] 83 stnp x10, x11, [x0, #64] 84 stnp x12, x13, [x0, #80] 85 stnp x14, x15, [x0, #96] 86 stnp x16, x17, [x0, #112] 87 88 ret 89ENDPROC(copy_page) 90EXPORT_SYMBOL(copy_page) 91