1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Accelerated CRC32(C) using AArch64 CRC instructions 4 * 5 * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org> 6 */ 7 8#include <linux/linkage.h> 9#include <asm/alternative.h> 10#include <asm/assembler.h> 11 12 .cpu generic+crc 13 14 .macro __crc32, c 15 cmp x2, #16 16 b.lt 8f // less than 16 bytes 17 18 and x7, x2, #0x1f 19 and x2, x2, #~0x1f 20 cbz x7, 32f // multiple of 32 bytes 21 22 and x8, x7, #0xf 23 ldp x3, x4, [x1] 24 add x8, x8, x1 25 add x1, x1, x7 26 ldp x5, x6, [x8] 27CPU_BE( rev x3, x3 ) 28CPU_BE( rev x4, x4 ) 29CPU_BE( rev x5, x5 ) 30CPU_BE( rev x6, x6 ) 31 32 tst x7, #8 33 crc32\c\()x w8, w0, x3 34 csel x3, x3, x4, eq 35 csel w0, w0, w8, eq 36 tst x7, #4 37 lsr x4, x3, #32 38 crc32\c\()w w8, w0, w3 39 csel x3, x3, x4, eq 40 csel w0, w0, w8, eq 41 tst x7, #2 42 lsr w4, w3, #16 43 crc32\c\()h w8, w0, w3 44 csel w3, w3, w4, eq 45 csel w0, w0, w8, eq 46 tst x7, #1 47 crc32\c\()b w8, w0, w3 48 csel w0, w0, w8, eq 49 tst x7, #16 50 crc32\c\()x w8, w0, x5 51 crc32\c\()x w8, w8, x6 52 csel w0, w0, w8, eq 53 cbz x2, 0f 54 5532: ldp x3, x4, [x1], #32 56 sub x2, x2, #32 57 ldp x5, x6, [x1, #-16] 58CPU_BE( rev x3, x3 ) 59CPU_BE( rev x4, x4 ) 60CPU_BE( rev x5, x5 ) 61CPU_BE( rev x6, x6 ) 62 crc32\c\()x w0, w0, x3 63 crc32\c\()x w0, w0, x4 64 crc32\c\()x w0, w0, x5 65 crc32\c\()x w0, w0, x6 66 cbnz x2, 32b 670: ret 68 698: tbz x2, #3, 4f 70 ldr x3, [x1], #8 71CPU_BE( rev x3, x3 ) 72 crc32\c\()x w0, w0, x3 734: tbz x2, #2, 2f 74 ldr w3, [x1], #4 75CPU_BE( rev w3, w3 ) 76 crc32\c\()w w0, w0, w3 772: tbz x2, #1, 1f 78 ldrh w3, [x1], #2 79CPU_BE( rev16 w3, w3 ) 80 crc32\c\()h w0, w0, w3 811: tbz x2, #0, 0f 82 ldrb w3, [x1] 83 crc32\c\()b w0, w0, w3 840: ret 85 .endm 86 87 .align 5 88SYM_FUNC_START(crc32_le) 89alternative_if_not ARM64_HAS_CRC32 90 b crc32_le_base 91alternative_else_nop_endif 92 __crc32 93SYM_FUNC_END(crc32_le) 94 95 .align 5 96SYM_FUNC_START(__crc32c_le) 97alternative_if_not ARM64_HAS_CRC32 98 b __crc32c_le_base 99alternative_else_nop_endif 100 __crc32 c 101SYM_FUNC_END(__crc32c_le) 102