1/* 2 * Accelerated CRC32(C) using AArch64 CRC instructions 3 * 4 * Copyright (C) 2016 - 2018 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11#include <linux/linkage.h> 12#include <asm/alternative.h> 13#include <asm/assembler.h> 14 15 .cpu generic+crc 16 17 .macro __crc32, c 18 cmp x2, #16 19 b.lt 8f // less than 16 bytes 20 21 and x7, x2, #0x1f 22 and x2, x2, #~0x1f 23 cbz x7, 32f // multiple of 32 bytes 24 25 and x8, x7, #0xf 26 ldp x3, x4, [x1] 27 add x8, x8, x1 28 add x1, x1, x7 29 ldp x5, x6, [x8] 30CPU_BE( rev x3, x3 ) 31CPU_BE( rev x4, x4 ) 32CPU_BE( rev x5, x5 ) 33CPU_BE( rev x6, x6 ) 34 35 tst x7, #8 36 crc32\c\()x w8, w0, x3 37 csel x3, x3, x4, eq 38 csel w0, w0, w8, eq 39 tst x7, #4 40 lsr x4, x3, #32 41 crc32\c\()w w8, w0, w3 42 csel x3, x3, x4, eq 43 csel w0, w0, w8, eq 44 tst x7, #2 45 lsr w4, w3, #16 46 crc32\c\()h w8, w0, w3 47 csel w3, w3, w4, eq 48 csel w0, w0, w8, eq 49 tst x7, #1 50 crc32\c\()b w8, w0, w3 51 csel w0, w0, w8, eq 52 tst x7, #16 53 crc32\c\()x w8, w0, x5 54 crc32\c\()x w8, w8, x6 55 csel w0, w0, w8, eq 56 cbz x2, 0f 57 5832: ldp x3, x4, [x1], #32 59 sub x2, x2, #32 60 ldp x5, x6, [x1, #-16] 61CPU_BE( rev x3, x3 ) 62CPU_BE( rev x4, x4 ) 63CPU_BE( rev x5, x5 ) 64CPU_BE( rev x6, x6 ) 65 crc32\c\()x w0, w0, x3 66 crc32\c\()x w0, w0, x4 67 crc32\c\()x w0, w0, x5 68 crc32\c\()x w0, w0, x6 69 cbnz x2, 32b 700: ret 71 728: tbz x2, #3, 4f 73 ldr x3, [x1], #8 74CPU_BE( rev x3, x3 ) 75 crc32\c\()x w0, w0, x3 764: tbz x2, #2, 2f 77 ldr w3, [x1], #4 78CPU_BE( rev w3, w3 ) 79 crc32\c\()w w0, w0, w3 802: tbz x2, #1, 1f 81 ldrh w3, [x1], #2 82CPU_BE( rev16 w3, w3 ) 83 crc32\c\()h w0, w0, w3 841: tbz x2, #0, 0f 85 ldrb w3, [x1] 86 crc32\c\()b w0, w0, w3 870: ret 88 .endm 89 90 .align 5 91ENTRY(crc32_le) 92alternative_if_not ARM64_HAS_CRC32 93 b crc32_le_base 94alternative_else_nop_endif 95 __crc32 96ENDPROC(crc32_le) 97 98 .align 5 99ENTRY(__crc32c_le) 100alternative_if_not ARM64_HAS_CRC32 101 b __crc32c_le_base 102alternative_else_nop_endif 103 __crc32 c 104ENDPROC(__crc32c_le) 105