1/* SPDX-License-Identifier: GPL-2.0-or-later */ 2/* 3 * INET An implementation of the TCP/IP protocol suite for the LINUX 4 * operating system. INET is implemented using the BSD Socket 5 * interface as the means of communication with the user level. 6 * 7 * IP/TCP/UDP checksumming routines 8 * 9 * Xtensa version: Copyright (C) 2001 Tensilica, Inc. by Kevin Chea 10 * Optimized by Joe Taylor 11 */ 12 13#include <linux/errno.h> 14#include <linux/linkage.h> 15#include <asm/asmmacro.h> 16#include <asm/core.h> 17 18/* 19 * computes a partial checksum, e.g. for TCP/UDP fragments 20 */ 21 22/* 23 * unsigned int csum_partial(const unsigned char *buf, int len, 24 * unsigned int sum); 25 * a2 = buf 26 * a3 = len 27 * a4 = sum 28 * 29 * This function assumes 2- or 4-byte alignment. Other alignments will fail! 30 */ 31 32/* ONES_ADD converts twos-complement math to ones-complement. */ 33#define ONES_ADD(sum, val) \ 34 add sum, sum, val ; \ 35 bgeu sum, val, 99f ; \ 36 addi sum, sum, 1 ; \ 3799: ; 38 39.text 40ENTRY(csum_partial) 41 42 /* 43 * Experiments with Ethernet and SLIP connections show that buf 44 * is aligned on either a 2-byte or 4-byte boundary. 45 */ 46 abi_entry_default 47 extui a5, a2, 0, 2 48 bnez a5, 8f /* branch if 2-byte aligned */ 49 /* Fall-through on common case, 4-byte alignment */ 501: 51 srli a5, a3, 5 /* 32-byte chunks */ 52#if XCHAL_HAVE_LOOPS 53 loopgtz a5, 2f 54#else 55 beqz a5, 2f 56 slli a5, a5, 5 57 add a5, a5, a2 /* a5 = end of last 32-byte chunk */ 58.Loop1: 59#endif 60 l32i a6, a2, 0 61 l32i a7, a2, 4 62 ONES_ADD(a4, a6) 63 ONES_ADD(a4, a7) 64 l32i a6, a2, 8 65 l32i a7, a2, 12 66 ONES_ADD(a4, a6) 67 ONES_ADD(a4, a7) 68 l32i a6, a2, 16 69 l32i a7, a2, 20 70 ONES_ADD(a4, a6) 71 ONES_ADD(a4, a7) 72 l32i a6, a2, 24 73 l32i a7, a2, 28 74 ONES_ADD(a4, a6) 75 ONES_ADD(a4, a7) 76 addi a2, a2, 4*8 77#if !XCHAL_HAVE_LOOPS 78 blt a2, a5, .Loop1 79#endif 802: 81 extui a5, a3, 2, 3 /* remaining 4-byte chunks */ 82#if XCHAL_HAVE_LOOPS 83 loopgtz a5, 3f 84#else 85 beqz a5, 3f 86 slli a5, a5, 2 87 add a5, a5, a2 /* a5 = end of last 4-byte chunk */ 88.Loop2: 89#endif 90 l32i a6, a2, 0 91 ONES_ADD(a4, a6) 92 addi a2, a2, 4 93#if !XCHAL_HAVE_LOOPS 94 blt a2, a5, .Loop2 95#endif 963: 97 _bbci.l a3, 1, 5f /* remaining 2-byte chunk */ 98 l16ui a6, a2, 0 99 ONES_ADD(a4, a6) 100 addi a2, a2, 2 1015: 102 _bbci.l a3, 0, 7f /* remaining 1-byte chunk */ 1036: l8ui a6, a2, 0 104#ifdef __XTENSA_EB__ 105 slli a6, a6, 8 /* load byte into bits 8..15 */ 106#endif 107 ONES_ADD(a4, a6) 1087: 109 mov a2, a4 110 abi_ret_default 111 112 /* uncommon case, buf is 2-byte aligned */ 1138: 114 beqz a3, 7b /* branch if len == 0 */ 115 beqi a3, 1, 6b /* branch if len == 1 */ 116 117 extui a5, a2, 0, 1 118 bnez a5, 8f /* branch if 1-byte aligned */ 119 120 l16ui a6, a2, 0 /* common case, len >= 2 */ 121 ONES_ADD(a4, a6) 122 addi a2, a2, 2 /* adjust buf */ 123 addi a3, a3, -2 /* adjust len */ 124 j 1b /* now buf is 4-byte aligned */ 125 126 /* case: odd-byte aligned, len > 1 127 * This case is dog slow, so don't give us an odd address. 128 * (I don't think this ever happens, but just in case.) 129 */ 1308: 131 srli a5, a3, 2 /* 4-byte chunks */ 132#if XCHAL_HAVE_LOOPS 133 loopgtz a5, 2f 134#else 135 beqz a5, 2f 136 slli a5, a5, 2 137 add a5, a5, a2 /* a5 = end of last 4-byte chunk */ 138.Loop3: 139#endif 140 l8ui a6, a2, 0 /* bits 24..31 */ 141 l16ui a7, a2, 1 /* bits 8..23 */ 142 l8ui a8, a2, 3 /* bits 0.. 8 */ 143#ifdef __XTENSA_EB__ 144 slli a6, a6, 24 145#else 146 slli a8, a8, 24 147#endif 148 slli a7, a7, 8 149 or a7, a7, a6 150 or a7, a7, a8 151 ONES_ADD(a4, a7) 152 addi a2, a2, 4 153#if !XCHAL_HAVE_LOOPS 154 blt a2, a5, .Loop3 155#endif 1562: 157 _bbci.l a3, 1, 3f /* remaining 2-byte chunk, still odd addr */ 158 l8ui a6, a2, 0 159 l8ui a7, a2, 1 160#ifdef __XTENSA_EB__ 161 slli a6, a6, 8 162#else 163 slli a7, a7, 8 164#endif 165 or a7, a7, a6 166 ONES_ADD(a4, a7) 167 addi a2, a2, 2 1683: 169 j 5b /* branch to handle the remaining byte */ 170 171ENDPROC(csum_partial) 172 173/* 174 * Copy from ds while checksumming, otherwise like csum_partial 175 */ 176 177/* 178unsigned int csum_partial_copy_generic (const char *src, char *dst, int len, 179 int sum, int *src_err_ptr, int *dst_err_ptr) 180 a2 = src 181 a3 = dst 182 a4 = len 183 a5 = sum 184 a6 = src_err_ptr 185 a7 = dst_err_ptr 186 a8 = temp 187 a9 = temp 188 a10 = temp 189 a11 = original len for exception handling 190 a12 = original dst for exception handling 191 192 This function is optimized for 4-byte aligned addresses. Other 193 alignments work, but not nearly as efficiently. 194 */ 195 196ENTRY(csum_partial_copy_generic) 197 198 abi_entry_default 199 mov a12, a3 200 mov a11, a4 201 or a10, a2, a3 202 203 /* We optimize the following alignment tests for the 4-byte 204 aligned case. Two bbsi.l instructions might seem more optimal 205 (commented out below). However, both labels 5: and 3: are out 206 of the imm8 range, so the assembler relaxes them into 207 equivalent bbci.l, j combinations, which is actually 208 slower. */ 209 210 extui a9, a10, 0, 2 211 beqz a9, 1f /* branch if both are 4-byte aligned */ 212 bbsi.l a10, 0, 5f /* branch if one address is odd */ 213 j 3f /* one address is 2-byte aligned */ 214 215/* _bbsi.l a10, 0, 5f */ /* branch if odd address */ 216/* _bbsi.l a10, 1, 3f */ /* branch if 2-byte-aligned address */ 217 2181: 219 /* src and dst are both 4-byte aligned */ 220 srli a10, a4, 5 /* 32-byte chunks */ 221#if XCHAL_HAVE_LOOPS 222 loopgtz a10, 2f 223#else 224 beqz a10, 2f 225 slli a10, a10, 5 226 add a10, a10, a2 /* a10 = end of last 32-byte src chunk */ 227.Loop5: 228#endif 229EX(10f) l32i a9, a2, 0 230EX(10f) l32i a8, a2, 4 231EX(11f) s32i a9, a3, 0 232EX(11f) s32i a8, a3, 4 233 ONES_ADD(a5, a9) 234 ONES_ADD(a5, a8) 235EX(10f) l32i a9, a2, 8 236EX(10f) l32i a8, a2, 12 237EX(11f) s32i a9, a3, 8 238EX(11f) s32i a8, a3, 12 239 ONES_ADD(a5, a9) 240 ONES_ADD(a5, a8) 241EX(10f) l32i a9, a2, 16 242EX(10f) l32i a8, a2, 20 243EX(11f) s32i a9, a3, 16 244EX(11f) s32i a8, a3, 20 245 ONES_ADD(a5, a9) 246 ONES_ADD(a5, a8) 247EX(10f) l32i a9, a2, 24 248EX(10f) l32i a8, a2, 28 249EX(11f) s32i a9, a3, 24 250EX(11f) s32i a8, a3, 28 251 ONES_ADD(a5, a9) 252 ONES_ADD(a5, a8) 253 addi a2, a2, 32 254 addi a3, a3, 32 255#if !XCHAL_HAVE_LOOPS 256 blt a2, a10, .Loop5 257#endif 2582: 259 extui a10, a4, 2, 3 /* remaining 4-byte chunks */ 260 extui a4, a4, 0, 2 /* reset len for general-case, 2-byte chunks */ 261#if XCHAL_HAVE_LOOPS 262 loopgtz a10, 3f 263#else 264 beqz a10, 3f 265 slli a10, a10, 2 266 add a10, a10, a2 /* a10 = end of last 4-byte src chunk */ 267.Loop6: 268#endif 269EX(10f) l32i a9, a2, 0 270EX(11f) s32i a9, a3, 0 271 ONES_ADD(a5, a9) 272 addi a2, a2, 4 273 addi a3, a3, 4 274#if !XCHAL_HAVE_LOOPS 275 blt a2, a10, .Loop6 276#endif 2773: 278 /* 279 Control comes to here in two cases: (1) It may fall through 280 to here from the 4-byte alignment case to process, at most, 281 one 2-byte chunk. (2) It branches to here from above if 282 either src or dst is 2-byte aligned, and we process all bytes 283 here, except for perhaps a trailing odd byte. It's 284 inefficient, so align your addresses to 4-byte boundaries. 285 286 a2 = src 287 a3 = dst 288 a4 = len 289 a5 = sum 290 */ 291 srli a10, a4, 1 /* 2-byte chunks */ 292#if XCHAL_HAVE_LOOPS 293 loopgtz a10, 4f 294#else 295 beqz a10, 4f 296 slli a10, a10, 1 297 add a10, a10, a2 /* a10 = end of last 2-byte src chunk */ 298.Loop7: 299#endif 300EX(10f) l16ui a9, a2, 0 301EX(11f) s16i a9, a3, 0 302 ONES_ADD(a5, a9) 303 addi a2, a2, 2 304 addi a3, a3, 2 305#if !XCHAL_HAVE_LOOPS 306 blt a2, a10, .Loop7 307#endif 3084: 309 /* This section processes a possible trailing odd byte. */ 310 _bbci.l a4, 0, 8f /* 1-byte chunk */ 311EX(10f) l8ui a9, a2, 0 312EX(11f) s8i a9, a3, 0 313#ifdef __XTENSA_EB__ 314 slli a9, a9, 8 /* shift byte to bits 8..15 */ 315#endif 316 ONES_ADD(a5, a9) 3178: 318 mov a2, a5 319 abi_ret_default 320 3215: 322 /* Control branch to here when either src or dst is odd. We 323 process all bytes using 8-bit accesses. Grossly inefficient, 324 so don't feed us an odd address. */ 325 326 srli a10, a4, 1 /* handle in pairs for 16-bit csum */ 327#if XCHAL_HAVE_LOOPS 328 loopgtz a10, 6f 329#else 330 beqz a10, 6f 331 slli a10, a10, 1 332 add a10, a10, a2 /* a10 = end of last odd-aligned, 2-byte src chunk */ 333.Loop8: 334#endif 335EX(10f) l8ui a9, a2, 0 336EX(10f) l8ui a8, a2, 1 337EX(11f) s8i a9, a3, 0 338EX(11f) s8i a8, a3, 1 339#ifdef __XTENSA_EB__ 340 slli a9, a9, 8 /* combine into a single 16-bit value */ 341#else /* for checksum computation */ 342 slli a8, a8, 8 343#endif 344 or a9, a9, a8 345 ONES_ADD(a5, a9) 346 addi a2, a2, 2 347 addi a3, a3, 2 348#if !XCHAL_HAVE_LOOPS 349 blt a2, a10, .Loop8 350#endif 3516: 352 j 4b /* process the possible trailing odd byte */ 353 354ENDPROC(csum_partial_copy_generic) 355 356 357# Exception handler: 358.section .fixup, "ax" 359/* 360 a6 = src_err_ptr 361 a7 = dst_err_ptr 362 a11 = original len for exception handling 363 a12 = original dst for exception handling 364*/ 365 36610: 367 _movi a2, -EFAULT 368 s32i a2, a6, 0 /* src_err_ptr */ 369 370 # clear the complete destination - computing the rest 371 # is too much work 372 movi a2, 0 373#if XCHAL_HAVE_LOOPS 374 loopgtz a11, 2f 375#else 376 beqz a11, 2f 377 add a11, a11, a12 /* a11 = ending address */ 378.Leloop: 379#endif 380 s8i a2, a12, 0 381 addi a12, a12, 1 382#if !XCHAL_HAVE_LOOPS 383 blt a12, a11, .Leloop 384#endif 3852: 386 abi_ret_default 387 38811: 389 movi a2, -EFAULT 390 s32i a2, a7, 0 /* dst_err_ptr */ 391 movi a2, 0 392 abi_ret_default 393 394.previous 395