1/* 2 * linux/arch/arm64/crypto/aes-modes.S - chaining mode wrappers for AES 3 * 4 * Copyright (C) 2013 - 2017 Linaro Ltd <ard.biesheuvel@linaro.org> 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 */ 10 11/* included by aes-ce.S and aes-neon.S */ 12 13 .text 14 .align 4 15 16aes_encrypt_block4x: 17 encrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 18 ret 19ENDPROC(aes_encrypt_block4x) 20 21aes_decrypt_block4x: 22 decrypt_block4x v0, v1, v2, v3, w3, x2, x8, w7 23 ret 24ENDPROC(aes_decrypt_block4x) 25 26 /* 27 * aes_ecb_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, 28 * int blocks) 29 * aes_ecb_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, 30 * int blocks) 31 */ 32 33AES_ENTRY(aes_ecb_encrypt) 34 stp x29, x30, [sp, #-16]! 35 mov x29, sp 36 37 enc_prepare w3, x2, x5 38 39.LecbencloopNx: 40 subs w4, w4, #4 41 bmi .Lecbenc1x 42 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ 43 bl aes_encrypt_block4x 44 st1 {v0.16b-v3.16b}, [x0], #64 45 b .LecbencloopNx 46.Lecbenc1x: 47 adds w4, w4, #4 48 beq .Lecbencout 49.Lecbencloop: 50 ld1 {v0.16b}, [x1], #16 /* get next pt block */ 51 encrypt_block v0, w3, x2, x5, w6 52 st1 {v0.16b}, [x0], #16 53 subs w4, w4, #1 54 bne .Lecbencloop 55.Lecbencout: 56 ldp x29, x30, [sp], #16 57 ret 58AES_ENDPROC(aes_ecb_encrypt) 59 60 61AES_ENTRY(aes_ecb_decrypt) 62 stp x29, x30, [sp, #-16]! 63 mov x29, sp 64 65 dec_prepare w3, x2, x5 66 67.LecbdecloopNx: 68 subs w4, w4, #4 69 bmi .Lecbdec1x 70 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ 71 bl aes_decrypt_block4x 72 st1 {v0.16b-v3.16b}, [x0], #64 73 b .LecbdecloopNx 74.Lecbdec1x: 75 adds w4, w4, #4 76 beq .Lecbdecout 77.Lecbdecloop: 78 ld1 {v0.16b}, [x1], #16 /* get next ct block */ 79 decrypt_block v0, w3, x2, x5, w6 80 st1 {v0.16b}, [x0], #16 81 subs w4, w4, #1 82 bne .Lecbdecloop 83.Lecbdecout: 84 ldp x29, x30, [sp], #16 85 ret 86AES_ENDPROC(aes_ecb_decrypt) 87 88 89 /* 90 * aes_cbc_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, 91 * int blocks, u8 iv[]) 92 * aes_cbc_decrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, 93 * int blocks, u8 iv[]) 94 */ 95 96AES_ENTRY(aes_cbc_encrypt) 97 ld1 {v4.16b}, [x5] /* get iv */ 98 enc_prepare w3, x2, x6 99 100.Lcbcencloop4x: 101 subs w4, w4, #4 102 bmi .Lcbcenc1x 103 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ 104 eor v0.16b, v0.16b, v4.16b /* ..and xor with iv */ 105 encrypt_block v0, w3, x2, x6, w7 106 eor v1.16b, v1.16b, v0.16b 107 encrypt_block v1, w3, x2, x6, w7 108 eor v2.16b, v2.16b, v1.16b 109 encrypt_block v2, w3, x2, x6, w7 110 eor v3.16b, v3.16b, v2.16b 111 encrypt_block v3, w3, x2, x6, w7 112 st1 {v0.16b-v3.16b}, [x0], #64 113 mov v4.16b, v3.16b 114 b .Lcbcencloop4x 115.Lcbcenc1x: 116 adds w4, w4, #4 117 beq .Lcbcencout 118.Lcbcencloop: 119 ld1 {v0.16b}, [x1], #16 /* get next pt block */ 120 eor v4.16b, v4.16b, v0.16b /* ..and xor with iv */ 121 encrypt_block v4, w3, x2, x6, w7 122 st1 {v4.16b}, [x0], #16 123 subs w4, w4, #1 124 bne .Lcbcencloop 125.Lcbcencout: 126 st1 {v4.16b}, [x5] /* return iv */ 127 ret 128AES_ENDPROC(aes_cbc_encrypt) 129 130 131AES_ENTRY(aes_cbc_decrypt) 132 stp x29, x30, [sp, #-16]! 133 mov x29, sp 134 135 ld1 {v7.16b}, [x5] /* get iv */ 136 dec_prepare w3, x2, x6 137 138.LcbcdecloopNx: 139 subs w4, w4, #4 140 bmi .Lcbcdec1x 141 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ 142 mov v4.16b, v0.16b 143 mov v5.16b, v1.16b 144 mov v6.16b, v2.16b 145 bl aes_decrypt_block4x 146 sub x1, x1, #16 147 eor v0.16b, v0.16b, v7.16b 148 eor v1.16b, v1.16b, v4.16b 149 ld1 {v7.16b}, [x1], #16 /* reload 1 ct block */ 150 eor v2.16b, v2.16b, v5.16b 151 eor v3.16b, v3.16b, v6.16b 152 st1 {v0.16b-v3.16b}, [x0], #64 153 b .LcbcdecloopNx 154.Lcbcdec1x: 155 adds w4, w4, #4 156 beq .Lcbcdecout 157.Lcbcdecloop: 158 ld1 {v1.16b}, [x1], #16 /* get next ct block */ 159 mov v0.16b, v1.16b /* ...and copy to v0 */ 160 decrypt_block v0, w3, x2, x6, w7 161 eor v0.16b, v0.16b, v7.16b /* xor with iv => pt */ 162 mov v7.16b, v1.16b /* ct is next iv */ 163 st1 {v0.16b}, [x0], #16 164 subs w4, w4, #1 165 bne .Lcbcdecloop 166.Lcbcdecout: 167 st1 {v7.16b}, [x5] /* return iv */ 168 ldp x29, x30, [sp], #16 169 ret 170AES_ENDPROC(aes_cbc_decrypt) 171 172 173 /* 174 * aes_cbc_cts_encrypt(u8 out[], u8 const in[], u32 const rk[], 175 * int rounds, int bytes, u8 const iv[]) 176 * aes_cbc_cts_decrypt(u8 out[], u8 const in[], u32 const rk[], 177 * int rounds, int bytes, u8 const iv[]) 178 */ 179 180AES_ENTRY(aes_cbc_cts_encrypt) 181 adr_l x8, .Lcts_permute_table 182 sub x4, x4, #16 183 add x9, x8, #32 184 add x8, x8, x4 185 sub x9, x9, x4 186 ld1 {v3.16b}, [x8] 187 ld1 {v4.16b}, [x9] 188 189 ld1 {v0.16b}, [x1], x4 /* overlapping loads */ 190 ld1 {v1.16b}, [x1] 191 192 ld1 {v5.16b}, [x5] /* get iv */ 193 enc_prepare w3, x2, x6 194 195 eor v0.16b, v0.16b, v5.16b /* xor with iv */ 196 tbl v1.16b, {v1.16b}, v4.16b 197 encrypt_block v0, w3, x2, x6, w7 198 199 eor v1.16b, v1.16b, v0.16b 200 tbl v0.16b, {v0.16b}, v3.16b 201 encrypt_block v1, w3, x2, x6, w7 202 203 add x4, x0, x4 204 st1 {v0.16b}, [x4] /* overlapping stores */ 205 st1 {v1.16b}, [x0] 206 ret 207AES_ENDPROC(aes_cbc_cts_encrypt) 208 209AES_ENTRY(aes_cbc_cts_decrypt) 210 adr_l x8, .Lcts_permute_table 211 sub x4, x4, #16 212 add x9, x8, #32 213 add x8, x8, x4 214 sub x9, x9, x4 215 ld1 {v3.16b}, [x8] 216 ld1 {v4.16b}, [x9] 217 218 ld1 {v0.16b}, [x1], x4 /* overlapping loads */ 219 ld1 {v1.16b}, [x1] 220 221 ld1 {v5.16b}, [x5] /* get iv */ 222 dec_prepare w3, x2, x6 223 224 tbl v2.16b, {v1.16b}, v4.16b 225 decrypt_block v0, w3, x2, x6, w7 226 eor v2.16b, v2.16b, v0.16b 227 228 tbx v0.16b, {v1.16b}, v4.16b 229 tbl v2.16b, {v2.16b}, v3.16b 230 decrypt_block v0, w3, x2, x6, w7 231 eor v0.16b, v0.16b, v5.16b /* xor with iv */ 232 233 add x4, x0, x4 234 st1 {v2.16b}, [x4] /* overlapping stores */ 235 st1 {v0.16b}, [x0] 236 ret 237AES_ENDPROC(aes_cbc_cts_decrypt) 238 239 .section ".rodata", "a" 240 .align 6 241.Lcts_permute_table: 242 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 243 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 244 .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 245 .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe, 0xf 246 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 247 .byte 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff 248 .previous 249 250 251 /* 252 * aes_ctr_encrypt(u8 out[], u8 const in[], u8 const rk[], int rounds, 253 * int blocks, u8 ctr[]) 254 */ 255 256AES_ENTRY(aes_ctr_encrypt) 257 stp x29, x30, [sp, #-16]! 258 mov x29, sp 259 260 enc_prepare w3, x2, x6 261 ld1 {v4.16b}, [x5] 262 263 umov x6, v4.d[1] /* keep swabbed ctr in reg */ 264 rev x6, x6 265 cmn w6, w4 /* 32 bit overflow? */ 266 bcs .Lctrloop 267.LctrloopNx: 268 subs w4, w4, #4 269 bmi .Lctr1x 270 add w7, w6, #1 271 mov v0.16b, v4.16b 272 add w8, w6, #2 273 mov v1.16b, v4.16b 274 add w9, w6, #3 275 mov v2.16b, v4.16b 276 rev w7, w7 277 mov v3.16b, v4.16b 278 rev w8, w8 279 mov v1.s[3], w7 280 rev w9, w9 281 mov v2.s[3], w8 282 mov v3.s[3], w9 283 ld1 {v5.16b-v7.16b}, [x1], #48 /* get 3 input blocks */ 284 bl aes_encrypt_block4x 285 eor v0.16b, v5.16b, v0.16b 286 ld1 {v5.16b}, [x1], #16 /* get 1 input block */ 287 eor v1.16b, v6.16b, v1.16b 288 eor v2.16b, v7.16b, v2.16b 289 eor v3.16b, v5.16b, v3.16b 290 st1 {v0.16b-v3.16b}, [x0], #64 291 add x6, x6, #4 292 rev x7, x6 293 ins v4.d[1], x7 294 cbz w4, .Lctrout 295 b .LctrloopNx 296.Lctr1x: 297 adds w4, w4, #4 298 beq .Lctrout 299.Lctrloop: 300 mov v0.16b, v4.16b 301 encrypt_block v0, w3, x2, x8, w7 302 303 adds x6, x6, #1 /* increment BE ctr */ 304 rev x7, x6 305 ins v4.d[1], x7 306 bcs .Lctrcarry /* overflow? */ 307 308.Lctrcarrydone: 309 subs w4, w4, #1 310 bmi .Lctrtailblock /* blocks <0 means tail block */ 311 ld1 {v3.16b}, [x1], #16 312 eor v3.16b, v0.16b, v3.16b 313 st1 {v3.16b}, [x0], #16 314 bne .Lctrloop 315 316.Lctrout: 317 st1 {v4.16b}, [x5] /* return next CTR value */ 318 ldp x29, x30, [sp], #16 319 ret 320 321.Lctrtailblock: 322 st1 {v0.16b}, [x0] 323 ldp x29, x30, [sp], #16 324 ret 325 326.Lctrcarry: 327 umov x7, v4.d[0] /* load upper word of ctr */ 328 rev x7, x7 /* ... to handle the carry */ 329 add x7, x7, #1 330 rev x7, x7 331 ins v4.d[0], x7 332 b .Lctrcarrydone 333AES_ENDPROC(aes_ctr_encrypt) 334 335 336 /* 337 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, 338 * int blocks, u8 const rk2[], u8 iv[], int first) 339 * aes_xts_decrypt(u8 out[], u8 const in[], u8 const rk1[], int rounds, 340 * int blocks, u8 const rk2[], u8 iv[], int first) 341 */ 342 343 .macro next_tweak, out, in, tmp 344 sshr \tmp\().2d, \in\().2d, #63 345 and \tmp\().16b, \tmp\().16b, xtsmask.16b 346 add \out\().2d, \in\().2d, \in\().2d 347 ext \tmp\().16b, \tmp\().16b, \tmp\().16b, #8 348 eor \out\().16b, \out\().16b, \tmp\().16b 349 .endm 350 351 .macro xts_load_mask, tmp 352 movi xtsmask.2s, #0x1 353 movi \tmp\().2s, #0x87 354 uzp1 xtsmask.4s, xtsmask.4s, \tmp\().4s 355 .endm 356 357AES_ENTRY(aes_xts_encrypt) 358 stp x29, x30, [sp, #-16]! 359 mov x29, sp 360 361 ld1 {v4.16b}, [x6] 362 xts_load_mask v8 363 cbz w7, .Lxtsencnotfirst 364 365 enc_prepare w3, x5, x8 366 encrypt_block v4, w3, x5, x8, w7 /* first tweak */ 367 enc_switch_key w3, x2, x8 368 b .LxtsencNx 369 370.Lxtsencnotfirst: 371 enc_prepare w3, x2, x8 372.LxtsencloopNx: 373 next_tweak v4, v4, v8 374.LxtsencNx: 375 subs w4, w4, #4 376 bmi .Lxtsenc1x 377 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 pt blocks */ 378 next_tweak v5, v4, v8 379 eor v0.16b, v0.16b, v4.16b 380 next_tweak v6, v5, v8 381 eor v1.16b, v1.16b, v5.16b 382 eor v2.16b, v2.16b, v6.16b 383 next_tweak v7, v6, v8 384 eor v3.16b, v3.16b, v7.16b 385 bl aes_encrypt_block4x 386 eor v3.16b, v3.16b, v7.16b 387 eor v0.16b, v0.16b, v4.16b 388 eor v1.16b, v1.16b, v5.16b 389 eor v2.16b, v2.16b, v6.16b 390 st1 {v0.16b-v3.16b}, [x0], #64 391 mov v4.16b, v7.16b 392 cbz w4, .Lxtsencout 393 xts_reload_mask v8 394 b .LxtsencloopNx 395.Lxtsenc1x: 396 adds w4, w4, #4 397 beq .Lxtsencout 398.Lxtsencloop: 399 ld1 {v1.16b}, [x1], #16 400 eor v0.16b, v1.16b, v4.16b 401 encrypt_block v0, w3, x2, x8, w7 402 eor v0.16b, v0.16b, v4.16b 403 st1 {v0.16b}, [x0], #16 404 subs w4, w4, #1 405 beq .Lxtsencout 406 next_tweak v4, v4, v8 407 b .Lxtsencloop 408.Lxtsencout: 409 st1 {v4.16b}, [x6] 410 ldp x29, x30, [sp], #16 411 ret 412AES_ENDPROC(aes_xts_encrypt) 413 414 415AES_ENTRY(aes_xts_decrypt) 416 stp x29, x30, [sp, #-16]! 417 mov x29, sp 418 419 ld1 {v4.16b}, [x6] 420 xts_load_mask v8 421 cbz w7, .Lxtsdecnotfirst 422 423 enc_prepare w3, x5, x8 424 encrypt_block v4, w3, x5, x8, w7 /* first tweak */ 425 dec_prepare w3, x2, x8 426 b .LxtsdecNx 427 428.Lxtsdecnotfirst: 429 dec_prepare w3, x2, x8 430.LxtsdecloopNx: 431 next_tweak v4, v4, v8 432.LxtsdecNx: 433 subs w4, w4, #4 434 bmi .Lxtsdec1x 435 ld1 {v0.16b-v3.16b}, [x1], #64 /* get 4 ct blocks */ 436 next_tweak v5, v4, v8 437 eor v0.16b, v0.16b, v4.16b 438 next_tweak v6, v5, v8 439 eor v1.16b, v1.16b, v5.16b 440 eor v2.16b, v2.16b, v6.16b 441 next_tweak v7, v6, v8 442 eor v3.16b, v3.16b, v7.16b 443 bl aes_decrypt_block4x 444 eor v3.16b, v3.16b, v7.16b 445 eor v0.16b, v0.16b, v4.16b 446 eor v1.16b, v1.16b, v5.16b 447 eor v2.16b, v2.16b, v6.16b 448 st1 {v0.16b-v3.16b}, [x0], #64 449 mov v4.16b, v7.16b 450 cbz w4, .Lxtsdecout 451 xts_reload_mask v8 452 b .LxtsdecloopNx 453.Lxtsdec1x: 454 adds w4, w4, #4 455 beq .Lxtsdecout 456.Lxtsdecloop: 457 ld1 {v1.16b}, [x1], #16 458 eor v0.16b, v1.16b, v4.16b 459 decrypt_block v0, w3, x2, x8, w7 460 eor v0.16b, v0.16b, v4.16b 461 st1 {v0.16b}, [x0], #16 462 subs w4, w4, #1 463 beq .Lxtsdecout 464 next_tweak v4, v4, v8 465 b .Lxtsdecloop 466.Lxtsdecout: 467 st1 {v4.16b}, [x6] 468 ldp x29, x30, [sp], #16 469 ret 470AES_ENDPROC(aes_xts_decrypt) 471 472 /* 473 * aes_mac_update(u8 const in[], u32 const rk[], int rounds, 474 * int blocks, u8 dg[], int enc_before, int enc_after) 475 */ 476AES_ENTRY(aes_mac_update) 477 frame_push 6 478 479 mov x19, x0 480 mov x20, x1 481 mov x21, x2 482 mov x22, x3 483 mov x23, x4 484 mov x24, x6 485 486 ld1 {v0.16b}, [x23] /* get dg */ 487 enc_prepare w2, x1, x7 488 cbz w5, .Lmacloop4x 489 490 encrypt_block v0, w2, x1, x7, w8 491 492.Lmacloop4x: 493 subs w22, w22, #4 494 bmi .Lmac1x 495 ld1 {v1.16b-v4.16b}, [x19], #64 /* get next pt block */ 496 eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ 497 encrypt_block v0, w21, x20, x7, w8 498 eor v0.16b, v0.16b, v2.16b 499 encrypt_block v0, w21, x20, x7, w8 500 eor v0.16b, v0.16b, v3.16b 501 encrypt_block v0, w21, x20, x7, w8 502 eor v0.16b, v0.16b, v4.16b 503 cmp w22, wzr 504 csinv x5, x24, xzr, eq 505 cbz w5, .Lmacout 506 encrypt_block v0, w21, x20, x7, w8 507 st1 {v0.16b}, [x23] /* return dg */ 508 cond_yield_neon .Lmacrestart 509 b .Lmacloop4x 510.Lmac1x: 511 add w22, w22, #4 512.Lmacloop: 513 cbz w22, .Lmacout 514 ld1 {v1.16b}, [x19], #16 /* get next pt block */ 515 eor v0.16b, v0.16b, v1.16b /* ..and xor with dg */ 516 517 subs w22, w22, #1 518 csinv x5, x24, xzr, eq 519 cbz w5, .Lmacout 520 521.Lmacenc: 522 encrypt_block v0, w21, x20, x7, w8 523 b .Lmacloop 524 525.Lmacout: 526 st1 {v0.16b}, [x23] /* return dg */ 527 frame_pop 528 ret 529 530.Lmacrestart: 531 ld1 {v0.16b}, [x23] /* get dg */ 532 enc_prepare w21, x20, x0 533 b .Lmacloop4x 534AES_ENDPROC(aes_mac_update) 535