17694b6caSKeerthy // SPDX-License-Identifier: GPL-2.0 27694b6caSKeerthy /* 37694b6caSKeerthy * K3 SA2UL crypto accelerator driver 47694b6caSKeerthy * 57694b6caSKeerthy * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com 67694b6caSKeerthy * 77694b6caSKeerthy * Authors: Keerthy 87694b6caSKeerthy * Vitaly Andrianov 97694b6caSKeerthy * Tero Kristo 107694b6caSKeerthy */ 117694b6caSKeerthy #include <linux/clk.h> 12ad0bb4e4SHerbert Xu #include <linux/dma-mapping.h> 137694b6caSKeerthy #include <linux/dmaengine.h> 147694b6caSKeerthy #include <linux/dmapool.h> 15ad0bb4e4SHerbert Xu #include <linux/kernel.h> 167694b6caSKeerthy #include <linux/module.h> 177694b6caSKeerthy #include <linux/of_device.h> 187694b6caSKeerthy #include <linux/platform_device.h> 197694b6caSKeerthy #include <linux/pm_runtime.h> 207694b6caSKeerthy 217694b6caSKeerthy #include <crypto/aes.h> 22d2c8ac18SKeerthy #include <crypto/authenc.h> 237694b6caSKeerthy #include <crypto/des.h> 24d2c8ac18SKeerthy #include <crypto/internal/aead.h> 252dc53d00SKeerthy #include <crypto/internal/hash.h> 267694b6caSKeerthy #include <crypto/internal/skcipher.h> 277694b6caSKeerthy #include <crypto/scatterwalk.h> 28a24d22b2SEric Biggers #include <crypto/sha1.h> 29a24d22b2SEric Biggers #include <crypto/sha2.h> 307694b6caSKeerthy 317694b6caSKeerthy #include "sa2ul.h" 327694b6caSKeerthy 337694b6caSKeerthy /* Byte offset for key in encryption security context */ 347694b6caSKeerthy #define SC_ENC_KEY_OFFSET (1 + 27 + 4) 357694b6caSKeerthy /* Byte offset for Aux-1 in encryption security context */ 367694b6caSKeerthy #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32) 377694b6caSKeerthy 387694b6caSKeerthy #define SA_CMDL_UPD_ENC 0x0001 397694b6caSKeerthy #define SA_CMDL_UPD_AUTH 0x0002 407694b6caSKeerthy #define SA_CMDL_UPD_ENC_IV 0x0004 417694b6caSKeerthy #define SA_CMDL_UPD_AUTH_IV 0x0008 427694b6caSKeerthy #define SA_CMDL_UPD_AUX_KEY 0x0010 437694b6caSKeerthy 447694b6caSKeerthy #define SA_AUTH_SUBKEY_LEN 16 457694b6caSKeerthy #define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF 467694b6caSKeerthy #define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000 477694b6caSKeerthy 487694b6caSKeerthy #define MODE_CONTROL_BYTES 27 497694b6caSKeerthy #define SA_HASH_PROCESSING 0 507694b6caSKeerthy #define SA_CRYPTO_PROCESSING 0 517694b6caSKeerthy #define SA_UPLOAD_HASH_TO_TLR BIT(6) 527694b6caSKeerthy 537694b6caSKeerthy #define SA_SW0_FLAGS_MASK 0xF0000 547694b6caSKeerthy #define SA_SW0_CMDL_INFO_MASK 0x1F00000 557694b6caSKeerthy #define SA_SW0_CMDL_PRESENT BIT(4) 567694b6caSKeerthy #define SA_SW0_ENG_ID_MASK 0x3E000000 577694b6caSKeerthy #define SA_SW0_DEST_INFO_PRESENT BIT(30) 587694b6caSKeerthy #define SA_SW2_EGRESS_LENGTH 0xFF000000 597694b6caSKeerthy #define SA_BASIC_HASH 0x10 607694b6caSKeerthy 617694b6caSKeerthy #define SHA256_DIGEST_WORDS 8 627694b6caSKeerthy /* Make 32-bit word from 4 bytes */ 637694b6caSKeerthy #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \ 647694b6caSKeerthy ((b2) << 8) | (b3)) 657694b6caSKeerthy 667694b6caSKeerthy /* size of SCCTL structure in bytes */ 677694b6caSKeerthy #define SA_SCCTL_SZ 16 687694b6caSKeerthy 697694b6caSKeerthy /* Max Authentication tag size */ 707694b6caSKeerthy #define SA_MAX_AUTH_TAG_SZ 64 717694b6caSKeerthy 727694b6caSKeerthy #define PRIV_ID 0x1 737694b6caSKeerthy #define PRIV 0x1 747694b6caSKeerthy 757694b6caSKeerthy static struct device *sa_k3_dev; 767694b6caSKeerthy 777694b6caSKeerthy /** 787694b6caSKeerthy * struct sa_cmdl_cfg - Command label configuration descriptor 792dc53d00SKeerthy * @aalg: authentication algorithm ID 807694b6caSKeerthy * @enc_eng_id: Encryption Engine ID supported by the SA hardware 812dc53d00SKeerthy * @auth_eng_id: Authentication Engine ID 827694b6caSKeerthy * @iv_size: Initialization Vector size 832dc53d00SKeerthy * @akey: Authentication key 842dc53d00SKeerthy * @akey_len: Authentication key length 85d2c8ac18SKeerthy * @enc: True, if this is an encode request 867694b6caSKeerthy */ 877694b6caSKeerthy struct sa_cmdl_cfg { 882dc53d00SKeerthy int aalg; 897694b6caSKeerthy u8 enc_eng_id; 902dc53d00SKeerthy u8 auth_eng_id; 917694b6caSKeerthy u8 iv_size; 922dc53d00SKeerthy const u8 *akey; 932dc53d00SKeerthy u16 akey_len; 94d2c8ac18SKeerthy bool enc; 957694b6caSKeerthy }; 967694b6caSKeerthy 977694b6caSKeerthy /** 987694b6caSKeerthy * struct algo_data - Crypto algorithm specific data 997694b6caSKeerthy * @enc_eng: Encryption engine info structure 1002dc53d00SKeerthy * @auth_eng: Authentication engine info structure 1012dc53d00SKeerthy * @auth_ctrl: Authentication control word 1022dc53d00SKeerthy * @hash_size: Size of digest 1037694b6caSKeerthy * @iv_idx: iv index in psdata 1047694b6caSKeerthy * @iv_out_size: iv out size 1057694b6caSKeerthy * @ealg_id: Encryption Algorithm ID 1062dc53d00SKeerthy * @aalg_id: Authentication algorithm ID 1077694b6caSKeerthy * @mci_enc: Mode Control Instruction for Encryption algorithm 1087694b6caSKeerthy * @mci_dec: Mode Control Instruction for Decryption 1097694b6caSKeerthy * @inv_key: Whether the encryption algorithm demands key inversion 1107694b6caSKeerthy * @ctx: Pointer to the algorithm context 111d2c8ac18SKeerthy * @keyed_mac: Whether the authentication algorithm has key 112d2c8ac18SKeerthy * @prep_iopad: Function pointer to generate intermediate ipad/opad 1137694b6caSKeerthy */ 1147694b6caSKeerthy struct algo_data { 1157694b6caSKeerthy struct sa_eng_info enc_eng; 1162dc53d00SKeerthy struct sa_eng_info auth_eng; 1172dc53d00SKeerthy u8 auth_ctrl; 1182dc53d00SKeerthy u8 hash_size; 1197694b6caSKeerthy u8 iv_idx; 1207694b6caSKeerthy u8 iv_out_size; 1217694b6caSKeerthy u8 ealg_id; 1222dc53d00SKeerthy u8 aalg_id; 1237694b6caSKeerthy u8 *mci_enc; 1247694b6caSKeerthy u8 *mci_dec; 1257694b6caSKeerthy bool inv_key; 1267694b6caSKeerthy struct sa_tfm_ctx *ctx; 127d2c8ac18SKeerthy bool keyed_mac; 128d2c8ac18SKeerthy void (*prep_iopad)(struct algo_data *algo, const u8 *key, 129d2c8ac18SKeerthy u16 key_sz, __be32 *ipad, __be32 *opad); 1307694b6caSKeerthy }; 1317694b6caSKeerthy 1327694b6caSKeerthy /** 1337694b6caSKeerthy * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms 1347694b6caSKeerthy * @type: Type of the crypto algorithm. 1357694b6caSKeerthy * @alg: Union of crypto algorithm definitions. 1367694b6caSKeerthy * @registered: Flag indicating if the crypto algorithm is already registered 1377694b6caSKeerthy */ 1387694b6caSKeerthy struct sa_alg_tmpl { 1397694b6caSKeerthy u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */ 1407694b6caSKeerthy union { 1417694b6caSKeerthy struct skcipher_alg skcipher; 1422dc53d00SKeerthy struct ahash_alg ahash; 143d2c8ac18SKeerthy struct aead_alg aead; 1447694b6caSKeerthy } alg; 1457694b6caSKeerthy bool registered; 1467694b6caSKeerthy }; 1477694b6caSKeerthy 1487694b6caSKeerthy /** 14900c9211fSPeter Ujfalusi * struct sa_mapped_sg: scatterlist information for tx and rx 15000c9211fSPeter Ujfalusi * @mapped: Set to true if the @sgt is mapped 15100c9211fSPeter Ujfalusi * @dir: mapping direction used for @sgt 15200c9211fSPeter Ujfalusi * @split_sg: Set if the sg is split and needs to be freed up 15300c9211fSPeter Ujfalusi * @static_sg: Static scatterlist entry for overriding data 15400c9211fSPeter Ujfalusi * @sgt: scatterlist table for DMA API use 15500c9211fSPeter Ujfalusi */ 15600c9211fSPeter Ujfalusi struct sa_mapped_sg { 15700c9211fSPeter Ujfalusi bool mapped; 15800c9211fSPeter Ujfalusi enum dma_data_direction dir; 15900c9211fSPeter Ujfalusi struct scatterlist static_sg; 16000c9211fSPeter Ujfalusi struct scatterlist *split_sg; 16100c9211fSPeter Ujfalusi struct sg_table sgt; 16200c9211fSPeter Ujfalusi }; 16300c9211fSPeter Ujfalusi /** 1647694b6caSKeerthy * struct sa_rx_data: RX Packet miscellaneous data place holder 1657694b6caSKeerthy * @req: crypto request data pointer 1667694b6caSKeerthy * @ddev: pointer to the DMA device 1677694b6caSKeerthy * @tx_in: dma_async_tx_descriptor pointer for rx channel 16800c9211fSPeter Ujfalusi * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping 1697694b6caSKeerthy * @enc: Flag indicating either encryption or decryption 1707694b6caSKeerthy * @enc_iv_size: Initialisation vector size 1717694b6caSKeerthy * @iv_idx: Initialisation vector index 1727694b6caSKeerthy */ 1737694b6caSKeerthy struct sa_rx_data { 1747694b6caSKeerthy void *req; 1757694b6caSKeerthy struct device *ddev; 1767694b6caSKeerthy struct dma_async_tx_descriptor *tx_in; 17700c9211fSPeter Ujfalusi struct sa_mapped_sg mapped_sg[2]; 1787694b6caSKeerthy u8 enc; 1797694b6caSKeerthy u8 enc_iv_size; 1807694b6caSKeerthy u8 iv_idx; 1817694b6caSKeerthy }; 1827694b6caSKeerthy 1837694b6caSKeerthy /** 1847694b6caSKeerthy * struct sa_req: SA request definition 1857694b6caSKeerthy * @dev: device for the request 1867694b6caSKeerthy * @size: total data to the xmitted via DMA 1877694b6caSKeerthy * @enc_offset: offset of cipher data 1887694b6caSKeerthy * @enc_size: data to be passed to cipher engine 1897694b6caSKeerthy * @enc_iv: cipher IV 1902dc53d00SKeerthy * @auth_offset: offset of the authentication data 1912dc53d00SKeerthy * @auth_size: size of the authentication data 1922dc53d00SKeerthy * @auth_iv: authentication IV 1937694b6caSKeerthy * @type: algorithm type for the request 1947694b6caSKeerthy * @cmdl: command label pointer 1957694b6caSKeerthy * @base: pointer to the base request 1967694b6caSKeerthy * @ctx: pointer to the algorithm context data 1977694b6caSKeerthy * @enc: true if this is an encode request 1987694b6caSKeerthy * @src: source data 1997694b6caSKeerthy * @dst: destination data 2007694b6caSKeerthy * @callback: DMA callback for the request 2017694b6caSKeerthy * @mdata_size: metadata size passed to DMA 2027694b6caSKeerthy */ 2037694b6caSKeerthy struct sa_req { 2047694b6caSKeerthy struct device *dev; 2057694b6caSKeerthy u16 size; 2067694b6caSKeerthy u8 enc_offset; 2077694b6caSKeerthy u16 enc_size; 2087694b6caSKeerthy u8 *enc_iv; 2092dc53d00SKeerthy u8 auth_offset; 2102dc53d00SKeerthy u16 auth_size; 2112dc53d00SKeerthy u8 *auth_iv; 2127694b6caSKeerthy u32 type; 2137694b6caSKeerthy u32 *cmdl; 2147694b6caSKeerthy struct crypto_async_request *base; 2157694b6caSKeerthy struct sa_tfm_ctx *ctx; 2167694b6caSKeerthy bool enc; 2177694b6caSKeerthy struct scatterlist *src; 2187694b6caSKeerthy struct scatterlist *dst; 2197694b6caSKeerthy dma_async_tx_callback callback; 2207694b6caSKeerthy u16 mdata_size; 2217694b6caSKeerthy }; 2227694b6caSKeerthy 2237694b6caSKeerthy /* 2247694b6caSKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256 2257694b6caSKeerthy * For CBC (Cipher Block Chaining) mode for encryption 2267694b6caSKeerthy */ 2277694b6caSKeerthy static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = { 2287694b6caSKeerthy { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 2297694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2307694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 2317694b6caSKeerthy { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 2327694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2337694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 2347694b6caSKeerthy { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 2357694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2367694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 2377694b6caSKeerthy }; 2387694b6caSKeerthy 2397694b6caSKeerthy /* 2407694b6caSKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256 2417694b6caSKeerthy * For CBC (Cipher Block Chaining) mode for decryption 2427694b6caSKeerthy */ 2437694b6caSKeerthy static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = { 2447694b6caSKeerthy { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 2457694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2467694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 2477694b6caSKeerthy { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 2487694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2497694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 2507694b6caSKeerthy { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 2517694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2527694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 2537694b6caSKeerthy }; 2547694b6caSKeerthy 2557694b6caSKeerthy /* 2567694b6caSKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256 257d2c8ac18SKeerthy * For CBC (Cipher Block Chaining) mode for encryption 258d2c8ac18SKeerthy */ 259d2c8ac18SKeerthy static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = { 260d2c8ac18SKeerthy { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00, 261d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 262d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 263d2c8ac18SKeerthy { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00, 264d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 265d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 266d2c8ac18SKeerthy { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00, 267d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 268d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 269d2c8ac18SKeerthy }; 270d2c8ac18SKeerthy 271d2c8ac18SKeerthy /* 272d2c8ac18SKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256 273d2c8ac18SKeerthy * For CBC (Cipher Block Chaining) mode for decryption 274d2c8ac18SKeerthy */ 275d2c8ac18SKeerthy static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = { 276d2c8ac18SKeerthy { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 277d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 278d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 279d2c8ac18SKeerthy { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 280d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 281d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 282d2c8ac18SKeerthy { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 283d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 284d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 285d2c8ac18SKeerthy }; 286d2c8ac18SKeerthy 287d2c8ac18SKeerthy /* 288d2c8ac18SKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256 2897694b6caSKeerthy * For ECB (Electronic Code Book) mode for encryption 2907694b6caSKeerthy */ 2917694b6caSKeerthy static u8 mci_ecb_enc_array[3][27] = { 2927694b6caSKeerthy { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 2937694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2947694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 2957694b6caSKeerthy { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 2967694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 2977694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 2987694b6caSKeerthy { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 2997694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 3007694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 3017694b6caSKeerthy }; 3027694b6caSKeerthy 3037694b6caSKeerthy /* 3047694b6caSKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256 3057694b6caSKeerthy * For ECB (Electronic Code Book) mode for decryption 3067694b6caSKeerthy */ 3077694b6caSKeerthy static u8 mci_ecb_dec_array[3][27] = { 3087694b6caSKeerthy { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 3097694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 3107694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 3117694b6caSKeerthy { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 3127694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 3137694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 3147694b6caSKeerthy { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00, 3157694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 3167694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, 3177694b6caSKeerthy }; 3187694b6caSKeerthy 3197694b6caSKeerthy /* 3207694b6caSKeerthy * Mode Control Instructions for DES algorithm 3217694b6caSKeerthy * For CBC (Cipher Block Chaining) mode and ECB mode 3227694b6caSKeerthy * encryption and for decryption respectively 3237694b6caSKeerthy */ 3247694b6caSKeerthy static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = { 3257694b6caSKeerthy 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00, 3267694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 3277694b6caSKeerthy 0x00, 0x00, 0x00, 3287694b6caSKeerthy }; 3297694b6caSKeerthy 3307694b6caSKeerthy static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = { 3317694b6caSKeerthy 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00, 3327694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 3337694b6caSKeerthy 0x00, 0x00, 0x00, 3347694b6caSKeerthy }; 3357694b6caSKeerthy 3367694b6caSKeerthy static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = { 3377694b6caSKeerthy 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 3387694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 3397694b6caSKeerthy 0x00, 0x00, 0x00, 3407694b6caSKeerthy }; 3417694b6caSKeerthy 3427694b6caSKeerthy static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = { 3437694b6caSKeerthy 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00, 3447694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 3457694b6caSKeerthy 0x00, 0x00, 0x00, 3467694b6caSKeerthy }; 3477694b6caSKeerthy 3487694b6caSKeerthy /* 3497694b6caSKeerthy * Perform 16 byte or 128 bit swizzling 3507694b6caSKeerthy * The SA2UL Expects the security context to 3517694b6caSKeerthy * be in little Endian and the bus width is 128 bits or 16 bytes 3527694b6caSKeerthy * Hence swap 16 bytes at a time from higher to lower address 3537694b6caSKeerthy */ 3547694b6caSKeerthy static void sa_swiz_128(u8 *in, u16 len) 3557694b6caSKeerthy { 3567694b6caSKeerthy u8 data[16]; 3577694b6caSKeerthy int i, j; 3587694b6caSKeerthy 3597694b6caSKeerthy for (i = 0; i < len; i += 16) { 3607694b6caSKeerthy memcpy(data, &in[i], 16); 3617694b6caSKeerthy for (j = 0; j < 16; j++) 3627694b6caSKeerthy in[i + j] = data[15 - j]; 3637694b6caSKeerthy } 3647694b6caSKeerthy } 3657694b6caSKeerthy 366d2c8ac18SKeerthy /* Prepare the ipad and opad from key as per SHA algorithm step 1*/ 367ad0bb4e4SHerbert Xu static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz) 368d2c8ac18SKeerthy { 369d2c8ac18SKeerthy int i; 370d2c8ac18SKeerthy 371ad0bb4e4SHerbert Xu for (i = 0; i < key_sz; i++) 372d2c8ac18SKeerthy k_ipad[i] = key[i] ^ 0x36; 373d2c8ac18SKeerthy 374d2c8ac18SKeerthy /* Instead of XOR with 0 */ 375ad0bb4e4SHerbert Xu for (; i < SHA1_BLOCK_SIZE; i++) 376d2c8ac18SKeerthy k_ipad[i] = 0x36; 377d2c8ac18SKeerthy } 378d2c8ac18SKeerthy 379ad0bb4e4SHerbert Xu static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz) 380ad0bb4e4SHerbert Xu { 381ad0bb4e4SHerbert Xu int i; 382ad0bb4e4SHerbert Xu 383ad0bb4e4SHerbert Xu for (i = 0; i < key_sz; i++) 384ad0bb4e4SHerbert Xu k_opad[i] = key[i] ^ 0x5c; 385ad0bb4e4SHerbert Xu 386ad0bb4e4SHerbert Xu /* Instead of XOR with 0 */ 387ad0bb4e4SHerbert Xu for (; i < SHA1_BLOCK_SIZE; i++) 388ad0bb4e4SHerbert Xu k_opad[i] = 0x5c; 389ad0bb4e4SHerbert Xu } 390ad0bb4e4SHerbert Xu 391ad0bb4e4SHerbert Xu static void sa_export_shash(void *state, struct shash_desc *hash, 392d2c8ac18SKeerthy int digest_size, __be32 *out) 393d2c8ac18SKeerthy { 394ad0bb4e4SHerbert Xu struct sha1_state *sha1; 395ad0bb4e4SHerbert Xu struct sha256_state *sha256; 396d2c8ac18SKeerthy u32 *result; 397d2c8ac18SKeerthy 398d2c8ac18SKeerthy switch (digest_size) { 399d2c8ac18SKeerthy case SHA1_DIGEST_SIZE: 400ad0bb4e4SHerbert Xu sha1 = state; 401ad0bb4e4SHerbert Xu result = sha1->state; 402d2c8ac18SKeerthy break; 403d2c8ac18SKeerthy case SHA256_DIGEST_SIZE: 404ad0bb4e4SHerbert Xu sha256 = state; 405ad0bb4e4SHerbert Xu result = sha256->state; 406d2c8ac18SKeerthy break; 407d2c8ac18SKeerthy default: 408d2c8ac18SKeerthy dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__, 409d2c8ac18SKeerthy digest_size); 410d2c8ac18SKeerthy return; 411d2c8ac18SKeerthy } 412d2c8ac18SKeerthy 413d2c8ac18SKeerthy crypto_shash_export(hash, state); 414d2c8ac18SKeerthy 415ad0bb4e4SHerbert Xu cpu_to_be32_array(out, result, digest_size / 4); 416d2c8ac18SKeerthy } 417d2c8ac18SKeerthy 418d2c8ac18SKeerthy static void sa_prepare_iopads(struct algo_data *data, const u8 *key, 419d2c8ac18SKeerthy u16 key_sz, __be32 *ipad, __be32 *opad) 420d2c8ac18SKeerthy { 421d2c8ac18SKeerthy SHASH_DESC_ON_STACK(shash, data->ctx->shash); 422d2c8ac18SKeerthy int block_size = crypto_shash_blocksize(data->ctx->shash); 423d2c8ac18SKeerthy int digest_size = crypto_shash_digestsize(data->ctx->shash); 424ad0bb4e4SHerbert Xu union { 425ad0bb4e4SHerbert Xu struct sha1_state sha1; 426ad0bb4e4SHerbert Xu struct sha256_state sha256; 427ad0bb4e4SHerbert Xu u8 k_pad[SHA1_BLOCK_SIZE]; 428ad0bb4e4SHerbert Xu } sha; 429d2c8ac18SKeerthy 430d2c8ac18SKeerthy shash->tfm = data->ctx->shash; 431d2c8ac18SKeerthy 432ad0bb4e4SHerbert Xu prepare_kipad(sha.k_pad, key, key_sz); 433d2c8ac18SKeerthy 434d2c8ac18SKeerthy crypto_shash_init(shash); 435ad0bb4e4SHerbert Xu crypto_shash_update(shash, sha.k_pad, block_size); 436ad0bb4e4SHerbert Xu sa_export_shash(&sha, shash, digest_size, ipad); 437ad0bb4e4SHerbert Xu 438ad0bb4e4SHerbert Xu prepare_kopad(sha.k_pad, key, key_sz); 439d2c8ac18SKeerthy 440d2c8ac18SKeerthy crypto_shash_init(shash); 441ad0bb4e4SHerbert Xu crypto_shash_update(shash, sha.k_pad, block_size); 442d2c8ac18SKeerthy 443ad0bb4e4SHerbert Xu sa_export_shash(&sha, shash, digest_size, opad); 444ad0bb4e4SHerbert Xu 445ad0bb4e4SHerbert Xu memzero_explicit(&sha, sizeof(sha)); 446d2c8ac18SKeerthy } 447d2c8ac18SKeerthy 4487694b6caSKeerthy /* Derive the inverse key used in AES-CBC decryption operation */ 4497694b6caSKeerthy static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz) 4507694b6caSKeerthy { 4517694b6caSKeerthy struct crypto_aes_ctx ctx; 4527694b6caSKeerthy int key_pos; 4537694b6caSKeerthy 4547694b6caSKeerthy if (aes_expandkey(&ctx, key, key_sz)) { 4557694b6caSKeerthy dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 4567694b6caSKeerthy return -EINVAL; 4577694b6caSKeerthy } 4587694b6caSKeerthy 4597694b6caSKeerthy /* work around to get the right inverse for AES_KEYSIZE_192 size keys */ 4607694b6caSKeerthy if (key_sz == AES_KEYSIZE_192) { 4617694b6caSKeerthy ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46]; 4627694b6caSKeerthy ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47]; 4637694b6caSKeerthy } 4647694b6caSKeerthy 4657694b6caSKeerthy /* Based crypto_aes_expand_key logic */ 4667694b6caSKeerthy switch (key_sz) { 4677694b6caSKeerthy case AES_KEYSIZE_128: 4687694b6caSKeerthy case AES_KEYSIZE_192: 4697694b6caSKeerthy key_pos = key_sz + 24; 4707694b6caSKeerthy break; 4717694b6caSKeerthy 4727694b6caSKeerthy case AES_KEYSIZE_256: 4737694b6caSKeerthy key_pos = key_sz + 24 - 4; 4747694b6caSKeerthy break; 4757694b6caSKeerthy 4767694b6caSKeerthy default: 4777694b6caSKeerthy dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz); 4787694b6caSKeerthy return -EINVAL; 4797694b6caSKeerthy } 4807694b6caSKeerthy 4817694b6caSKeerthy memcpy(inv_key, &ctx.key_enc[key_pos], key_sz); 4827694b6caSKeerthy return 0; 4837694b6caSKeerthy } 4847694b6caSKeerthy 4857694b6caSKeerthy /* Set Security context for the encryption engine */ 4867694b6caSKeerthy static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz, 4877694b6caSKeerthy u8 enc, u8 *sc_buf) 4887694b6caSKeerthy { 4897694b6caSKeerthy const u8 *mci = NULL; 4907694b6caSKeerthy 4917694b6caSKeerthy /* Set Encryption mode selector to crypto processing */ 4927694b6caSKeerthy sc_buf[0] = SA_CRYPTO_PROCESSING; 4937694b6caSKeerthy 4947694b6caSKeerthy if (enc) 4957694b6caSKeerthy mci = ad->mci_enc; 4967694b6caSKeerthy else 4977694b6caSKeerthy mci = ad->mci_dec; 4987694b6caSKeerthy /* Set the mode control instructions in security context */ 4997694b6caSKeerthy if (mci) 5007694b6caSKeerthy memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES); 5017694b6caSKeerthy 5027694b6caSKeerthy /* For AES-CBC decryption get the inverse key */ 5037694b6caSKeerthy if (ad->inv_key && !enc) { 5047694b6caSKeerthy if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz)) 5057694b6caSKeerthy return -EINVAL; 5067694b6caSKeerthy /* For all other cases: key is used */ 5077694b6caSKeerthy } else { 5087694b6caSKeerthy memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz); 5097694b6caSKeerthy } 5107694b6caSKeerthy 5117694b6caSKeerthy return 0; 5127694b6caSKeerthy } 5137694b6caSKeerthy 5142dc53d00SKeerthy /* Set Security context for the authentication engine */ 5152dc53d00SKeerthy static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz, 5162dc53d00SKeerthy u8 *sc_buf) 5172dc53d00SKeerthy { 518ad0bb4e4SHerbert Xu __be32 *ipad = (void *)(sc_buf + 32); 519ad0bb4e4SHerbert Xu __be32 *opad = (void *)(sc_buf + 64); 520d2c8ac18SKeerthy 5212dc53d00SKeerthy /* Set Authentication mode selector to hash processing */ 5222dc53d00SKeerthy sc_buf[0] = SA_HASH_PROCESSING; 5232dc53d00SKeerthy /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */ 5242dc53d00SKeerthy sc_buf[1] = SA_UPLOAD_HASH_TO_TLR; 5252dc53d00SKeerthy sc_buf[1] |= ad->auth_ctrl; 5262dc53d00SKeerthy 527d2c8ac18SKeerthy /* Copy the keys or ipad/opad */ 528ad0bb4e4SHerbert Xu if (ad->keyed_mac) 529d2c8ac18SKeerthy ad->prep_iopad(ad, key, key_sz, ipad, opad); 530ad0bb4e4SHerbert Xu else { 5312dc53d00SKeerthy /* basic hash */ 5322dc53d00SKeerthy sc_buf[1] |= SA_BASIC_HASH; 5332dc53d00SKeerthy } 534d2c8ac18SKeerthy } 5352dc53d00SKeerthy 5367694b6caSKeerthy static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16) 5377694b6caSKeerthy { 5387694b6caSKeerthy int j; 5397694b6caSKeerthy 5407694b6caSKeerthy for (j = 0; j < ((size16) ? 4 : 2); j++) { 5417694b6caSKeerthy *out = cpu_to_be32(*((u32 *)iv)); 5427694b6caSKeerthy iv += 4; 5437694b6caSKeerthy out++; 5447694b6caSKeerthy } 5457694b6caSKeerthy } 5467694b6caSKeerthy 5477694b6caSKeerthy /* Format general command label */ 5487694b6caSKeerthy static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl, 5497694b6caSKeerthy struct sa_cmdl_upd_info *upd_info) 5507694b6caSKeerthy { 5512dc53d00SKeerthy u8 enc_offset = 0, auth_offset = 0, total = 0; 5527694b6caSKeerthy u8 enc_next_eng = SA_ENG_ID_OUTPORT2; 5532dc53d00SKeerthy u8 auth_next_eng = SA_ENG_ID_OUTPORT2; 5547694b6caSKeerthy u32 *word_ptr = (u32 *)cmdl; 5557694b6caSKeerthy int i; 5567694b6caSKeerthy 5577694b6caSKeerthy /* Clear the command label */ 5587694b6caSKeerthy memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32))); 5597694b6caSKeerthy 5607694b6caSKeerthy /* Iniialize the command update structure */ 5617694b6caSKeerthy memzero_explicit(upd_info, sizeof(*upd_info)); 5627694b6caSKeerthy 563d2c8ac18SKeerthy if (cfg->enc_eng_id && cfg->auth_eng_id) { 564d2c8ac18SKeerthy if (cfg->enc) { 565d2c8ac18SKeerthy auth_offset = SA_CMDL_HEADER_SIZE_BYTES; 566d2c8ac18SKeerthy enc_next_eng = cfg->auth_eng_id; 5677694b6caSKeerthy 5687694b6caSKeerthy if (cfg->iv_size) 569d2c8ac18SKeerthy auth_offset += cfg->iv_size; 570d2c8ac18SKeerthy } else { 571d2c8ac18SKeerthy enc_offset = SA_CMDL_HEADER_SIZE_BYTES; 572d2c8ac18SKeerthy auth_next_eng = cfg->enc_eng_id; 573d2c8ac18SKeerthy } 574d2c8ac18SKeerthy } 5757694b6caSKeerthy 5762dc53d00SKeerthy if (cfg->enc_eng_id) { 5777694b6caSKeerthy upd_info->flags |= SA_CMDL_UPD_ENC; 5787694b6caSKeerthy upd_info->enc_size.index = enc_offset >> 2; 5797694b6caSKeerthy upd_info->enc_offset.index = upd_info->enc_size.index + 1; 5807694b6caSKeerthy /* Encryption command label */ 5817694b6caSKeerthy cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng; 5827694b6caSKeerthy 5837694b6caSKeerthy /* Encryption modes requiring IV */ 5847694b6caSKeerthy if (cfg->iv_size) { 5857694b6caSKeerthy upd_info->flags |= SA_CMDL_UPD_ENC_IV; 5867694b6caSKeerthy upd_info->enc_iv.index = 5877694b6caSKeerthy (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2; 5887694b6caSKeerthy upd_info->enc_iv.size = cfg->iv_size; 5897694b6caSKeerthy 5907694b6caSKeerthy cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 5917694b6caSKeerthy SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 5927694b6caSKeerthy 5937694b6caSKeerthy cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] = 5947694b6caSKeerthy (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3)); 595d2c8ac18SKeerthy total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size; 5967694b6caSKeerthy } else { 5977694b6caSKeerthy cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] = 5987694b6caSKeerthy SA_CMDL_HEADER_SIZE_BYTES; 599d2c8ac18SKeerthy total += SA_CMDL_HEADER_SIZE_BYTES; 6007694b6caSKeerthy } 6017694b6caSKeerthy } 6027694b6caSKeerthy 6032dc53d00SKeerthy if (cfg->auth_eng_id) { 6042dc53d00SKeerthy upd_info->flags |= SA_CMDL_UPD_AUTH; 6052dc53d00SKeerthy upd_info->auth_size.index = auth_offset >> 2; 6062dc53d00SKeerthy upd_info->auth_offset.index = upd_info->auth_size.index + 1; 6072dc53d00SKeerthy cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng; 6082dc53d00SKeerthy cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] = 6092dc53d00SKeerthy SA_CMDL_HEADER_SIZE_BYTES; 6102dc53d00SKeerthy total += SA_CMDL_HEADER_SIZE_BYTES; 6112dc53d00SKeerthy } 6122dc53d00SKeerthy 6137694b6caSKeerthy total = roundup(total, 8); 6147694b6caSKeerthy 6157694b6caSKeerthy for (i = 0; i < total / 4; i++) 6167694b6caSKeerthy word_ptr[i] = swab32(word_ptr[i]); 6177694b6caSKeerthy 6187694b6caSKeerthy return total; 6197694b6caSKeerthy } 6207694b6caSKeerthy 6217694b6caSKeerthy /* Update Command label */ 6227694b6caSKeerthy static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl, 6237694b6caSKeerthy struct sa_cmdl_upd_info *upd_info) 6247694b6caSKeerthy { 6257694b6caSKeerthy int i = 0, j; 6267694b6caSKeerthy 6277694b6caSKeerthy if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) { 6287694b6caSKeerthy cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 6297694b6caSKeerthy cmdl[upd_info->enc_size.index] |= req->enc_size; 6307694b6caSKeerthy cmdl[upd_info->enc_offset.index] &= 6317694b6caSKeerthy ~SA_CMDL_SOP_BYPASS_LEN_MASK; 6327694b6caSKeerthy cmdl[upd_info->enc_offset.index] |= 6337694b6caSKeerthy ((u32)req->enc_offset << 6347694b6caSKeerthy __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK)); 6357694b6caSKeerthy 6367694b6caSKeerthy if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) { 6377694b6caSKeerthy __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index]; 6387694b6caSKeerthy u32 *enc_iv = (u32 *)req->enc_iv; 6397694b6caSKeerthy 6407694b6caSKeerthy for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) { 6417694b6caSKeerthy data[j] = cpu_to_be32(*enc_iv); 6427694b6caSKeerthy enc_iv++; 6437694b6caSKeerthy } 6447694b6caSKeerthy } 6457694b6caSKeerthy } 6462dc53d00SKeerthy 6472dc53d00SKeerthy if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) { 6482dc53d00SKeerthy cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK; 6492dc53d00SKeerthy cmdl[upd_info->auth_size.index] |= req->auth_size; 6502dc53d00SKeerthy cmdl[upd_info->auth_offset.index] &= 6512dc53d00SKeerthy ~SA_CMDL_SOP_BYPASS_LEN_MASK; 6522dc53d00SKeerthy cmdl[upd_info->auth_offset.index] |= 6532dc53d00SKeerthy ((u32)req->auth_offset << 6542dc53d00SKeerthy __ffs(SA_CMDL_SOP_BYPASS_LEN_MASK)); 6552dc53d00SKeerthy if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) { 6562dc53d00SKeerthy sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index], 6572dc53d00SKeerthy req->auth_iv, 6582dc53d00SKeerthy (upd_info->auth_iv.size > 8)); 6592dc53d00SKeerthy } 6602dc53d00SKeerthy if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) { 6612dc53d00SKeerthy int offset = (req->auth_size & 0xF) ? 4 : 0; 6622dc53d00SKeerthy 6632dc53d00SKeerthy memcpy(&cmdl[upd_info->aux_key_info.index], 6642dc53d00SKeerthy &upd_info->aux_key[offset], 16); 6652dc53d00SKeerthy } 6662dc53d00SKeerthy } 6677694b6caSKeerthy } 6687694b6caSKeerthy 6697694b6caSKeerthy /* Format SWINFO words to be sent to SA */ 6707694b6caSKeerthy static 6717694b6caSKeerthy void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys, 6727694b6caSKeerthy u8 cmdl_present, u8 cmdl_offset, u8 flags, 6737694b6caSKeerthy u8 hash_size, u32 *swinfo) 6747694b6caSKeerthy { 6757694b6caSKeerthy swinfo[0] = sc_id; 6767694b6caSKeerthy swinfo[0] |= (flags << __ffs(SA_SW0_FLAGS_MASK)); 6777694b6caSKeerthy if (likely(cmdl_present)) 6787694b6caSKeerthy swinfo[0] |= ((cmdl_offset | SA_SW0_CMDL_PRESENT) << 6797694b6caSKeerthy __ffs(SA_SW0_CMDL_INFO_MASK)); 6807694b6caSKeerthy swinfo[0] |= (eng_id << __ffs(SA_SW0_ENG_ID_MASK)); 6817694b6caSKeerthy 6827694b6caSKeerthy swinfo[0] |= SA_SW0_DEST_INFO_PRESENT; 6837694b6caSKeerthy swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL); 6847694b6caSKeerthy swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32); 6857694b6caSKeerthy swinfo[2] |= (hash_size << __ffs(SA_SW2_EGRESS_LENGTH)); 6867694b6caSKeerthy } 6877694b6caSKeerthy 6887694b6caSKeerthy /* Dump the security context */ 6897694b6caSKeerthy static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr) 6907694b6caSKeerthy { 6917694b6caSKeerthy #ifdef DEBUG 6927694b6caSKeerthy dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr); 6937694b6caSKeerthy print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET, 6947694b6caSKeerthy 16, 1, buf, SA_CTX_MAX_SZ, false); 6957694b6caSKeerthy #endif 6967694b6caSKeerthy } 6977694b6caSKeerthy 6987694b6caSKeerthy static 6997694b6caSKeerthy int sa_init_sc(struct sa_ctx_info *ctx, const u8 *enc_key, 7002dc53d00SKeerthy u16 enc_key_sz, const u8 *auth_key, u16 auth_key_sz, 7012dc53d00SKeerthy struct algo_data *ad, u8 enc, u32 *swinfo) 7027694b6caSKeerthy { 7037694b6caSKeerthy int enc_sc_offset = 0; 7042dc53d00SKeerthy int auth_sc_offset = 0; 7057694b6caSKeerthy u8 *sc_buf = ctx->sc; 7067694b6caSKeerthy u16 sc_id = ctx->sc_id; 707d2c8ac18SKeerthy u8 first_engine = 0; 7087694b6caSKeerthy 7097694b6caSKeerthy memzero_explicit(sc_buf, SA_CTX_MAX_SZ); 7107694b6caSKeerthy 711d2c8ac18SKeerthy if (ad->auth_eng.eng_id) { 712d2c8ac18SKeerthy if (enc) 7132dc53d00SKeerthy first_engine = ad->enc_eng.eng_id; 714d2c8ac18SKeerthy else 715d2c8ac18SKeerthy first_engine = ad->auth_eng.eng_id; 716d2c8ac18SKeerthy 7172dc53d00SKeerthy enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 7182dc53d00SKeerthy auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size; 7192dc53d00SKeerthy sc_buf[1] = SA_SCCTL_FE_AUTH_ENC; 7202dc53d00SKeerthy if (!ad->hash_size) 7212dc53d00SKeerthy return -EINVAL; 7222dc53d00SKeerthy ad->hash_size = roundup(ad->hash_size, 8); 723d2c8ac18SKeerthy 724d2c8ac18SKeerthy } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) { 725d2c8ac18SKeerthy enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ; 726d2c8ac18SKeerthy first_engine = ad->enc_eng.eng_id; 727d2c8ac18SKeerthy sc_buf[1] = SA_SCCTL_FE_ENC; 728d2c8ac18SKeerthy ad->hash_size = ad->iv_out_size; 7292dc53d00SKeerthy } 7307694b6caSKeerthy 7317694b6caSKeerthy /* SCCTL Owner info: 0=host, 1=CP_ACE */ 7327694b6caSKeerthy sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0; 7337694b6caSKeerthy memcpy(&sc_buf[2], &sc_id, 2); 7347694b6caSKeerthy sc_buf[4] = 0x0; 7357694b6caSKeerthy sc_buf[5] = PRIV_ID; 7367694b6caSKeerthy sc_buf[6] = PRIV; 7377694b6caSKeerthy sc_buf[7] = 0x0; 7387694b6caSKeerthy 7397694b6caSKeerthy /* Prepare context for encryption engine */ 7407694b6caSKeerthy if (ad->enc_eng.sc_size) { 7417694b6caSKeerthy if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc, 7427694b6caSKeerthy &sc_buf[enc_sc_offset])) 7437694b6caSKeerthy return -EINVAL; 7447694b6caSKeerthy } 7457694b6caSKeerthy 7462dc53d00SKeerthy /* Prepare context for authentication engine */ 7472dc53d00SKeerthy if (ad->auth_eng.sc_size) 7482dc53d00SKeerthy sa_set_sc_auth(ad, auth_key, auth_key_sz, 7492dc53d00SKeerthy &sc_buf[auth_sc_offset]); 7502dc53d00SKeerthy 7517694b6caSKeerthy /* Set the ownership of context to CP_ACE */ 7527694b6caSKeerthy sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80; 7537694b6caSKeerthy 7547694b6caSKeerthy /* swizzle the security context */ 7557694b6caSKeerthy sa_swiz_128(sc_buf, SA_CTX_MAX_SZ); 7567694b6caSKeerthy 7577694b6caSKeerthy sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0, 7582dc53d00SKeerthy SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo); 7597694b6caSKeerthy 7607694b6caSKeerthy sa_dump_sc(sc_buf, ctx->sc_phys); 7617694b6caSKeerthy 7627694b6caSKeerthy return 0; 7637694b6caSKeerthy } 7647694b6caSKeerthy 7657694b6caSKeerthy /* Free the per direction context memory */ 7667694b6caSKeerthy static void sa_free_ctx_info(struct sa_ctx_info *ctx, 7677694b6caSKeerthy struct sa_crypto_data *data) 7687694b6caSKeerthy { 7697694b6caSKeerthy unsigned long bn; 7707694b6caSKeerthy 7717694b6caSKeerthy bn = ctx->sc_id - data->sc_id_start; 7727694b6caSKeerthy spin_lock(&data->scid_lock); 7737694b6caSKeerthy __clear_bit(bn, data->ctx_bm); 7747694b6caSKeerthy data->sc_id--; 7757694b6caSKeerthy spin_unlock(&data->scid_lock); 7767694b6caSKeerthy 7777694b6caSKeerthy if (ctx->sc) { 7787694b6caSKeerthy dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys); 7797694b6caSKeerthy ctx->sc = NULL; 7807694b6caSKeerthy } 7817694b6caSKeerthy } 7827694b6caSKeerthy 7837694b6caSKeerthy static int sa_init_ctx_info(struct sa_ctx_info *ctx, 7847694b6caSKeerthy struct sa_crypto_data *data) 7857694b6caSKeerthy { 7867694b6caSKeerthy unsigned long bn; 7877694b6caSKeerthy int err; 7887694b6caSKeerthy 7897694b6caSKeerthy spin_lock(&data->scid_lock); 7907694b6caSKeerthy bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX); 7917694b6caSKeerthy __set_bit(bn, data->ctx_bm); 7927694b6caSKeerthy data->sc_id++; 7937694b6caSKeerthy spin_unlock(&data->scid_lock); 7947694b6caSKeerthy 7957694b6caSKeerthy ctx->sc_id = (u16)(data->sc_id_start + bn); 7967694b6caSKeerthy 7977694b6caSKeerthy ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys); 7987694b6caSKeerthy if (!ctx->sc) { 7997694b6caSKeerthy dev_err(&data->pdev->dev, "Failed to allocate SC memory\n"); 8007694b6caSKeerthy err = -ENOMEM; 8017694b6caSKeerthy goto scid_rollback; 8027694b6caSKeerthy } 8037694b6caSKeerthy 8047694b6caSKeerthy return 0; 8057694b6caSKeerthy 8067694b6caSKeerthy scid_rollback: 8077694b6caSKeerthy spin_lock(&data->scid_lock); 8087694b6caSKeerthy __clear_bit(bn, data->ctx_bm); 8097694b6caSKeerthy data->sc_id--; 8107694b6caSKeerthy spin_unlock(&data->scid_lock); 8117694b6caSKeerthy 8127694b6caSKeerthy return err; 8137694b6caSKeerthy } 8147694b6caSKeerthy 8157694b6caSKeerthy static void sa_cipher_cra_exit(struct crypto_skcipher *tfm) 8167694b6caSKeerthy { 8177694b6caSKeerthy struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 8187694b6caSKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 8197694b6caSKeerthy 8207694b6caSKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 8217694b6caSKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 8227694b6caSKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys); 8237694b6caSKeerthy 8247694b6caSKeerthy sa_free_ctx_info(&ctx->enc, data); 8257694b6caSKeerthy sa_free_ctx_info(&ctx->dec, data); 8267694b6caSKeerthy 827ad0bb4e4SHerbert Xu crypto_free_skcipher(ctx->fallback.skcipher); 8287694b6caSKeerthy } 8297694b6caSKeerthy 8307694b6caSKeerthy static int sa_cipher_cra_init(struct crypto_skcipher *tfm) 8317694b6caSKeerthy { 8327694b6caSKeerthy struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 8337694b6caSKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 8347694b6caSKeerthy const char *name = crypto_tfm_alg_name(&tfm->base); 835ad0bb4e4SHerbert Xu struct crypto_skcipher *child; 8367694b6caSKeerthy int ret; 8377694b6caSKeerthy 8387694b6caSKeerthy memzero_explicit(ctx, sizeof(*ctx)); 8397694b6caSKeerthy ctx->dev_data = data; 8407694b6caSKeerthy 8417694b6caSKeerthy ret = sa_init_ctx_info(&ctx->enc, data); 8427694b6caSKeerthy if (ret) 8437694b6caSKeerthy return ret; 8447694b6caSKeerthy ret = sa_init_ctx_info(&ctx->dec, data); 8457694b6caSKeerthy if (ret) { 8467694b6caSKeerthy sa_free_ctx_info(&ctx->enc, data); 8477694b6caSKeerthy return ret; 8487694b6caSKeerthy } 8497694b6caSKeerthy 850ad0bb4e4SHerbert Xu child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK); 8517694b6caSKeerthy 852ad0bb4e4SHerbert Xu if (IS_ERR(child)) { 8537694b6caSKeerthy dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name); 854ad0bb4e4SHerbert Xu return PTR_ERR(child); 8557694b6caSKeerthy } 8567694b6caSKeerthy 857ad0bb4e4SHerbert Xu ctx->fallback.skcipher = child; 858ad0bb4e4SHerbert Xu crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) + 859ad0bb4e4SHerbert Xu sizeof(struct skcipher_request)); 860ad0bb4e4SHerbert Xu 8617694b6caSKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 8627694b6caSKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 8637694b6caSKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys); 8647694b6caSKeerthy return 0; 8657694b6caSKeerthy } 8667694b6caSKeerthy 8677694b6caSKeerthy static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key, 8687694b6caSKeerthy unsigned int keylen, struct algo_data *ad) 8697694b6caSKeerthy { 8707694b6caSKeerthy struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm); 871ad0bb4e4SHerbert Xu struct crypto_skcipher *child = ctx->fallback.skcipher; 8727694b6caSKeerthy int cmdl_len; 8737694b6caSKeerthy struct sa_cmdl_cfg cfg; 8747694b6caSKeerthy int ret; 8757694b6caSKeerthy 8767694b6caSKeerthy if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 && 8777694b6caSKeerthy keylen != AES_KEYSIZE_256) 8787694b6caSKeerthy return -EINVAL; 8797694b6caSKeerthy 8807694b6caSKeerthy ad->enc_eng.eng_id = SA_ENG_ID_EM1; 8817694b6caSKeerthy ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 8827694b6caSKeerthy 8837694b6caSKeerthy memzero_explicit(&cfg, sizeof(cfg)); 8847694b6caSKeerthy cfg.enc_eng_id = ad->enc_eng.eng_id; 8857694b6caSKeerthy cfg.iv_size = crypto_skcipher_ivsize(tfm); 8867694b6caSKeerthy 887ad0bb4e4SHerbert Xu crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK); 888ad0bb4e4SHerbert Xu crypto_skcipher_set_flags(child, tfm->base.crt_flags & 8897694b6caSKeerthy CRYPTO_TFM_REQ_MASK); 890ad0bb4e4SHerbert Xu ret = crypto_skcipher_setkey(child, key, keylen); 8917694b6caSKeerthy if (ret) 8927694b6caSKeerthy return ret; 8937694b6caSKeerthy 8947694b6caSKeerthy /* Setup Encryption Security Context & Command label template */ 8952dc53d00SKeerthy if (sa_init_sc(&ctx->enc, key, keylen, NULL, 0, ad, 1, 8962dc53d00SKeerthy &ctx->enc.epib[1])) 8977694b6caSKeerthy goto badkey; 8987694b6caSKeerthy 8997694b6caSKeerthy cmdl_len = sa_format_cmdl_gen(&cfg, 9007694b6caSKeerthy (u8 *)ctx->enc.cmdl, 9017694b6caSKeerthy &ctx->enc.cmdl_upd_info); 9027694b6caSKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 9037694b6caSKeerthy goto badkey; 9047694b6caSKeerthy 9057694b6caSKeerthy ctx->enc.cmdl_size = cmdl_len; 9067694b6caSKeerthy 9077694b6caSKeerthy /* Setup Decryption Security Context & Command label template */ 9082dc53d00SKeerthy if (sa_init_sc(&ctx->dec, key, keylen, NULL, 0, ad, 0, 9092dc53d00SKeerthy &ctx->dec.epib[1])) 9107694b6caSKeerthy goto badkey; 9117694b6caSKeerthy 9127694b6caSKeerthy cfg.enc_eng_id = ad->enc_eng.eng_id; 9137694b6caSKeerthy cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 9147694b6caSKeerthy &ctx->dec.cmdl_upd_info); 9157694b6caSKeerthy 9167694b6caSKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 9177694b6caSKeerthy goto badkey; 9187694b6caSKeerthy 9197694b6caSKeerthy ctx->dec.cmdl_size = cmdl_len; 9207694b6caSKeerthy ctx->iv_idx = ad->iv_idx; 9217694b6caSKeerthy 9227694b6caSKeerthy return 0; 9237694b6caSKeerthy 9247694b6caSKeerthy badkey: 9257694b6caSKeerthy dev_err(sa_k3_dev, "%s: badkey\n", __func__); 9267694b6caSKeerthy return -EINVAL; 9277694b6caSKeerthy } 9287694b6caSKeerthy 9297694b6caSKeerthy static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 9307694b6caSKeerthy unsigned int keylen) 9317694b6caSKeerthy { 9327694b6caSKeerthy struct algo_data ad = { 0 }; 9337694b6caSKeerthy /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 9347694b6caSKeerthy int key_idx = (keylen >> 3) - 2; 9357694b6caSKeerthy 9367694b6caSKeerthy if (key_idx >= 3) 9377694b6caSKeerthy return -EINVAL; 9387694b6caSKeerthy 9397694b6caSKeerthy ad.mci_enc = mci_cbc_enc_array[key_idx]; 9407694b6caSKeerthy ad.mci_dec = mci_cbc_dec_array[key_idx]; 9417694b6caSKeerthy ad.inv_key = true; 9427694b6caSKeerthy ad.ealg_id = SA_EALG_ID_AES_CBC; 9437694b6caSKeerthy ad.iv_idx = 4; 9447694b6caSKeerthy ad.iv_out_size = 16; 9457694b6caSKeerthy 9467694b6caSKeerthy return sa_cipher_setkey(tfm, key, keylen, &ad); 9477694b6caSKeerthy } 9487694b6caSKeerthy 9497694b6caSKeerthy static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 9507694b6caSKeerthy unsigned int keylen) 9517694b6caSKeerthy { 9527694b6caSKeerthy struct algo_data ad = { 0 }; 9537694b6caSKeerthy /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 9547694b6caSKeerthy int key_idx = (keylen >> 3) - 2; 9557694b6caSKeerthy 9567694b6caSKeerthy if (key_idx >= 3) 9577694b6caSKeerthy return -EINVAL; 9587694b6caSKeerthy 9597694b6caSKeerthy ad.mci_enc = mci_ecb_enc_array[key_idx]; 9607694b6caSKeerthy ad.mci_dec = mci_ecb_dec_array[key_idx]; 9617694b6caSKeerthy ad.inv_key = true; 9627694b6caSKeerthy ad.ealg_id = SA_EALG_ID_AES_ECB; 9637694b6caSKeerthy 9647694b6caSKeerthy return sa_cipher_setkey(tfm, key, keylen, &ad); 9657694b6caSKeerthy } 9667694b6caSKeerthy 9677694b6caSKeerthy static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key, 9687694b6caSKeerthy unsigned int keylen) 9697694b6caSKeerthy { 9707694b6caSKeerthy struct algo_data ad = { 0 }; 9717694b6caSKeerthy 9727694b6caSKeerthy ad.mci_enc = mci_cbc_3des_enc_array; 9737694b6caSKeerthy ad.mci_dec = mci_cbc_3des_dec_array; 9747694b6caSKeerthy ad.ealg_id = SA_EALG_ID_3DES_CBC; 9757694b6caSKeerthy ad.iv_idx = 6; 9767694b6caSKeerthy ad.iv_out_size = 8; 9777694b6caSKeerthy 9787694b6caSKeerthy return sa_cipher_setkey(tfm, key, keylen, &ad); 9797694b6caSKeerthy } 9807694b6caSKeerthy 9817694b6caSKeerthy static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key, 9827694b6caSKeerthy unsigned int keylen) 9837694b6caSKeerthy { 9847694b6caSKeerthy struct algo_data ad = { 0 }; 9857694b6caSKeerthy 9867694b6caSKeerthy ad.mci_enc = mci_ecb_3des_enc_array; 9877694b6caSKeerthy ad.mci_dec = mci_ecb_3des_dec_array; 9887694b6caSKeerthy 9897694b6caSKeerthy return sa_cipher_setkey(tfm, key, keylen, &ad); 9907694b6caSKeerthy } 9917694b6caSKeerthy 99200c9211fSPeter Ujfalusi static void sa_sync_from_device(struct sa_rx_data *rxd) 99300c9211fSPeter Ujfalusi { 99400c9211fSPeter Ujfalusi struct sg_table *sgt; 99500c9211fSPeter Ujfalusi 99600c9211fSPeter Ujfalusi if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL) 99700c9211fSPeter Ujfalusi sgt = &rxd->mapped_sg[0].sgt; 99800c9211fSPeter Ujfalusi else 99900c9211fSPeter Ujfalusi sgt = &rxd->mapped_sg[1].sgt; 100000c9211fSPeter Ujfalusi 100100c9211fSPeter Ujfalusi dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE); 100200c9211fSPeter Ujfalusi } 100300c9211fSPeter Ujfalusi 100400c9211fSPeter Ujfalusi static void sa_free_sa_rx_data(struct sa_rx_data *rxd) 100500c9211fSPeter Ujfalusi { 100600c9211fSPeter Ujfalusi int i; 100700c9211fSPeter Ujfalusi 100800c9211fSPeter Ujfalusi for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) { 100900c9211fSPeter Ujfalusi struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i]; 101000c9211fSPeter Ujfalusi 101100c9211fSPeter Ujfalusi if (mapped_sg->mapped) { 101200c9211fSPeter Ujfalusi dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt, 101300c9211fSPeter Ujfalusi mapped_sg->dir, 0); 101400c9211fSPeter Ujfalusi kfree(mapped_sg->split_sg); 101500c9211fSPeter Ujfalusi } 101600c9211fSPeter Ujfalusi } 101700c9211fSPeter Ujfalusi 101800c9211fSPeter Ujfalusi kfree(rxd); 101900c9211fSPeter Ujfalusi } 102000c9211fSPeter Ujfalusi 10217694b6caSKeerthy static void sa_aes_dma_in_callback(void *data) 10227694b6caSKeerthy { 10237694b6caSKeerthy struct sa_rx_data *rxd = (struct sa_rx_data *)data; 10247694b6caSKeerthy struct skcipher_request *req; 10257694b6caSKeerthy u32 *result; 10267694b6caSKeerthy __be32 *mdptr; 10277694b6caSKeerthy size_t ml, pl; 10287694b6caSKeerthy int i; 10297694b6caSKeerthy 103000c9211fSPeter Ujfalusi sa_sync_from_device(rxd); 10317694b6caSKeerthy req = container_of(rxd->req, struct skcipher_request, base); 10327694b6caSKeerthy 10337694b6caSKeerthy if (req->iv) { 10347694b6caSKeerthy mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, 10357694b6caSKeerthy &ml); 10367694b6caSKeerthy result = (u32 *)req->iv; 10377694b6caSKeerthy 10387694b6caSKeerthy for (i = 0; i < (rxd->enc_iv_size / 4); i++) 10397694b6caSKeerthy result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]); 10407694b6caSKeerthy } 10417694b6caSKeerthy 104200c9211fSPeter Ujfalusi sa_free_sa_rx_data(rxd); 10437694b6caSKeerthy 10447694b6caSKeerthy skcipher_request_complete(req, 0); 10457694b6caSKeerthy } 10467694b6caSKeerthy 10477694b6caSKeerthy static void 10487694b6caSKeerthy sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib) 10497694b6caSKeerthy { 10507694b6caSKeerthy u32 *out, *in; 10517694b6caSKeerthy int i; 10527694b6caSKeerthy 10537694b6caSKeerthy for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++) 10547694b6caSKeerthy *out++ = *in++; 10557694b6caSKeerthy 10567694b6caSKeerthy mdptr[4] = (0xFFFF << 16); 10577694b6caSKeerthy for (out = &mdptr[5], in = psdata, i = 0; 10587694b6caSKeerthy i < pslen / sizeof(u32); i++) 10597694b6caSKeerthy *out++ = *in++; 10607694b6caSKeerthy } 10617694b6caSKeerthy 10627694b6caSKeerthy static int sa_run(struct sa_req *req) 10637694b6caSKeerthy { 10647694b6caSKeerthy struct sa_rx_data *rxd; 10657694b6caSKeerthy gfp_t gfp_flags; 10667694b6caSKeerthy u32 cmdl[SA_MAX_CMDL_WORDS]; 10677694b6caSKeerthy struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev); 10687694b6caSKeerthy struct device *ddev; 10697694b6caSKeerthy struct dma_chan *dma_rx; 10707694b6caSKeerthy int sg_nents, src_nents, dst_nents; 10717694b6caSKeerthy struct scatterlist *src, *dst; 10727694b6caSKeerthy size_t pl, ml, split_size; 10737694b6caSKeerthy struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec; 10747694b6caSKeerthy int ret; 10757694b6caSKeerthy struct dma_async_tx_descriptor *tx_out; 10767694b6caSKeerthy u32 *mdptr; 10777694b6caSKeerthy bool diff_dst; 10787694b6caSKeerthy enum dma_data_direction dir_src; 107900c9211fSPeter Ujfalusi struct sa_mapped_sg *mapped_sg; 10807694b6caSKeerthy 10817694b6caSKeerthy gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ? 10827694b6caSKeerthy GFP_KERNEL : GFP_ATOMIC; 10837694b6caSKeerthy 10847694b6caSKeerthy rxd = kzalloc(sizeof(*rxd), gfp_flags); 10857694b6caSKeerthy if (!rxd) 10867694b6caSKeerthy return -ENOMEM; 10877694b6caSKeerthy 10887694b6caSKeerthy if (req->src != req->dst) { 10897694b6caSKeerthy diff_dst = true; 10907694b6caSKeerthy dir_src = DMA_TO_DEVICE; 10917694b6caSKeerthy } else { 10927694b6caSKeerthy diff_dst = false; 10937694b6caSKeerthy dir_src = DMA_BIDIRECTIONAL; 10947694b6caSKeerthy } 10957694b6caSKeerthy 10967694b6caSKeerthy /* 10977694b6caSKeerthy * SA2UL has an interesting feature where the receive DMA channel 10987694b6caSKeerthy * is selected based on the data passed to the engine. Within the 10997694b6caSKeerthy * transition range, there is also a space where it is impossible 11007694b6caSKeerthy * to determine where the data will end up, and this should be 11017694b6caSKeerthy * avoided. This will be handled by the SW fallback mechanism by 11027694b6caSKeerthy * the individual algorithm implementations. 11037694b6caSKeerthy */ 11047694b6caSKeerthy if (req->size >= 256) 11057694b6caSKeerthy dma_rx = pdata->dma_rx2; 11067694b6caSKeerthy else 11077694b6caSKeerthy dma_rx = pdata->dma_rx1; 11087694b6caSKeerthy 1109*e8a4529dSPeter Ujfalusi ddev = dmaengine_get_dma_device(pdata->dma_tx); 111000c9211fSPeter Ujfalusi rxd->ddev = ddev; 11117694b6caSKeerthy 11127694b6caSKeerthy memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size); 11137694b6caSKeerthy 11147694b6caSKeerthy sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info); 11157694b6caSKeerthy 11167694b6caSKeerthy if (req->type != CRYPTO_ALG_TYPE_AHASH) { 11177694b6caSKeerthy if (req->enc) 11187694b6caSKeerthy req->type |= 11197694b6caSKeerthy (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT); 11207694b6caSKeerthy else 11217694b6caSKeerthy req->type |= 11227694b6caSKeerthy (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT); 11237694b6caSKeerthy } 11247694b6caSKeerthy 11257694b6caSKeerthy cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type; 11267694b6caSKeerthy 11277694b6caSKeerthy /* 11287694b6caSKeerthy * Map the packets, first we check if the data fits into a single 11297694b6caSKeerthy * sg entry and use that if possible. If it does not fit, we check 11307694b6caSKeerthy * if we need to do sg_split to align the scatterlist data on the 11317694b6caSKeerthy * actual data size being processed by the crypto engine. 11327694b6caSKeerthy */ 11337694b6caSKeerthy src = req->src; 11347694b6caSKeerthy sg_nents = sg_nents_for_len(src, req->size); 11357694b6caSKeerthy 11367694b6caSKeerthy split_size = req->size; 11377694b6caSKeerthy 113800c9211fSPeter Ujfalusi mapped_sg = &rxd->mapped_sg[0]; 11397694b6caSKeerthy if (sg_nents == 1 && split_size <= req->src->length) { 114000c9211fSPeter Ujfalusi src = &mapped_sg->static_sg; 114100c9211fSPeter Ujfalusi src_nents = 1; 11427694b6caSKeerthy sg_init_table(src, 1); 11437694b6caSKeerthy sg_set_page(src, sg_page(req->src), split_size, 11447694b6caSKeerthy req->src->offset); 114500c9211fSPeter Ujfalusi 114600c9211fSPeter Ujfalusi mapped_sg->sgt.sgl = src; 114700c9211fSPeter Ujfalusi mapped_sg->sgt.orig_nents = src_nents; 114800c9211fSPeter Ujfalusi ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); 1149854b7737SColin Ian King if (ret) { 1150854b7737SColin Ian King kfree(rxd); 115100c9211fSPeter Ujfalusi return ret; 1152854b7737SColin Ian King } 115300c9211fSPeter Ujfalusi 115400c9211fSPeter Ujfalusi mapped_sg->dir = dir_src; 115500c9211fSPeter Ujfalusi mapped_sg->mapped = true; 11567694b6caSKeerthy } else { 115700c9211fSPeter Ujfalusi mapped_sg->sgt.sgl = req->src; 115800c9211fSPeter Ujfalusi mapped_sg->sgt.orig_nents = sg_nents; 115900c9211fSPeter Ujfalusi ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0); 1160854b7737SColin Ian King if (ret) { 1161854b7737SColin Ian King kfree(rxd); 116200c9211fSPeter Ujfalusi return ret; 1163854b7737SColin Ian King } 116400c9211fSPeter Ujfalusi 116500c9211fSPeter Ujfalusi mapped_sg->dir = dir_src; 116600c9211fSPeter Ujfalusi mapped_sg->mapped = true; 116700c9211fSPeter Ujfalusi 116800c9211fSPeter Ujfalusi ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1, 116900c9211fSPeter Ujfalusi &split_size, &src, &src_nents, gfp_flags); 11707694b6caSKeerthy if (ret) { 117100c9211fSPeter Ujfalusi src_nents = mapped_sg->sgt.nents; 117200c9211fSPeter Ujfalusi src = mapped_sg->sgt.sgl; 11737694b6caSKeerthy } else { 117400c9211fSPeter Ujfalusi mapped_sg->split_sg = src; 11757694b6caSKeerthy } 11767694b6caSKeerthy } 11777694b6caSKeerthy 117800c9211fSPeter Ujfalusi dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE); 117900c9211fSPeter Ujfalusi 11807694b6caSKeerthy if (!diff_dst) { 11817694b6caSKeerthy dst_nents = src_nents; 11827694b6caSKeerthy dst = src; 11837694b6caSKeerthy } else { 11847694b6caSKeerthy dst_nents = sg_nents_for_len(req->dst, req->size); 118500c9211fSPeter Ujfalusi mapped_sg = &rxd->mapped_sg[1]; 11867694b6caSKeerthy 11877694b6caSKeerthy if (dst_nents == 1 && split_size <= req->dst->length) { 118800c9211fSPeter Ujfalusi dst = &mapped_sg->static_sg; 118900c9211fSPeter Ujfalusi dst_nents = 1; 11907694b6caSKeerthy sg_init_table(dst, 1); 11917694b6caSKeerthy sg_set_page(dst, sg_page(req->dst), split_size, 11927694b6caSKeerthy req->dst->offset); 11937694b6caSKeerthy 119400c9211fSPeter Ujfalusi mapped_sg->sgt.sgl = dst; 119500c9211fSPeter Ujfalusi mapped_sg->sgt.orig_nents = dst_nents; 119600c9211fSPeter Ujfalusi ret = dma_map_sgtable(ddev, &mapped_sg->sgt, 119700c9211fSPeter Ujfalusi DMA_FROM_DEVICE, 0); 119800c9211fSPeter Ujfalusi if (ret) 11997694b6caSKeerthy goto err_cleanup; 120000c9211fSPeter Ujfalusi 120100c9211fSPeter Ujfalusi mapped_sg->dir = DMA_FROM_DEVICE; 120200c9211fSPeter Ujfalusi mapped_sg->mapped = true; 120300c9211fSPeter Ujfalusi } else { 120400c9211fSPeter Ujfalusi mapped_sg->sgt.sgl = req->dst; 120500c9211fSPeter Ujfalusi mapped_sg->sgt.orig_nents = dst_nents; 120600c9211fSPeter Ujfalusi ret = dma_map_sgtable(ddev, &mapped_sg->sgt, 120700c9211fSPeter Ujfalusi DMA_FROM_DEVICE, 0); 120800c9211fSPeter Ujfalusi if (ret) 120900c9211fSPeter Ujfalusi goto err_cleanup; 121000c9211fSPeter Ujfalusi 121100c9211fSPeter Ujfalusi mapped_sg->dir = DMA_FROM_DEVICE; 121200c9211fSPeter Ujfalusi mapped_sg->mapped = true; 121300c9211fSPeter Ujfalusi 121400c9211fSPeter Ujfalusi ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 121500c9211fSPeter Ujfalusi 0, 1, &split_size, &dst, &dst_nents, 121600c9211fSPeter Ujfalusi gfp_flags); 121700c9211fSPeter Ujfalusi if (ret) { 121800c9211fSPeter Ujfalusi dst_nents = mapped_sg->sgt.nents; 121900c9211fSPeter Ujfalusi dst = mapped_sg->sgt.sgl; 122000c9211fSPeter Ujfalusi } else { 122100c9211fSPeter Ujfalusi mapped_sg->split_sg = dst; 122200c9211fSPeter Ujfalusi } 122300c9211fSPeter Ujfalusi } 12247694b6caSKeerthy } 12257694b6caSKeerthy 12267694b6caSKeerthy rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents, 12277694b6caSKeerthy DMA_DEV_TO_MEM, 12287694b6caSKeerthy DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 12297694b6caSKeerthy if (!rxd->tx_in) { 12307694b6caSKeerthy dev_err(pdata->dev, "IN prep_slave_sg() failed\n"); 12317694b6caSKeerthy ret = -EINVAL; 12327694b6caSKeerthy goto err_cleanup; 12337694b6caSKeerthy } 12347694b6caSKeerthy 12357694b6caSKeerthy rxd->req = (void *)req->base; 12367694b6caSKeerthy rxd->enc = req->enc; 12377694b6caSKeerthy rxd->iv_idx = req->ctx->iv_idx; 12387694b6caSKeerthy rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size; 12397694b6caSKeerthy rxd->tx_in->callback = req->callback; 12407694b6caSKeerthy rxd->tx_in->callback_param = rxd; 12417694b6caSKeerthy 12427694b6caSKeerthy tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src, 12437694b6caSKeerthy src_nents, DMA_MEM_TO_DEV, 12447694b6caSKeerthy DMA_PREP_INTERRUPT | DMA_CTRL_ACK); 12457694b6caSKeerthy 12467694b6caSKeerthy if (!tx_out) { 12477694b6caSKeerthy dev_err(pdata->dev, "OUT prep_slave_sg() failed\n"); 12487694b6caSKeerthy ret = -EINVAL; 12497694b6caSKeerthy goto err_cleanup; 12507694b6caSKeerthy } 12517694b6caSKeerthy 12527694b6caSKeerthy /* 12537694b6caSKeerthy * Prepare metadata for DMA engine. This essentially describes the 12547694b6caSKeerthy * crypto algorithm to be used, data sizes, different keys etc. 12557694b6caSKeerthy */ 12567694b6caSKeerthy mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml); 12577694b6caSKeerthy 12587694b6caSKeerthy sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * 12597694b6caSKeerthy sizeof(u32))), cmdl, sizeof(sa_ctx->epib), 12607694b6caSKeerthy sa_ctx->epib); 12617694b6caSKeerthy 12627694b6caSKeerthy ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32)); 12637694b6caSKeerthy dmaengine_desc_set_metadata_len(tx_out, req->mdata_size); 12647694b6caSKeerthy 12657694b6caSKeerthy dmaengine_submit(tx_out); 12667694b6caSKeerthy dmaengine_submit(rxd->tx_in); 12677694b6caSKeerthy 12687694b6caSKeerthy dma_async_issue_pending(dma_rx); 12697694b6caSKeerthy dma_async_issue_pending(pdata->dma_tx); 12707694b6caSKeerthy 12717694b6caSKeerthy return -EINPROGRESS; 12727694b6caSKeerthy 12737694b6caSKeerthy err_cleanup: 127400c9211fSPeter Ujfalusi sa_free_sa_rx_data(rxd); 12757694b6caSKeerthy 12767694b6caSKeerthy return ret; 12777694b6caSKeerthy } 12787694b6caSKeerthy 12797694b6caSKeerthy static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc) 12807694b6caSKeerthy { 12817694b6caSKeerthy struct sa_tfm_ctx *ctx = 12827694b6caSKeerthy crypto_skcipher_ctx(crypto_skcipher_reqtfm(req)); 12837694b6caSKeerthy struct crypto_alg *alg = req->base.tfm->__crt_alg; 12847694b6caSKeerthy struct sa_req sa_req = { 0 }; 12857694b6caSKeerthy 12867694b6caSKeerthy if (!req->cryptlen) 12877694b6caSKeerthy return 0; 12887694b6caSKeerthy 12897694b6caSKeerthy if (req->cryptlen % alg->cra_blocksize) 12907694b6caSKeerthy return -EINVAL; 12917694b6caSKeerthy 12927694b6caSKeerthy /* Use SW fallback if the data size is not supported */ 12937694b6caSKeerthy if (req->cryptlen > SA_MAX_DATA_SZ || 12947694b6caSKeerthy (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN && 12957694b6caSKeerthy req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) { 1296ad0bb4e4SHerbert Xu struct skcipher_request *subreq = skcipher_request_ctx(req); 12977694b6caSKeerthy 1298ad0bb4e4SHerbert Xu skcipher_request_set_tfm(subreq, ctx->fallback.skcipher); 12997694b6caSKeerthy skcipher_request_set_callback(subreq, req->base.flags, 1300ad0bb4e4SHerbert Xu req->base.complete, 1301ad0bb4e4SHerbert Xu req->base.data); 13027694b6caSKeerthy skcipher_request_set_crypt(subreq, req->src, req->dst, 13037694b6caSKeerthy req->cryptlen, req->iv); 13047694b6caSKeerthy if (enc) 1305ad0bb4e4SHerbert Xu return crypto_skcipher_encrypt(subreq); 13067694b6caSKeerthy else 1307ad0bb4e4SHerbert Xu return crypto_skcipher_decrypt(subreq); 13087694b6caSKeerthy } 13097694b6caSKeerthy 13107694b6caSKeerthy sa_req.size = req->cryptlen; 13117694b6caSKeerthy sa_req.enc_size = req->cryptlen; 13127694b6caSKeerthy sa_req.src = req->src; 13137694b6caSKeerthy sa_req.dst = req->dst; 13147694b6caSKeerthy sa_req.enc_iv = iv; 13157694b6caSKeerthy sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER; 13167694b6caSKeerthy sa_req.enc = enc; 13177694b6caSKeerthy sa_req.callback = sa_aes_dma_in_callback; 13187694b6caSKeerthy sa_req.mdata_size = 44; 13197694b6caSKeerthy sa_req.base = &req->base; 13207694b6caSKeerthy sa_req.ctx = ctx; 13217694b6caSKeerthy 13227694b6caSKeerthy return sa_run(&sa_req); 13237694b6caSKeerthy } 13247694b6caSKeerthy 13257694b6caSKeerthy static int sa_encrypt(struct skcipher_request *req) 13267694b6caSKeerthy { 13277694b6caSKeerthy return sa_cipher_run(req, req->iv, 1); 13287694b6caSKeerthy } 13297694b6caSKeerthy 13307694b6caSKeerthy static int sa_decrypt(struct skcipher_request *req) 13317694b6caSKeerthy { 13327694b6caSKeerthy return sa_cipher_run(req, req->iv, 0); 13337694b6caSKeerthy } 13347694b6caSKeerthy 13352dc53d00SKeerthy static void sa_sha_dma_in_callback(void *data) 13362dc53d00SKeerthy { 13372dc53d00SKeerthy struct sa_rx_data *rxd = (struct sa_rx_data *)data; 13382dc53d00SKeerthy struct ahash_request *req; 13392dc53d00SKeerthy struct crypto_ahash *tfm; 13402dc53d00SKeerthy unsigned int authsize; 134100c9211fSPeter Ujfalusi int i; 13422dc53d00SKeerthy size_t ml, pl; 13432dc53d00SKeerthy u32 *result; 13442dc53d00SKeerthy __be32 *mdptr; 13452dc53d00SKeerthy 134600c9211fSPeter Ujfalusi sa_sync_from_device(rxd); 13472dc53d00SKeerthy req = container_of(rxd->req, struct ahash_request, base); 13482dc53d00SKeerthy tfm = crypto_ahash_reqtfm(req); 13492dc53d00SKeerthy authsize = crypto_ahash_digestsize(tfm); 13502dc53d00SKeerthy 13512dc53d00SKeerthy mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 13522dc53d00SKeerthy result = (u32 *)req->result; 13532dc53d00SKeerthy 13542dc53d00SKeerthy for (i = 0; i < (authsize / 4); i++) 13552dc53d00SKeerthy result[i] = be32_to_cpu(mdptr[i + 4]); 13562dc53d00SKeerthy 135700c9211fSPeter Ujfalusi sa_free_sa_rx_data(rxd); 13582dc53d00SKeerthy 13592dc53d00SKeerthy ahash_request_complete(req, 0); 13602dc53d00SKeerthy } 13612dc53d00SKeerthy 13622dc53d00SKeerthy static int zero_message_process(struct ahash_request *req) 13632dc53d00SKeerthy { 13642dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 13652dc53d00SKeerthy int sa_digest_size = crypto_ahash_digestsize(tfm); 13662dc53d00SKeerthy 13672dc53d00SKeerthy switch (sa_digest_size) { 13682dc53d00SKeerthy case SHA1_DIGEST_SIZE: 13692dc53d00SKeerthy memcpy(req->result, sha1_zero_message_hash, sa_digest_size); 13702dc53d00SKeerthy break; 13712dc53d00SKeerthy case SHA256_DIGEST_SIZE: 13722dc53d00SKeerthy memcpy(req->result, sha256_zero_message_hash, sa_digest_size); 13732dc53d00SKeerthy break; 13742dc53d00SKeerthy case SHA512_DIGEST_SIZE: 13752dc53d00SKeerthy memcpy(req->result, sha512_zero_message_hash, sa_digest_size); 13762dc53d00SKeerthy break; 13772dc53d00SKeerthy default: 13782dc53d00SKeerthy return -EINVAL; 13792dc53d00SKeerthy } 13802dc53d00SKeerthy 13812dc53d00SKeerthy return 0; 13822dc53d00SKeerthy } 13832dc53d00SKeerthy 13842dc53d00SKeerthy static int sa_sha_run(struct ahash_request *req) 13852dc53d00SKeerthy { 13862dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req)); 13872dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 13882dc53d00SKeerthy struct sa_req sa_req = { 0 }; 13892dc53d00SKeerthy size_t auth_len; 13902dc53d00SKeerthy 13912dc53d00SKeerthy auth_len = req->nbytes; 13922dc53d00SKeerthy 13932dc53d00SKeerthy if (!auth_len) 13942dc53d00SKeerthy return zero_message_process(req); 13952dc53d00SKeerthy 13962dc53d00SKeerthy if (auth_len > SA_MAX_DATA_SZ || 13972dc53d00SKeerthy (auth_len >= SA_UNSAFE_DATA_SZ_MIN && 13982dc53d00SKeerthy auth_len <= SA_UNSAFE_DATA_SZ_MAX)) { 13992dc53d00SKeerthy struct ahash_request *subreq = &rctx->fallback_req; 14002dc53d00SKeerthy int ret = 0; 14012dc53d00SKeerthy 14022dc53d00SKeerthy ahash_request_set_tfm(subreq, ctx->fallback.ahash); 14032dc53d00SKeerthy subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 14042dc53d00SKeerthy 14052dc53d00SKeerthy crypto_ahash_init(subreq); 14062dc53d00SKeerthy 14072dc53d00SKeerthy subreq->nbytes = auth_len; 14082dc53d00SKeerthy subreq->src = req->src; 14092dc53d00SKeerthy subreq->result = req->result; 14102dc53d00SKeerthy 14112dc53d00SKeerthy ret |= crypto_ahash_update(subreq); 14122dc53d00SKeerthy 14132dc53d00SKeerthy subreq->nbytes = 0; 14142dc53d00SKeerthy 14152dc53d00SKeerthy ret |= crypto_ahash_final(subreq); 14162dc53d00SKeerthy 14172dc53d00SKeerthy return ret; 14182dc53d00SKeerthy } 14192dc53d00SKeerthy 14202dc53d00SKeerthy sa_req.size = auth_len; 14212dc53d00SKeerthy sa_req.auth_size = auth_len; 14222dc53d00SKeerthy sa_req.src = req->src; 14232dc53d00SKeerthy sa_req.dst = req->src; 14242dc53d00SKeerthy sa_req.enc = true; 14252dc53d00SKeerthy sa_req.type = CRYPTO_ALG_TYPE_AHASH; 14262dc53d00SKeerthy sa_req.callback = sa_sha_dma_in_callback; 14272dc53d00SKeerthy sa_req.mdata_size = 28; 14282dc53d00SKeerthy sa_req.ctx = ctx; 14292dc53d00SKeerthy sa_req.base = &req->base; 14302dc53d00SKeerthy 14312dc53d00SKeerthy return sa_run(&sa_req); 14322dc53d00SKeerthy } 14332dc53d00SKeerthy 14342dc53d00SKeerthy static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad) 14352dc53d00SKeerthy { 14362dc53d00SKeerthy int bs = crypto_shash_blocksize(ctx->shash); 14372dc53d00SKeerthy int cmdl_len; 14382dc53d00SKeerthy struct sa_cmdl_cfg cfg; 14392dc53d00SKeerthy 14402dc53d00SKeerthy ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 14412dc53d00SKeerthy ad->auth_eng.eng_id = SA_ENG_ID_AM1; 14422dc53d00SKeerthy ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 14432dc53d00SKeerthy 14442dc53d00SKeerthy memset(ctx->authkey, 0, bs); 14452dc53d00SKeerthy memset(&cfg, 0, sizeof(cfg)); 14462dc53d00SKeerthy cfg.aalg = ad->aalg_id; 14472dc53d00SKeerthy cfg.enc_eng_id = ad->enc_eng.eng_id; 14482dc53d00SKeerthy cfg.auth_eng_id = ad->auth_eng.eng_id; 14492dc53d00SKeerthy cfg.iv_size = 0; 14502dc53d00SKeerthy cfg.akey = NULL; 14512dc53d00SKeerthy cfg.akey_len = 0; 14522dc53d00SKeerthy 14532dc53d00SKeerthy /* Setup Encryption Security Context & Command label template */ 14542dc53d00SKeerthy if (sa_init_sc(&ctx->enc, NULL, 0, NULL, 0, ad, 0, 14552dc53d00SKeerthy &ctx->enc.epib[1])) 14562dc53d00SKeerthy goto badkey; 14572dc53d00SKeerthy 14582dc53d00SKeerthy cmdl_len = sa_format_cmdl_gen(&cfg, 14592dc53d00SKeerthy (u8 *)ctx->enc.cmdl, 14602dc53d00SKeerthy &ctx->enc.cmdl_upd_info); 14612dc53d00SKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 14622dc53d00SKeerthy goto badkey; 14632dc53d00SKeerthy 14642dc53d00SKeerthy ctx->enc.cmdl_size = cmdl_len; 14652dc53d00SKeerthy 14662dc53d00SKeerthy return 0; 14672dc53d00SKeerthy 14682dc53d00SKeerthy badkey: 14692dc53d00SKeerthy dev_err(sa_k3_dev, "%s: badkey\n", __func__); 14702dc53d00SKeerthy return -EINVAL; 14712dc53d00SKeerthy } 14722dc53d00SKeerthy 14732dc53d00SKeerthy static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base) 14742dc53d00SKeerthy { 14752dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 14762dc53d00SKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 14772dc53d00SKeerthy int ret; 14782dc53d00SKeerthy 14792dc53d00SKeerthy memset(ctx, 0, sizeof(*ctx)); 14802dc53d00SKeerthy ctx->dev_data = data; 14812dc53d00SKeerthy ret = sa_init_ctx_info(&ctx->enc, data); 14822dc53d00SKeerthy if (ret) 14832dc53d00SKeerthy return ret; 14842dc53d00SKeerthy 14852dc53d00SKeerthy if (alg_base) { 14862dc53d00SKeerthy ctx->shash = crypto_alloc_shash(alg_base, 0, 14872dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK); 14882dc53d00SKeerthy if (IS_ERR(ctx->shash)) { 14892dc53d00SKeerthy dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", 14902dc53d00SKeerthy alg_base); 14912dc53d00SKeerthy return PTR_ERR(ctx->shash); 14922dc53d00SKeerthy } 14932dc53d00SKeerthy /* for fallback */ 14942dc53d00SKeerthy ctx->fallback.ahash = 14952dc53d00SKeerthy crypto_alloc_ahash(alg_base, 0, 14962dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK); 14972dc53d00SKeerthy if (IS_ERR(ctx->fallback.ahash)) { 14982dc53d00SKeerthy dev_err(ctx->dev_data->dev, 14992dc53d00SKeerthy "Could not load fallback driver\n"); 15002dc53d00SKeerthy return PTR_ERR(ctx->fallback.ahash); 15012dc53d00SKeerthy } 15022dc53d00SKeerthy } 15032dc53d00SKeerthy 15042dc53d00SKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 15052dc53d00SKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 15062dc53d00SKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys); 15072dc53d00SKeerthy 15082dc53d00SKeerthy crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm), 15092dc53d00SKeerthy sizeof(struct sa_sha_req_ctx) + 15102dc53d00SKeerthy crypto_ahash_reqsize(ctx->fallback.ahash)); 15112dc53d00SKeerthy 15122dc53d00SKeerthy return 0; 15132dc53d00SKeerthy } 15142dc53d00SKeerthy 15152dc53d00SKeerthy static int sa_sha_digest(struct ahash_request *req) 15162dc53d00SKeerthy { 15172dc53d00SKeerthy return sa_sha_run(req); 15182dc53d00SKeerthy } 15192dc53d00SKeerthy 15202dc53d00SKeerthy static int sa_sha_init(struct ahash_request *req) 15212dc53d00SKeerthy { 15222dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 15232dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 15242dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 15252dc53d00SKeerthy 1526ea066b7aSYueHaibing dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n", 1527ea066b7aSYueHaibing crypto_ahash_digestsize(tfm), rctx); 15282dc53d00SKeerthy 15292dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 15302dc53d00SKeerthy rctx->fallback_req.base.flags = 15312dc53d00SKeerthy req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 15322dc53d00SKeerthy 15332dc53d00SKeerthy return crypto_ahash_init(&rctx->fallback_req); 15342dc53d00SKeerthy } 15352dc53d00SKeerthy 15362dc53d00SKeerthy static int sa_sha_update(struct ahash_request *req) 15372dc53d00SKeerthy { 15382dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 15392dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 15402dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 15412dc53d00SKeerthy 15422dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 15432dc53d00SKeerthy rctx->fallback_req.base.flags = 15442dc53d00SKeerthy req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 15452dc53d00SKeerthy rctx->fallback_req.nbytes = req->nbytes; 15462dc53d00SKeerthy rctx->fallback_req.src = req->src; 15472dc53d00SKeerthy 15482dc53d00SKeerthy return crypto_ahash_update(&rctx->fallback_req); 15492dc53d00SKeerthy } 15502dc53d00SKeerthy 15512dc53d00SKeerthy static int sa_sha_final(struct ahash_request *req) 15522dc53d00SKeerthy { 15532dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 15542dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 15552dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 15562dc53d00SKeerthy 15572dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 15582dc53d00SKeerthy rctx->fallback_req.base.flags = 15592dc53d00SKeerthy req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 15602dc53d00SKeerthy rctx->fallback_req.result = req->result; 15612dc53d00SKeerthy 15622dc53d00SKeerthy return crypto_ahash_final(&rctx->fallback_req); 15632dc53d00SKeerthy } 15642dc53d00SKeerthy 15652dc53d00SKeerthy static int sa_sha_finup(struct ahash_request *req) 15662dc53d00SKeerthy { 15672dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 15682dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 15692dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 15702dc53d00SKeerthy 15712dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 15722dc53d00SKeerthy rctx->fallback_req.base.flags = 15732dc53d00SKeerthy req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 15742dc53d00SKeerthy 15752dc53d00SKeerthy rctx->fallback_req.nbytes = req->nbytes; 15762dc53d00SKeerthy rctx->fallback_req.src = req->src; 15772dc53d00SKeerthy rctx->fallback_req.result = req->result; 15782dc53d00SKeerthy 15792dc53d00SKeerthy return crypto_ahash_finup(&rctx->fallback_req); 15802dc53d00SKeerthy } 15812dc53d00SKeerthy 15822dc53d00SKeerthy static int sa_sha_import(struct ahash_request *req, const void *in) 15832dc53d00SKeerthy { 15842dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 15852dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 15862dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 15872dc53d00SKeerthy 15882dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash); 15892dc53d00SKeerthy rctx->fallback_req.base.flags = req->base.flags & 15902dc53d00SKeerthy CRYPTO_TFM_REQ_MAY_SLEEP; 15912dc53d00SKeerthy 15922dc53d00SKeerthy return crypto_ahash_import(&rctx->fallback_req, in); 15932dc53d00SKeerthy } 15942dc53d00SKeerthy 15952dc53d00SKeerthy static int sa_sha_export(struct ahash_request *req, void *out) 15962dc53d00SKeerthy { 15972dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req); 15982dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); 15992dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm); 16002dc53d00SKeerthy struct ahash_request *subreq = &rctx->fallback_req; 16012dc53d00SKeerthy 16022dc53d00SKeerthy ahash_request_set_tfm(subreq, ctx->fallback.ahash); 16032dc53d00SKeerthy subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP; 16042dc53d00SKeerthy 16052dc53d00SKeerthy return crypto_ahash_export(subreq, out); 16062dc53d00SKeerthy } 16072dc53d00SKeerthy 16082dc53d00SKeerthy static int sa_sha1_cra_init(struct crypto_tfm *tfm) 16092dc53d00SKeerthy { 16102dc53d00SKeerthy struct algo_data ad = { 0 }; 16112dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 16122dc53d00SKeerthy 16132dc53d00SKeerthy sa_sha_cra_init_alg(tfm, "sha1"); 16142dc53d00SKeerthy 16152dc53d00SKeerthy ad.aalg_id = SA_AALG_ID_SHA1; 16162dc53d00SKeerthy ad.hash_size = SHA1_DIGEST_SIZE; 16172dc53d00SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 16182dc53d00SKeerthy 16192dc53d00SKeerthy sa_sha_setup(ctx, &ad); 16202dc53d00SKeerthy 16212dc53d00SKeerthy return 0; 16222dc53d00SKeerthy } 16232dc53d00SKeerthy 16242dc53d00SKeerthy static int sa_sha256_cra_init(struct crypto_tfm *tfm) 16252dc53d00SKeerthy { 16262dc53d00SKeerthy struct algo_data ad = { 0 }; 16272dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 16282dc53d00SKeerthy 16292dc53d00SKeerthy sa_sha_cra_init_alg(tfm, "sha256"); 16302dc53d00SKeerthy 16312dc53d00SKeerthy ad.aalg_id = SA_AALG_ID_SHA2_256; 16322dc53d00SKeerthy ad.hash_size = SHA256_DIGEST_SIZE; 16332dc53d00SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 16342dc53d00SKeerthy 16352dc53d00SKeerthy sa_sha_setup(ctx, &ad); 16362dc53d00SKeerthy 16372dc53d00SKeerthy return 0; 16382dc53d00SKeerthy } 16392dc53d00SKeerthy 16402dc53d00SKeerthy static int sa_sha512_cra_init(struct crypto_tfm *tfm) 16412dc53d00SKeerthy { 16422dc53d00SKeerthy struct algo_data ad = { 0 }; 16432dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 16442dc53d00SKeerthy 16452dc53d00SKeerthy sa_sha_cra_init_alg(tfm, "sha512"); 16462dc53d00SKeerthy 16472dc53d00SKeerthy ad.aalg_id = SA_AALG_ID_SHA2_512; 16482dc53d00SKeerthy ad.hash_size = SHA512_DIGEST_SIZE; 16492dc53d00SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512; 16502dc53d00SKeerthy 16512dc53d00SKeerthy sa_sha_setup(ctx, &ad); 16522dc53d00SKeerthy 16532dc53d00SKeerthy return 0; 16542dc53d00SKeerthy } 16552dc53d00SKeerthy 16562dc53d00SKeerthy static void sa_sha_cra_exit(struct crypto_tfm *tfm) 16572dc53d00SKeerthy { 16582dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm); 16592dc53d00SKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 16602dc53d00SKeerthy 16612dc53d00SKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 16622dc53d00SKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 16632dc53d00SKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys); 16642dc53d00SKeerthy 16652dc53d00SKeerthy if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH) 16662dc53d00SKeerthy sa_free_ctx_info(&ctx->enc, data); 16672dc53d00SKeerthy 16682dc53d00SKeerthy crypto_free_shash(ctx->shash); 16692dc53d00SKeerthy crypto_free_ahash(ctx->fallback.ahash); 16702dc53d00SKeerthy } 16712dc53d00SKeerthy 1672d2c8ac18SKeerthy static void sa_aead_dma_in_callback(void *data) 1673d2c8ac18SKeerthy { 1674d2c8ac18SKeerthy struct sa_rx_data *rxd = (struct sa_rx_data *)data; 1675d2c8ac18SKeerthy struct aead_request *req; 1676d2c8ac18SKeerthy struct crypto_aead *tfm; 1677d2c8ac18SKeerthy unsigned int start; 1678d2c8ac18SKeerthy unsigned int authsize; 1679d2c8ac18SKeerthy u8 auth_tag[SA_MAX_AUTH_TAG_SZ]; 1680d2c8ac18SKeerthy size_t pl, ml; 168100c9211fSPeter Ujfalusi int i; 1682d2c8ac18SKeerthy int err = 0; 1683d2c8ac18SKeerthy u16 auth_len; 1684d2c8ac18SKeerthy u32 *mdptr; 1685d2c8ac18SKeerthy 168600c9211fSPeter Ujfalusi sa_sync_from_device(rxd); 1687d2c8ac18SKeerthy req = container_of(rxd->req, struct aead_request, base); 1688d2c8ac18SKeerthy tfm = crypto_aead_reqtfm(req); 1689d2c8ac18SKeerthy start = req->assoclen + req->cryptlen; 1690d2c8ac18SKeerthy authsize = crypto_aead_authsize(tfm); 1691d2c8ac18SKeerthy 1692d2c8ac18SKeerthy mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml); 1693d2c8ac18SKeerthy for (i = 0; i < (authsize / 4); i++) 1694d2c8ac18SKeerthy mdptr[i + 4] = swab32(mdptr[i + 4]); 1695d2c8ac18SKeerthy 1696d2c8ac18SKeerthy auth_len = req->assoclen + req->cryptlen; 1697d2c8ac18SKeerthy 1698d2c8ac18SKeerthy if (rxd->enc) { 1699d2c8ac18SKeerthy scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize, 1700d2c8ac18SKeerthy 1); 1701d2c8ac18SKeerthy } else { 170200c9211fSPeter Ujfalusi auth_len -= authsize; 1703d2c8ac18SKeerthy start -= authsize; 1704d2c8ac18SKeerthy scatterwalk_map_and_copy(auth_tag, req->src, start, authsize, 1705d2c8ac18SKeerthy 0); 1706d2c8ac18SKeerthy 1707d2c8ac18SKeerthy err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0; 1708d2c8ac18SKeerthy } 1709d2c8ac18SKeerthy 171000c9211fSPeter Ujfalusi sa_free_sa_rx_data(rxd); 1711d2c8ac18SKeerthy 1712d2c8ac18SKeerthy aead_request_complete(req, err); 1713d2c8ac18SKeerthy } 1714d2c8ac18SKeerthy 1715d2c8ac18SKeerthy static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash, 1716d2c8ac18SKeerthy const char *fallback) 1717d2c8ac18SKeerthy { 1718d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1719d2c8ac18SKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1720d2c8ac18SKeerthy int ret; 1721d2c8ac18SKeerthy 1722d2c8ac18SKeerthy memzero_explicit(ctx, sizeof(*ctx)); 1723d2c8ac18SKeerthy 1724d2c8ac18SKeerthy ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK); 1725d2c8ac18SKeerthy if (IS_ERR(ctx->shash)) { 1726d2c8ac18SKeerthy dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash); 1727d2c8ac18SKeerthy return PTR_ERR(ctx->shash); 1728d2c8ac18SKeerthy } 1729d2c8ac18SKeerthy 1730d2c8ac18SKeerthy ctx->fallback.aead = crypto_alloc_aead(fallback, 0, 1731d2c8ac18SKeerthy CRYPTO_ALG_NEED_FALLBACK); 1732d2c8ac18SKeerthy 1733d2c8ac18SKeerthy if (IS_ERR(ctx->fallback.aead)) { 1734d2c8ac18SKeerthy dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n", 1735d2c8ac18SKeerthy fallback); 1736d2c8ac18SKeerthy return PTR_ERR(ctx->fallback.aead); 1737d2c8ac18SKeerthy } 1738d2c8ac18SKeerthy 1739d2c8ac18SKeerthy crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) + 1740d2c8ac18SKeerthy crypto_aead_reqsize(ctx->fallback.aead)); 1741d2c8ac18SKeerthy 1742d2c8ac18SKeerthy ret = sa_init_ctx_info(&ctx->enc, data); 1743d2c8ac18SKeerthy if (ret) 1744d2c8ac18SKeerthy return ret; 1745d2c8ac18SKeerthy 1746d2c8ac18SKeerthy ret = sa_init_ctx_info(&ctx->dec, data); 1747d2c8ac18SKeerthy if (ret) { 1748d2c8ac18SKeerthy sa_free_ctx_info(&ctx->enc, data); 1749d2c8ac18SKeerthy return ret; 1750d2c8ac18SKeerthy } 1751d2c8ac18SKeerthy 1752d2c8ac18SKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n", 1753d2c8ac18SKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys, 1754d2c8ac18SKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys); 1755d2c8ac18SKeerthy 1756d2c8ac18SKeerthy return ret; 1757d2c8ac18SKeerthy } 1758d2c8ac18SKeerthy 1759d2c8ac18SKeerthy static int sa_cra_init_aead_sha1(struct crypto_aead *tfm) 1760d2c8ac18SKeerthy { 1761d2c8ac18SKeerthy return sa_cra_init_aead(tfm, "sha1", 1762d2c8ac18SKeerthy "authenc(hmac(sha1-ce),cbc(aes-ce))"); 1763d2c8ac18SKeerthy } 1764d2c8ac18SKeerthy 1765d2c8ac18SKeerthy static int sa_cra_init_aead_sha256(struct crypto_aead *tfm) 1766d2c8ac18SKeerthy { 1767d2c8ac18SKeerthy return sa_cra_init_aead(tfm, "sha256", 1768d2c8ac18SKeerthy "authenc(hmac(sha256-ce),cbc(aes-ce))"); 1769d2c8ac18SKeerthy } 1770d2c8ac18SKeerthy 1771d2c8ac18SKeerthy static void sa_exit_tfm_aead(struct crypto_aead *tfm) 1772d2c8ac18SKeerthy { 1773d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1774d2c8ac18SKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev); 1775d2c8ac18SKeerthy 1776d2c8ac18SKeerthy crypto_free_shash(ctx->shash); 1777d2c8ac18SKeerthy crypto_free_aead(ctx->fallback.aead); 1778d2c8ac18SKeerthy 1779d2c8ac18SKeerthy sa_free_ctx_info(&ctx->enc, data); 1780d2c8ac18SKeerthy sa_free_ctx_info(&ctx->dec, data); 1781d2c8ac18SKeerthy } 1782d2c8ac18SKeerthy 1783d2c8ac18SKeerthy /* AEAD algorithm configuration interface function */ 1784d2c8ac18SKeerthy static int sa_aead_setkey(struct crypto_aead *authenc, 1785d2c8ac18SKeerthy const u8 *key, unsigned int keylen, 1786d2c8ac18SKeerthy struct algo_data *ad) 1787d2c8ac18SKeerthy { 1788d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc); 1789d2c8ac18SKeerthy struct crypto_authenc_keys keys; 1790d2c8ac18SKeerthy int cmdl_len; 1791d2c8ac18SKeerthy struct sa_cmdl_cfg cfg; 1792d2c8ac18SKeerthy int key_idx; 1793d2c8ac18SKeerthy 1794d2c8ac18SKeerthy if (crypto_authenc_extractkeys(&keys, key, keylen) != 0) 1795d2c8ac18SKeerthy return -EINVAL; 1796d2c8ac18SKeerthy 1797d2c8ac18SKeerthy /* Convert the key size (16/24/32) to the key size index (0/1/2) */ 1798d2c8ac18SKeerthy key_idx = (keys.enckeylen >> 3) - 2; 1799d2c8ac18SKeerthy if (key_idx >= 3) 1800d2c8ac18SKeerthy return -EINVAL; 1801d2c8ac18SKeerthy 1802d2c8ac18SKeerthy ad->ctx = ctx; 1803d2c8ac18SKeerthy ad->enc_eng.eng_id = SA_ENG_ID_EM1; 1804d2c8ac18SKeerthy ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ; 1805d2c8ac18SKeerthy ad->auth_eng.eng_id = SA_ENG_ID_AM1; 1806d2c8ac18SKeerthy ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ; 1807d2c8ac18SKeerthy ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx]; 1808d2c8ac18SKeerthy ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx]; 1809d2c8ac18SKeerthy ad->inv_key = true; 1810d2c8ac18SKeerthy ad->keyed_mac = true; 1811d2c8ac18SKeerthy ad->ealg_id = SA_EALG_ID_AES_CBC; 1812d2c8ac18SKeerthy ad->prep_iopad = sa_prepare_iopads; 1813d2c8ac18SKeerthy 1814d2c8ac18SKeerthy memset(&cfg, 0, sizeof(cfg)); 1815d2c8ac18SKeerthy cfg.enc = true; 1816d2c8ac18SKeerthy cfg.aalg = ad->aalg_id; 1817d2c8ac18SKeerthy cfg.enc_eng_id = ad->enc_eng.eng_id; 1818d2c8ac18SKeerthy cfg.auth_eng_id = ad->auth_eng.eng_id; 1819d2c8ac18SKeerthy cfg.iv_size = crypto_aead_ivsize(authenc); 1820d2c8ac18SKeerthy cfg.akey = keys.authkey; 1821d2c8ac18SKeerthy cfg.akey_len = keys.authkeylen; 1822d2c8ac18SKeerthy 1823d2c8ac18SKeerthy /* Setup Encryption Security Context & Command label template */ 1824d2c8ac18SKeerthy if (sa_init_sc(&ctx->enc, keys.enckey, keys.enckeylen, 1825d2c8ac18SKeerthy keys.authkey, keys.authkeylen, 1826d2c8ac18SKeerthy ad, 1, &ctx->enc.epib[1])) 1827d2c8ac18SKeerthy return -EINVAL; 1828d2c8ac18SKeerthy 1829d2c8ac18SKeerthy cmdl_len = sa_format_cmdl_gen(&cfg, 1830d2c8ac18SKeerthy (u8 *)ctx->enc.cmdl, 1831d2c8ac18SKeerthy &ctx->enc.cmdl_upd_info); 1832d2c8ac18SKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1833d2c8ac18SKeerthy return -EINVAL; 1834d2c8ac18SKeerthy 1835d2c8ac18SKeerthy ctx->enc.cmdl_size = cmdl_len; 1836d2c8ac18SKeerthy 1837d2c8ac18SKeerthy /* Setup Decryption Security Context & Command label template */ 1838d2c8ac18SKeerthy if (sa_init_sc(&ctx->dec, keys.enckey, keys.enckeylen, 1839d2c8ac18SKeerthy keys.authkey, keys.authkeylen, 1840d2c8ac18SKeerthy ad, 0, &ctx->dec.epib[1])) 1841d2c8ac18SKeerthy return -EINVAL; 1842d2c8ac18SKeerthy 1843d2c8ac18SKeerthy cfg.enc = false; 1844d2c8ac18SKeerthy cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl, 1845d2c8ac18SKeerthy &ctx->dec.cmdl_upd_info); 1846d2c8ac18SKeerthy 1847d2c8ac18SKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32))) 1848d2c8ac18SKeerthy return -EINVAL; 1849d2c8ac18SKeerthy 1850d2c8ac18SKeerthy ctx->dec.cmdl_size = cmdl_len; 1851d2c8ac18SKeerthy 1852d2c8ac18SKeerthy crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK); 1853d2c8ac18SKeerthy crypto_aead_set_flags(ctx->fallback.aead, 1854d2c8ac18SKeerthy crypto_aead_get_flags(authenc) & 1855d2c8ac18SKeerthy CRYPTO_TFM_REQ_MASK); 1856d2c8ac18SKeerthy crypto_aead_setkey(ctx->fallback.aead, key, keylen); 1857d2c8ac18SKeerthy 1858d2c8ac18SKeerthy return 0; 1859d2c8ac18SKeerthy } 1860d2c8ac18SKeerthy 1861d2c8ac18SKeerthy static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize) 1862d2c8ac18SKeerthy { 1863d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm)); 1864d2c8ac18SKeerthy 1865d2c8ac18SKeerthy return crypto_aead_setauthsize(ctx->fallback.aead, authsize); 1866d2c8ac18SKeerthy } 1867d2c8ac18SKeerthy 1868d2c8ac18SKeerthy static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc, 1869d2c8ac18SKeerthy const u8 *key, unsigned int keylen) 1870d2c8ac18SKeerthy { 1871d2c8ac18SKeerthy struct algo_data ad = { 0 }; 1872d2c8ac18SKeerthy 1873d2c8ac18SKeerthy ad.ealg_id = SA_EALG_ID_AES_CBC; 1874d2c8ac18SKeerthy ad.aalg_id = SA_AALG_ID_HMAC_SHA1; 1875d2c8ac18SKeerthy ad.hash_size = SHA1_DIGEST_SIZE; 1876d2c8ac18SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1; 1877d2c8ac18SKeerthy 1878d2c8ac18SKeerthy return sa_aead_setkey(authenc, key, keylen, &ad); 1879d2c8ac18SKeerthy } 1880d2c8ac18SKeerthy 1881d2c8ac18SKeerthy static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc, 1882d2c8ac18SKeerthy const u8 *key, unsigned int keylen) 1883d2c8ac18SKeerthy { 1884d2c8ac18SKeerthy struct algo_data ad = { 0 }; 1885d2c8ac18SKeerthy 1886d2c8ac18SKeerthy ad.ealg_id = SA_EALG_ID_AES_CBC; 1887d2c8ac18SKeerthy ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256; 1888d2c8ac18SKeerthy ad.hash_size = SHA256_DIGEST_SIZE; 1889d2c8ac18SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256; 1890d2c8ac18SKeerthy 1891d2c8ac18SKeerthy return sa_aead_setkey(authenc, key, keylen, &ad); 1892d2c8ac18SKeerthy } 1893d2c8ac18SKeerthy 1894d2c8ac18SKeerthy static int sa_aead_run(struct aead_request *req, u8 *iv, int enc) 1895d2c8ac18SKeerthy { 1896d2c8ac18SKeerthy struct crypto_aead *tfm = crypto_aead_reqtfm(req); 1897d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm); 1898d2c8ac18SKeerthy struct sa_req sa_req = { 0 }; 1899d2c8ac18SKeerthy size_t auth_size, enc_size; 1900d2c8ac18SKeerthy 1901d2c8ac18SKeerthy enc_size = req->cryptlen; 1902d2c8ac18SKeerthy auth_size = req->assoclen + req->cryptlen; 1903d2c8ac18SKeerthy 1904d2c8ac18SKeerthy if (!enc) { 1905d2c8ac18SKeerthy enc_size -= crypto_aead_authsize(tfm); 1906d2c8ac18SKeerthy auth_size -= crypto_aead_authsize(tfm); 1907d2c8ac18SKeerthy } 1908d2c8ac18SKeerthy 1909d2c8ac18SKeerthy if (auth_size > SA_MAX_DATA_SZ || 1910d2c8ac18SKeerthy (auth_size >= SA_UNSAFE_DATA_SZ_MIN && 1911d2c8ac18SKeerthy auth_size <= SA_UNSAFE_DATA_SZ_MAX)) { 1912d2c8ac18SKeerthy struct aead_request *subreq = aead_request_ctx(req); 1913d2c8ac18SKeerthy int ret; 1914d2c8ac18SKeerthy 1915d2c8ac18SKeerthy aead_request_set_tfm(subreq, ctx->fallback.aead); 1916d2c8ac18SKeerthy aead_request_set_callback(subreq, req->base.flags, 1917d2c8ac18SKeerthy req->base.complete, req->base.data); 1918d2c8ac18SKeerthy aead_request_set_crypt(subreq, req->src, req->dst, 1919d2c8ac18SKeerthy req->cryptlen, req->iv); 1920d2c8ac18SKeerthy aead_request_set_ad(subreq, req->assoclen); 1921d2c8ac18SKeerthy 1922d2c8ac18SKeerthy ret = enc ? crypto_aead_encrypt(subreq) : 1923d2c8ac18SKeerthy crypto_aead_decrypt(subreq); 1924d2c8ac18SKeerthy return ret; 1925d2c8ac18SKeerthy } 1926d2c8ac18SKeerthy 1927d2c8ac18SKeerthy sa_req.enc_offset = req->assoclen; 1928d2c8ac18SKeerthy sa_req.enc_size = enc_size; 1929d2c8ac18SKeerthy sa_req.auth_size = auth_size; 1930d2c8ac18SKeerthy sa_req.size = auth_size; 1931d2c8ac18SKeerthy sa_req.enc_iv = iv; 1932d2c8ac18SKeerthy sa_req.type = CRYPTO_ALG_TYPE_AEAD; 1933d2c8ac18SKeerthy sa_req.enc = enc; 1934d2c8ac18SKeerthy sa_req.callback = sa_aead_dma_in_callback; 1935d2c8ac18SKeerthy sa_req.mdata_size = 52; 1936d2c8ac18SKeerthy sa_req.base = &req->base; 1937d2c8ac18SKeerthy sa_req.ctx = ctx; 1938d2c8ac18SKeerthy sa_req.src = req->src; 1939d2c8ac18SKeerthy sa_req.dst = req->dst; 1940d2c8ac18SKeerthy 1941d2c8ac18SKeerthy return sa_run(&sa_req); 1942d2c8ac18SKeerthy } 1943d2c8ac18SKeerthy 1944d2c8ac18SKeerthy /* AEAD algorithm encrypt interface function */ 1945d2c8ac18SKeerthy static int sa_aead_encrypt(struct aead_request *req) 1946d2c8ac18SKeerthy { 1947d2c8ac18SKeerthy return sa_aead_run(req, req->iv, 1); 1948d2c8ac18SKeerthy } 1949d2c8ac18SKeerthy 1950d2c8ac18SKeerthy /* AEAD algorithm decrypt interface function */ 1951d2c8ac18SKeerthy static int sa_aead_decrypt(struct aead_request *req) 1952d2c8ac18SKeerthy { 1953d2c8ac18SKeerthy return sa_aead_run(req, req->iv, 0); 1954d2c8ac18SKeerthy } 1955d2c8ac18SKeerthy 19567694b6caSKeerthy static struct sa_alg_tmpl sa_algs[] = { 19577694b6caSKeerthy { 19587694b6caSKeerthy .type = CRYPTO_ALG_TYPE_SKCIPHER, 19597694b6caSKeerthy .alg.skcipher = { 19607694b6caSKeerthy .base.cra_name = "cbc(aes)", 19617694b6caSKeerthy .base.cra_driver_name = "cbc-aes-sa2ul", 19627694b6caSKeerthy .base.cra_priority = 30000, 19637694b6caSKeerthy .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 19647694b6caSKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY | 19657694b6caSKeerthy CRYPTO_ALG_ASYNC | 19667694b6caSKeerthy CRYPTO_ALG_NEED_FALLBACK, 19677694b6caSKeerthy .base.cra_blocksize = AES_BLOCK_SIZE, 19687694b6caSKeerthy .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 19697694b6caSKeerthy .base.cra_module = THIS_MODULE, 19707694b6caSKeerthy .init = sa_cipher_cra_init, 19717694b6caSKeerthy .exit = sa_cipher_cra_exit, 19727694b6caSKeerthy .min_keysize = AES_MIN_KEY_SIZE, 19737694b6caSKeerthy .max_keysize = AES_MAX_KEY_SIZE, 19747694b6caSKeerthy .ivsize = AES_BLOCK_SIZE, 19757694b6caSKeerthy .setkey = sa_aes_cbc_setkey, 19767694b6caSKeerthy .encrypt = sa_encrypt, 19777694b6caSKeerthy .decrypt = sa_decrypt, 19787694b6caSKeerthy } 19797694b6caSKeerthy }, 19807694b6caSKeerthy { 19817694b6caSKeerthy .type = CRYPTO_ALG_TYPE_SKCIPHER, 19827694b6caSKeerthy .alg.skcipher = { 19837694b6caSKeerthy .base.cra_name = "ecb(aes)", 19847694b6caSKeerthy .base.cra_driver_name = "ecb-aes-sa2ul", 19857694b6caSKeerthy .base.cra_priority = 30000, 19867694b6caSKeerthy .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 19877694b6caSKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY | 19887694b6caSKeerthy CRYPTO_ALG_ASYNC | 19897694b6caSKeerthy CRYPTO_ALG_NEED_FALLBACK, 19907694b6caSKeerthy .base.cra_blocksize = AES_BLOCK_SIZE, 19917694b6caSKeerthy .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 19927694b6caSKeerthy .base.cra_module = THIS_MODULE, 19937694b6caSKeerthy .init = sa_cipher_cra_init, 19947694b6caSKeerthy .exit = sa_cipher_cra_exit, 19957694b6caSKeerthy .min_keysize = AES_MIN_KEY_SIZE, 19967694b6caSKeerthy .max_keysize = AES_MAX_KEY_SIZE, 19977694b6caSKeerthy .setkey = sa_aes_ecb_setkey, 19987694b6caSKeerthy .encrypt = sa_encrypt, 19997694b6caSKeerthy .decrypt = sa_decrypt, 20007694b6caSKeerthy } 20017694b6caSKeerthy }, 20027694b6caSKeerthy { 20037694b6caSKeerthy .type = CRYPTO_ALG_TYPE_SKCIPHER, 20047694b6caSKeerthy .alg.skcipher = { 20057694b6caSKeerthy .base.cra_name = "cbc(des3_ede)", 20067694b6caSKeerthy .base.cra_driver_name = "cbc-des3-sa2ul", 20077694b6caSKeerthy .base.cra_priority = 30000, 20087694b6caSKeerthy .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 20097694b6caSKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY | 20107694b6caSKeerthy CRYPTO_ALG_ASYNC | 20117694b6caSKeerthy CRYPTO_ALG_NEED_FALLBACK, 20127694b6caSKeerthy .base.cra_blocksize = DES_BLOCK_SIZE, 20137694b6caSKeerthy .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 20147694b6caSKeerthy .base.cra_module = THIS_MODULE, 20157694b6caSKeerthy .init = sa_cipher_cra_init, 20167694b6caSKeerthy .exit = sa_cipher_cra_exit, 20177694b6caSKeerthy .min_keysize = 3 * DES_KEY_SIZE, 20187694b6caSKeerthy .max_keysize = 3 * DES_KEY_SIZE, 20197694b6caSKeerthy .ivsize = DES_BLOCK_SIZE, 20207694b6caSKeerthy .setkey = sa_3des_cbc_setkey, 20217694b6caSKeerthy .encrypt = sa_encrypt, 20227694b6caSKeerthy .decrypt = sa_decrypt, 20237694b6caSKeerthy } 20247694b6caSKeerthy }, 20257694b6caSKeerthy { 20267694b6caSKeerthy .type = CRYPTO_ALG_TYPE_SKCIPHER, 20277694b6caSKeerthy .alg.skcipher = { 20287694b6caSKeerthy .base.cra_name = "ecb(des3_ede)", 20297694b6caSKeerthy .base.cra_driver_name = "ecb-des3-sa2ul", 20307694b6caSKeerthy .base.cra_priority = 30000, 20317694b6caSKeerthy .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER | 20327694b6caSKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY | 20337694b6caSKeerthy CRYPTO_ALG_ASYNC | 20347694b6caSKeerthy CRYPTO_ALG_NEED_FALLBACK, 20357694b6caSKeerthy .base.cra_blocksize = DES_BLOCK_SIZE, 20367694b6caSKeerthy .base.cra_ctxsize = sizeof(struct sa_tfm_ctx), 20377694b6caSKeerthy .base.cra_module = THIS_MODULE, 20387694b6caSKeerthy .init = sa_cipher_cra_init, 20397694b6caSKeerthy .exit = sa_cipher_cra_exit, 20407694b6caSKeerthy .min_keysize = 3 * DES_KEY_SIZE, 20417694b6caSKeerthy .max_keysize = 3 * DES_KEY_SIZE, 20427694b6caSKeerthy .setkey = sa_3des_ecb_setkey, 20437694b6caSKeerthy .encrypt = sa_encrypt, 20447694b6caSKeerthy .decrypt = sa_decrypt, 20457694b6caSKeerthy } 20467694b6caSKeerthy }, 20472dc53d00SKeerthy { 20482dc53d00SKeerthy .type = CRYPTO_ALG_TYPE_AHASH, 20492dc53d00SKeerthy .alg.ahash = { 20502dc53d00SKeerthy .halg.base = { 20512dc53d00SKeerthy .cra_name = "sha1", 20522dc53d00SKeerthy .cra_driver_name = "sha1-sa2ul", 20532dc53d00SKeerthy .cra_priority = 400, 20542dc53d00SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AHASH | 20552dc53d00SKeerthy CRYPTO_ALG_ASYNC | 20562dc53d00SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY | 20572dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK, 20582dc53d00SKeerthy .cra_blocksize = SHA1_BLOCK_SIZE, 20592dc53d00SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx), 20602dc53d00SKeerthy .cra_module = THIS_MODULE, 20612dc53d00SKeerthy .cra_init = sa_sha1_cra_init, 20622dc53d00SKeerthy .cra_exit = sa_sha_cra_exit, 20632dc53d00SKeerthy }, 20642dc53d00SKeerthy .halg.digestsize = SHA1_DIGEST_SIZE, 20652dc53d00SKeerthy .halg.statesize = sizeof(struct sa_sha_req_ctx) + 20662dc53d00SKeerthy sizeof(struct sha1_state), 20672dc53d00SKeerthy .init = sa_sha_init, 20682dc53d00SKeerthy .update = sa_sha_update, 20692dc53d00SKeerthy .final = sa_sha_final, 20702dc53d00SKeerthy .finup = sa_sha_finup, 20712dc53d00SKeerthy .digest = sa_sha_digest, 20722dc53d00SKeerthy .export = sa_sha_export, 20732dc53d00SKeerthy .import = sa_sha_import, 20742dc53d00SKeerthy }, 20752dc53d00SKeerthy }, 20762dc53d00SKeerthy { 20772dc53d00SKeerthy .type = CRYPTO_ALG_TYPE_AHASH, 20782dc53d00SKeerthy .alg.ahash = { 20792dc53d00SKeerthy .halg.base = { 20802dc53d00SKeerthy .cra_name = "sha256", 20812dc53d00SKeerthy .cra_driver_name = "sha256-sa2ul", 20822dc53d00SKeerthy .cra_priority = 400, 20832dc53d00SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AHASH | 20842dc53d00SKeerthy CRYPTO_ALG_ASYNC | 20852dc53d00SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY | 20862dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK, 20872dc53d00SKeerthy .cra_blocksize = SHA256_BLOCK_SIZE, 20882dc53d00SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx), 20892dc53d00SKeerthy .cra_module = THIS_MODULE, 20902dc53d00SKeerthy .cra_init = sa_sha256_cra_init, 20912dc53d00SKeerthy .cra_exit = sa_sha_cra_exit, 20922dc53d00SKeerthy }, 20932dc53d00SKeerthy .halg.digestsize = SHA256_DIGEST_SIZE, 20942dc53d00SKeerthy .halg.statesize = sizeof(struct sa_sha_req_ctx) + 20952dc53d00SKeerthy sizeof(struct sha256_state), 20962dc53d00SKeerthy .init = sa_sha_init, 20972dc53d00SKeerthy .update = sa_sha_update, 20982dc53d00SKeerthy .final = sa_sha_final, 20992dc53d00SKeerthy .finup = sa_sha_finup, 21002dc53d00SKeerthy .digest = sa_sha_digest, 21012dc53d00SKeerthy .export = sa_sha_export, 21022dc53d00SKeerthy .import = sa_sha_import, 21032dc53d00SKeerthy }, 21042dc53d00SKeerthy }, 21052dc53d00SKeerthy { 21062dc53d00SKeerthy .type = CRYPTO_ALG_TYPE_AHASH, 21072dc53d00SKeerthy .alg.ahash = { 21082dc53d00SKeerthy .halg.base = { 21092dc53d00SKeerthy .cra_name = "sha512", 21102dc53d00SKeerthy .cra_driver_name = "sha512-sa2ul", 21112dc53d00SKeerthy .cra_priority = 400, 21122dc53d00SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AHASH | 21132dc53d00SKeerthy CRYPTO_ALG_ASYNC | 21142dc53d00SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY | 21152dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK, 21162dc53d00SKeerthy .cra_blocksize = SHA512_BLOCK_SIZE, 21172dc53d00SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx), 21182dc53d00SKeerthy .cra_module = THIS_MODULE, 21192dc53d00SKeerthy .cra_init = sa_sha512_cra_init, 21202dc53d00SKeerthy .cra_exit = sa_sha_cra_exit, 21212dc53d00SKeerthy }, 21222dc53d00SKeerthy .halg.digestsize = SHA512_DIGEST_SIZE, 21232dc53d00SKeerthy .halg.statesize = sizeof(struct sa_sha_req_ctx) + 21242dc53d00SKeerthy sizeof(struct sha512_state), 21252dc53d00SKeerthy .init = sa_sha_init, 21262dc53d00SKeerthy .update = sa_sha_update, 21272dc53d00SKeerthy .final = sa_sha_final, 21282dc53d00SKeerthy .finup = sa_sha_finup, 21292dc53d00SKeerthy .digest = sa_sha_digest, 21302dc53d00SKeerthy .export = sa_sha_export, 21312dc53d00SKeerthy .import = sa_sha_import, 21322dc53d00SKeerthy }, 21332dc53d00SKeerthy }, 2134d2c8ac18SKeerthy { 2135d2c8ac18SKeerthy .type = CRYPTO_ALG_TYPE_AEAD, 2136d2c8ac18SKeerthy .alg.aead = { 2137d2c8ac18SKeerthy .base = { 2138d2c8ac18SKeerthy .cra_name = "authenc(hmac(sha1),cbc(aes))", 2139d2c8ac18SKeerthy .cra_driver_name = 2140d2c8ac18SKeerthy "authenc(hmac(sha1),cbc(aes))-sa2ul", 2141d2c8ac18SKeerthy .cra_blocksize = AES_BLOCK_SIZE, 2142d2c8ac18SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2143d2c8ac18SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY | 2144d2c8ac18SKeerthy CRYPTO_ALG_ASYNC | 2145d2c8ac18SKeerthy CRYPTO_ALG_NEED_FALLBACK, 2146d2c8ac18SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2147d2c8ac18SKeerthy .cra_module = THIS_MODULE, 2148d2c8ac18SKeerthy .cra_priority = 3000, 2149d2c8ac18SKeerthy }, 2150d2c8ac18SKeerthy .ivsize = AES_BLOCK_SIZE, 2151d2c8ac18SKeerthy .maxauthsize = SHA1_DIGEST_SIZE, 2152d2c8ac18SKeerthy 2153d2c8ac18SKeerthy .init = sa_cra_init_aead_sha1, 2154d2c8ac18SKeerthy .exit = sa_exit_tfm_aead, 2155d2c8ac18SKeerthy .setkey = sa_aead_cbc_sha1_setkey, 2156d2c8ac18SKeerthy .setauthsize = sa_aead_setauthsize, 2157d2c8ac18SKeerthy .encrypt = sa_aead_encrypt, 2158d2c8ac18SKeerthy .decrypt = sa_aead_decrypt, 2159d2c8ac18SKeerthy }, 2160d2c8ac18SKeerthy }, 2161d2c8ac18SKeerthy { 2162d2c8ac18SKeerthy .type = CRYPTO_ALG_TYPE_AEAD, 2163d2c8ac18SKeerthy .alg.aead = { 2164d2c8ac18SKeerthy .base = { 2165d2c8ac18SKeerthy .cra_name = "authenc(hmac(sha256),cbc(aes))", 2166d2c8ac18SKeerthy .cra_driver_name = 2167d2c8ac18SKeerthy "authenc(hmac(sha256),cbc(aes))-sa2ul", 2168d2c8ac18SKeerthy .cra_blocksize = AES_BLOCK_SIZE, 2169d2c8ac18SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AEAD | 2170d2c8ac18SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY | 2171d2c8ac18SKeerthy CRYPTO_ALG_ASYNC | 2172d2c8ac18SKeerthy CRYPTO_ALG_NEED_FALLBACK, 2173d2c8ac18SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx), 2174d2c8ac18SKeerthy .cra_module = THIS_MODULE, 2175d2c8ac18SKeerthy .cra_alignmask = 0, 2176d2c8ac18SKeerthy .cra_priority = 3000, 2177d2c8ac18SKeerthy }, 2178d2c8ac18SKeerthy .ivsize = AES_BLOCK_SIZE, 2179d2c8ac18SKeerthy .maxauthsize = SHA256_DIGEST_SIZE, 2180d2c8ac18SKeerthy 2181d2c8ac18SKeerthy .init = sa_cra_init_aead_sha256, 2182d2c8ac18SKeerthy .exit = sa_exit_tfm_aead, 2183d2c8ac18SKeerthy .setkey = sa_aead_cbc_sha256_setkey, 2184d2c8ac18SKeerthy .setauthsize = sa_aead_setauthsize, 2185d2c8ac18SKeerthy .encrypt = sa_aead_encrypt, 2186d2c8ac18SKeerthy .decrypt = sa_aead_decrypt, 2187d2c8ac18SKeerthy }, 2188d2c8ac18SKeerthy }, 21897694b6caSKeerthy }; 21907694b6caSKeerthy 21917694b6caSKeerthy /* Register the algorithms in crypto framework */ 21927694b6caSKeerthy static void sa_register_algos(const struct device *dev) 21937694b6caSKeerthy { 21947694b6caSKeerthy char *alg_name; 21957694b6caSKeerthy u32 type; 21967694b6caSKeerthy int i, err; 21977694b6caSKeerthy 21987694b6caSKeerthy for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 21997694b6caSKeerthy type = sa_algs[i].type; 22007694b6caSKeerthy if (type == CRYPTO_ALG_TYPE_SKCIPHER) { 22017694b6caSKeerthy alg_name = sa_algs[i].alg.skcipher.base.cra_name; 22027694b6caSKeerthy err = crypto_register_skcipher(&sa_algs[i].alg.skcipher); 22032dc53d00SKeerthy } else if (type == CRYPTO_ALG_TYPE_AHASH) { 22042dc53d00SKeerthy alg_name = sa_algs[i].alg.ahash.halg.base.cra_name; 22052dc53d00SKeerthy err = crypto_register_ahash(&sa_algs[i].alg.ahash); 2206d2c8ac18SKeerthy } else if (type == CRYPTO_ALG_TYPE_AEAD) { 2207d2c8ac18SKeerthy alg_name = sa_algs[i].alg.aead.base.cra_name; 2208d2c8ac18SKeerthy err = crypto_register_aead(&sa_algs[i].alg.aead); 22097694b6caSKeerthy } else { 22107694b6caSKeerthy dev_err(dev, 22117694b6caSKeerthy "un-supported crypto algorithm (%d)", 22127694b6caSKeerthy sa_algs[i].type); 22137694b6caSKeerthy continue; 22147694b6caSKeerthy } 22157694b6caSKeerthy 22167694b6caSKeerthy if (err) 22177694b6caSKeerthy dev_err(dev, "Failed to register '%s'\n", alg_name); 22187694b6caSKeerthy else 22197694b6caSKeerthy sa_algs[i].registered = true; 22207694b6caSKeerthy } 22217694b6caSKeerthy } 22227694b6caSKeerthy 22237694b6caSKeerthy /* Unregister the algorithms in crypto framework */ 22247694b6caSKeerthy static void sa_unregister_algos(const struct device *dev) 22257694b6caSKeerthy { 22267694b6caSKeerthy u32 type; 22277694b6caSKeerthy int i; 22287694b6caSKeerthy 22297694b6caSKeerthy for (i = 0; i < ARRAY_SIZE(sa_algs); i++) { 22307694b6caSKeerthy type = sa_algs[i].type; 22317694b6caSKeerthy if (!sa_algs[i].registered) 22327694b6caSKeerthy continue; 22337694b6caSKeerthy if (type == CRYPTO_ALG_TYPE_SKCIPHER) 22347694b6caSKeerthy crypto_unregister_skcipher(&sa_algs[i].alg.skcipher); 22352dc53d00SKeerthy else if (type == CRYPTO_ALG_TYPE_AHASH) 22362dc53d00SKeerthy crypto_unregister_ahash(&sa_algs[i].alg.ahash); 2237d2c8ac18SKeerthy else if (type == CRYPTO_ALG_TYPE_AEAD) 2238d2c8ac18SKeerthy crypto_unregister_aead(&sa_algs[i].alg.aead); 22397694b6caSKeerthy 22407694b6caSKeerthy sa_algs[i].registered = false; 22417694b6caSKeerthy } 22427694b6caSKeerthy } 22437694b6caSKeerthy 22447694b6caSKeerthy static int sa_init_mem(struct sa_crypto_data *dev_data) 22457694b6caSKeerthy { 22467694b6caSKeerthy struct device *dev = &dev_data->pdev->dev; 22477694b6caSKeerthy /* Setup dma pool for security context buffers */ 22487694b6caSKeerthy dev_data->sc_pool = dma_pool_create("keystone-sc", dev, 22497694b6caSKeerthy SA_CTX_MAX_SZ, 64, 0); 22507694b6caSKeerthy if (!dev_data->sc_pool) { 22517694b6caSKeerthy dev_err(dev, "Failed to create dma pool"); 22527694b6caSKeerthy return -ENOMEM; 22537694b6caSKeerthy } 22547694b6caSKeerthy 22557694b6caSKeerthy return 0; 22567694b6caSKeerthy } 22577694b6caSKeerthy 22587694b6caSKeerthy static int sa_dma_init(struct sa_crypto_data *dd) 22597694b6caSKeerthy { 22607694b6caSKeerthy int ret; 22617694b6caSKeerthy struct dma_slave_config cfg; 22627694b6caSKeerthy 22637694b6caSKeerthy dd->dma_rx1 = NULL; 22647694b6caSKeerthy dd->dma_tx = NULL; 22657694b6caSKeerthy dd->dma_rx2 = NULL; 22667694b6caSKeerthy 22677694b6caSKeerthy ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48)); 22687694b6caSKeerthy if (ret) 22697694b6caSKeerthy return ret; 22707694b6caSKeerthy 22717694b6caSKeerthy dd->dma_rx1 = dma_request_chan(dd->dev, "rx1"); 22721b5d3a8bSKrzysztof Kozlowski if (IS_ERR(dd->dma_rx1)) 22731b5d3a8bSKrzysztof Kozlowski return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1), 22741b5d3a8bSKrzysztof Kozlowski "Unable to request rx1 DMA channel\n"); 22757694b6caSKeerthy 22767694b6caSKeerthy dd->dma_rx2 = dma_request_chan(dd->dev, "rx2"); 22777694b6caSKeerthy if (IS_ERR(dd->dma_rx2)) { 22787694b6caSKeerthy dma_release_channel(dd->dma_rx1); 22791b5d3a8bSKrzysztof Kozlowski return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2), 22801b5d3a8bSKrzysztof Kozlowski "Unable to request rx2 DMA channel\n"); 22817694b6caSKeerthy } 22827694b6caSKeerthy 22837694b6caSKeerthy dd->dma_tx = dma_request_chan(dd->dev, "tx"); 22847694b6caSKeerthy if (IS_ERR(dd->dma_tx)) { 22851b5d3a8bSKrzysztof Kozlowski ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx), 22861b5d3a8bSKrzysztof Kozlowski "Unable to request tx DMA channel\n"); 22877694b6caSKeerthy goto err_dma_tx; 22887694b6caSKeerthy } 22897694b6caSKeerthy 22907694b6caSKeerthy memzero_explicit(&cfg, sizeof(cfg)); 22917694b6caSKeerthy 22927694b6caSKeerthy cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 22937694b6caSKeerthy cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES; 22947694b6caSKeerthy cfg.src_maxburst = 4; 22957694b6caSKeerthy cfg.dst_maxburst = 4; 22967694b6caSKeerthy 22977694b6caSKeerthy ret = dmaengine_slave_config(dd->dma_rx1, &cfg); 22987694b6caSKeerthy if (ret) { 22997694b6caSKeerthy dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 23007694b6caSKeerthy ret); 23017694b6caSKeerthy return ret; 23027694b6caSKeerthy } 23037694b6caSKeerthy 23047694b6caSKeerthy ret = dmaengine_slave_config(dd->dma_rx2, &cfg); 23057694b6caSKeerthy if (ret) { 23067694b6caSKeerthy dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n", 23077694b6caSKeerthy ret); 23087694b6caSKeerthy return ret; 23097694b6caSKeerthy } 23107694b6caSKeerthy 23117694b6caSKeerthy ret = dmaengine_slave_config(dd->dma_tx, &cfg); 23127694b6caSKeerthy if (ret) { 23137694b6caSKeerthy dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n", 23147694b6caSKeerthy ret); 23157694b6caSKeerthy return ret; 23167694b6caSKeerthy } 23177694b6caSKeerthy 23187694b6caSKeerthy return 0; 23197694b6caSKeerthy 23207694b6caSKeerthy err_dma_tx: 23217694b6caSKeerthy dma_release_channel(dd->dma_rx1); 23227694b6caSKeerthy dma_release_channel(dd->dma_rx2); 23237694b6caSKeerthy 23247694b6caSKeerthy return ret; 23257694b6caSKeerthy } 23267694b6caSKeerthy 2327fd92028eSTero Kristo static int sa_link_child(struct device *dev, void *data) 2328fd92028eSTero Kristo { 2329fd92028eSTero Kristo struct device *parent = data; 2330fd92028eSTero Kristo 2331fd92028eSTero Kristo device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER); 2332fd92028eSTero Kristo 2333fd92028eSTero Kristo return 0; 2334fd92028eSTero Kristo } 2335fd92028eSTero Kristo 23367694b6caSKeerthy static int sa_ul_probe(struct platform_device *pdev) 23377694b6caSKeerthy { 23387694b6caSKeerthy struct device *dev = &pdev->dev; 23397694b6caSKeerthy struct device_node *node = dev->of_node; 23407694b6caSKeerthy struct resource *res; 23417694b6caSKeerthy static void __iomem *saul_base; 23427694b6caSKeerthy struct sa_crypto_data *dev_data; 23437694b6caSKeerthy u32 val; 23447694b6caSKeerthy int ret; 23457694b6caSKeerthy 23467694b6caSKeerthy dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL); 23477694b6caSKeerthy if (!dev_data) 23487694b6caSKeerthy return -ENOMEM; 23497694b6caSKeerthy 23507694b6caSKeerthy sa_k3_dev = dev; 23517694b6caSKeerthy dev_data->dev = dev; 23527694b6caSKeerthy dev_data->pdev = pdev; 23537694b6caSKeerthy platform_set_drvdata(pdev, dev_data); 23547694b6caSKeerthy dev_set_drvdata(sa_k3_dev, dev_data); 23557694b6caSKeerthy 23567694b6caSKeerthy pm_runtime_enable(dev); 235713343badSShixin Liu ret = pm_runtime_resume_and_get(dev); 23582baace5fSDan Carpenter if (ret < 0) { 23597694b6caSKeerthy dev_err(&pdev->dev, "%s: failed to get sync: %d\n", __func__, 23607694b6caSKeerthy ret); 23617694b6caSKeerthy return ret; 23627694b6caSKeerthy } 23637694b6caSKeerthy 23647694b6caSKeerthy sa_init_mem(dev_data); 23657694b6caSKeerthy ret = sa_dma_init(dev_data); 23667694b6caSKeerthy if (ret) 23677694b6caSKeerthy goto disable_pm_runtime; 23687694b6caSKeerthy 23697694b6caSKeerthy spin_lock_init(&dev_data->scid_lock); 23707694b6caSKeerthy res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 23717694b6caSKeerthy saul_base = devm_ioremap_resource(dev, res); 23727694b6caSKeerthy 23737694b6caSKeerthy dev_data->base = saul_base; 23747694b6caSKeerthy val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN | 23757694b6caSKeerthy SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN | 23767694b6caSKeerthy SA_EEC_TRNG_EN; 23777694b6caSKeerthy 23787694b6caSKeerthy writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL); 23797694b6caSKeerthy 23807694b6caSKeerthy sa_register_algos(dev); 23817694b6caSKeerthy 23827694b6caSKeerthy ret = of_platform_populate(node, NULL, NULL, &pdev->dev); 23837694b6caSKeerthy if (ret) 23847694b6caSKeerthy goto release_dma; 23857694b6caSKeerthy 2386fd92028eSTero Kristo device_for_each_child(&pdev->dev, &pdev->dev, sa_link_child); 2387fd92028eSTero Kristo 23887694b6caSKeerthy return 0; 23897694b6caSKeerthy 23907694b6caSKeerthy release_dma: 23917694b6caSKeerthy sa_unregister_algos(&pdev->dev); 23927694b6caSKeerthy 23937694b6caSKeerthy dma_release_channel(dev_data->dma_rx2); 23947694b6caSKeerthy dma_release_channel(dev_data->dma_rx1); 23957694b6caSKeerthy dma_release_channel(dev_data->dma_tx); 23967694b6caSKeerthy 23977694b6caSKeerthy dma_pool_destroy(dev_data->sc_pool); 23987694b6caSKeerthy 23997694b6caSKeerthy disable_pm_runtime: 24007694b6caSKeerthy pm_runtime_put_sync(&pdev->dev); 24017694b6caSKeerthy pm_runtime_disable(&pdev->dev); 24027694b6caSKeerthy 24037694b6caSKeerthy return ret; 24047694b6caSKeerthy } 24057694b6caSKeerthy 24067694b6caSKeerthy static int sa_ul_remove(struct platform_device *pdev) 24077694b6caSKeerthy { 24087694b6caSKeerthy struct sa_crypto_data *dev_data = platform_get_drvdata(pdev); 24097694b6caSKeerthy 24107694b6caSKeerthy sa_unregister_algos(&pdev->dev); 24117694b6caSKeerthy 24127694b6caSKeerthy dma_release_channel(dev_data->dma_rx2); 24137694b6caSKeerthy dma_release_channel(dev_data->dma_rx1); 24147694b6caSKeerthy dma_release_channel(dev_data->dma_tx); 24157694b6caSKeerthy 24167694b6caSKeerthy dma_pool_destroy(dev_data->sc_pool); 24177694b6caSKeerthy 24187694b6caSKeerthy platform_set_drvdata(pdev, NULL); 24197694b6caSKeerthy 24207694b6caSKeerthy pm_runtime_put_sync(&pdev->dev); 24217694b6caSKeerthy pm_runtime_disable(&pdev->dev); 24227694b6caSKeerthy 24237694b6caSKeerthy return 0; 24247694b6caSKeerthy } 24257694b6caSKeerthy 24267694b6caSKeerthy static const struct of_device_id of_match[] = { 24277694b6caSKeerthy {.compatible = "ti,j721e-sa2ul",}, 24287694b6caSKeerthy {.compatible = "ti,am654-sa2ul",}, 24297694b6caSKeerthy {}, 24307694b6caSKeerthy }; 24317694b6caSKeerthy MODULE_DEVICE_TABLE(of, of_match); 24327694b6caSKeerthy 24337694b6caSKeerthy static struct platform_driver sa_ul_driver = { 24347694b6caSKeerthy .probe = sa_ul_probe, 24357694b6caSKeerthy .remove = sa_ul_remove, 24367694b6caSKeerthy .driver = { 24377694b6caSKeerthy .name = "saul-crypto", 24387694b6caSKeerthy .of_match_table = of_match, 24397694b6caSKeerthy }, 24407694b6caSKeerthy }; 24417694b6caSKeerthy module_platform_driver(sa_ul_driver); 24427694b6caSKeerthy MODULE_LICENSE("GPL v2"); 2443