17694b6caSKeerthy // SPDX-License-Identifier: GPL-2.0
27694b6caSKeerthy /*
37694b6caSKeerthy * K3 SA2UL crypto accelerator driver
47694b6caSKeerthy *
57694b6caSKeerthy * Copyright (C) 2018-2020 Texas Instruments Incorporated - http://www.ti.com
67694b6caSKeerthy *
77694b6caSKeerthy * Authors: Keerthy
87694b6caSKeerthy * Vitaly Andrianov
97694b6caSKeerthy * Tero Kristo
107694b6caSKeerthy */
115876b0cbSGeert Uytterhoeven #include <linux/bitfield.h>
127694b6caSKeerthy #include <linux/clk.h>
13ad0bb4e4SHerbert Xu #include <linux/dma-mapping.h>
147694b6caSKeerthy #include <linux/dmaengine.h>
157694b6caSKeerthy #include <linux/dmapool.h>
16ad0bb4e4SHerbert Xu #include <linux/kernel.h>
177694b6caSKeerthy #include <linux/module.h>
18b0cc7491SRob Herring #include <linux/of.h>
19b0cc7491SRob Herring #include <linux/of_platform.h>
207694b6caSKeerthy #include <linux/platform_device.h>
217694b6caSKeerthy #include <linux/pm_runtime.h>
227694b6caSKeerthy
237694b6caSKeerthy #include <crypto/aes.h>
24d2c8ac18SKeerthy #include <crypto/authenc.h>
257694b6caSKeerthy #include <crypto/des.h>
26d2c8ac18SKeerthy #include <crypto/internal/aead.h>
272dc53d00SKeerthy #include <crypto/internal/hash.h>
287694b6caSKeerthy #include <crypto/internal/skcipher.h>
297694b6caSKeerthy #include <crypto/scatterwalk.h>
30a24d22b2SEric Biggers #include <crypto/sha1.h>
31a24d22b2SEric Biggers #include <crypto/sha2.h>
327694b6caSKeerthy
337694b6caSKeerthy #include "sa2ul.h"
347694b6caSKeerthy
357694b6caSKeerthy /* Byte offset for key in encryption security context */
367694b6caSKeerthy #define SC_ENC_KEY_OFFSET (1 + 27 + 4)
377694b6caSKeerthy /* Byte offset for Aux-1 in encryption security context */
387694b6caSKeerthy #define SC_ENC_AUX1_OFFSET (1 + 27 + 4 + 32)
397694b6caSKeerthy
407694b6caSKeerthy #define SA_CMDL_UPD_ENC 0x0001
417694b6caSKeerthy #define SA_CMDL_UPD_AUTH 0x0002
427694b6caSKeerthy #define SA_CMDL_UPD_ENC_IV 0x0004
437694b6caSKeerthy #define SA_CMDL_UPD_AUTH_IV 0x0008
447694b6caSKeerthy #define SA_CMDL_UPD_AUX_KEY 0x0010
457694b6caSKeerthy
467694b6caSKeerthy #define SA_AUTH_SUBKEY_LEN 16
477694b6caSKeerthy #define SA_CMDL_PAYLOAD_LENGTH_MASK 0xFFFF
487694b6caSKeerthy #define SA_CMDL_SOP_BYPASS_LEN_MASK 0xFF000000
497694b6caSKeerthy
507694b6caSKeerthy #define MODE_CONTROL_BYTES 27
517694b6caSKeerthy #define SA_HASH_PROCESSING 0
527694b6caSKeerthy #define SA_CRYPTO_PROCESSING 0
537694b6caSKeerthy #define SA_UPLOAD_HASH_TO_TLR BIT(6)
547694b6caSKeerthy
557694b6caSKeerthy #define SA_SW0_FLAGS_MASK 0xF0000
567694b6caSKeerthy #define SA_SW0_CMDL_INFO_MASK 0x1F00000
577694b6caSKeerthy #define SA_SW0_CMDL_PRESENT BIT(4)
587694b6caSKeerthy #define SA_SW0_ENG_ID_MASK 0x3E000000
597694b6caSKeerthy #define SA_SW0_DEST_INFO_PRESENT BIT(30)
607694b6caSKeerthy #define SA_SW2_EGRESS_LENGTH 0xFF000000
617694b6caSKeerthy #define SA_BASIC_HASH 0x10
627694b6caSKeerthy
637694b6caSKeerthy #define SHA256_DIGEST_WORDS 8
647694b6caSKeerthy /* Make 32-bit word from 4 bytes */
657694b6caSKeerthy #define SA_MK_U32(b0, b1, b2, b3) (((b0) << 24) | ((b1) << 16) | \
667694b6caSKeerthy ((b2) << 8) | (b3))
677694b6caSKeerthy
687694b6caSKeerthy /* size of SCCTL structure in bytes */
697694b6caSKeerthy #define SA_SCCTL_SZ 16
707694b6caSKeerthy
717694b6caSKeerthy /* Max Authentication tag size */
727694b6caSKeerthy #define SA_MAX_AUTH_TAG_SZ 64
737694b6caSKeerthy
740bc42311SPeter Ujfalusi enum sa_algo_id {
750bc42311SPeter Ujfalusi SA_ALG_CBC_AES = 0,
760bc42311SPeter Ujfalusi SA_ALG_EBC_AES,
770bc42311SPeter Ujfalusi SA_ALG_CBC_DES3,
780bc42311SPeter Ujfalusi SA_ALG_ECB_DES3,
790bc42311SPeter Ujfalusi SA_ALG_SHA1,
800bc42311SPeter Ujfalusi SA_ALG_SHA256,
810bc42311SPeter Ujfalusi SA_ALG_SHA512,
820bc42311SPeter Ujfalusi SA_ALG_AUTHENC_SHA1_AES,
830bc42311SPeter Ujfalusi SA_ALG_AUTHENC_SHA256_AES,
840bc42311SPeter Ujfalusi };
850bc42311SPeter Ujfalusi
860bc42311SPeter Ujfalusi struct sa_match_data {
870bc42311SPeter Ujfalusi u8 priv;
880bc42311SPeter Ujfalusi u8 priv_id;
890bc42311SPeter Ujfalusi u32 supported_algos;
900bc42311SPeter Ujfalusi };
917694b6caSKeerthy
927694b6caSKeerthy static struct device *sa_k3_dev;
937694b6caSKeerthy
947694b6caSKeerthy /**
957694b6caSKeerthy * struct sa_cmdl_cfg - Command label configuration descriptor
962dc53d00SKeerthy * @aalg: authentication algorithm ID
977694b6caSKeerthy * @enc_eng_id: Encryption Engine ID supported by the SA hardware
982dc53d00SKeerthy * @auth_eng_id: Authentication Engine ID
997694b6caSKeerthy * @iv_size: Initialization Vector size
1002dc53d00SKeerthy * @akey: Authentication key
1012dc53d00SKeerthy * @akey_len: Authentication key length
102d2c8ac18SKeerthy * @enc: True, if this is an encode request
1037694b6caSKeerthy */
1047694b6caSKeerthy struct sa_cmdl_cfg {
1052dc53d00SKeerthy int aalg;
1067694b6caSKeerthy u8 enc_eng_id;
1072dc53d00SKeerthy u8 auth_eng_id;
1087694b6caSKeerthy u8 iv_size;
1092dc53d00SKeerthy const u8 *akey;
1102dc53d00SKeerthy u16 akey_len;
111d2c8ac18SKeerthy bool enc;
1127694b6caSKeerthy };
1137694b6caSKeerthy
1147694b6caSKeerthy /**
1157694b6caSKeerthy * struct algo_data - Crypto algorithm specific data
1167694b6caSKeerthy * @enc_eng: Encryption engine info structure
1172dc53d00SKeerthy * @auth_eng: Authentication engine info structure
1182dc53d00SKeerthy * @auth_ctrl: Authentication control word
1192dc53d00SKeerthy * @hash_size: Size of digest
1207694b6caSKeerthy * @iv_idx: iv index in psdata
1217694b6caSKeerthy * @iv_out_size: iv out size
1227694b6caSKeerthy * @ealg_id: Encryption Algorithm ID
1232dc53d00SKeerthy * @aalg_id: Authentication algorithm ID
1247694b6caSKeerthy * @mci_enc: Mode Control Instruction for Encryption algorithm
1257694b6caSKeerthy * @mci_dec: Mode Control Instruction for Decryption
1267694b6caSKeerthy * @inv_key: Whether the encryption algorithm demands key inversion
1277694b6caSKeerthy * @ctx: Pointer to the algorithm context
128d2c8ac18SKeerthy * @keyed_mac: Whether the authentication algorithm has key
129d2c8ac18SKeerthy * @prep_iopad: Function pointer to generate intermediate ipad/opad
1307694b6caSKeerthy */
1317694b6caSKeerthy struct algo_data {
1327694b6caSKeerthy struct sa_eng_info enc_eng;
1332dc53d00SKeerthy struct sa_eng_info auth_eng;
1342dc53d00SKeerthy u8 auth_ctrl;
1352dc53d00SKeerthy u8 hash_size;
1367694b6caSKeerthy u8 iv_idx;
1377694b6caSKeerthy u8 iv_out_size;
1387694b6caSKeerthy u8 ealg_id;
1392dc53d00SKeerthy u8 aalg_id;
1407694b6caSKeerthy u8 *mci_enc;
1417694b6caSKeerthy u8 *mci_dec;
1427694b6caSKeerthy bool inv_key;
1437694b6caSKeerthy struct sa_tfm_ctx *ctx;
144d2c8ac18SKeerthy bool keyed_mac;
145d2c8ac18SKeerthy void (*prep_iopad)(struct algo_data *algo, const u8 *key,
146d2c8ac18SKeerthy u16 key_sz, __be32 *ipad, __be32 *opad);
1477694b6caSKeerthy };
1487694b6caSKeerthy
1497694b6caSKeerthy /**
1507694b6caSKeerthy * struct sa_alg_tmpl: A generic template encompassing crypto/aead algorithms
1517694b6caSKeerthy * @type: Type of the crypto algorithm.
1527694b6caSKeerthy * @alg: Union of crypto algorithm definitions.
1537694b6caSKeerthy * @registered: Flag indicating if the crypto algorithm is already registered
1547694b6caSKeerthy */
1557694b6caSKeerthy struct sa_alg_tmpl {
1567694b6caSKeerthy u32 type; /* CRYPTO_ALG_TYPE from <linux/crypto.h> */
1577694b6caSKeerthy union {
1587694b6caSKeerthy struct skcipher_alg skcipher;
1592dc53d00SKeerthy struct ahash_alg ahash;
160d2c8ac18SKeerthy struct aead_alg aead;
1617694b6caSKeerthy } alg;
1627694b6caSKeerthy bool registered;
1637694b6caSKeerthy };
1647694b6caSKeerthy
1657694b6caSKeerthy /**
16600c9211fSPeter Ujfalusi * struct sa_mapped_sg: scatterlist information for tx and rx
16700c9211fSPeter Ujfalusi * @mapped: Set to true if the @sgt is mapped
16800c9211fSPeter Ujfalusi * @dir: mapping direction used for @sgt
16900c9211fSPeter Ujfalusi * @split_sg: Set if the sg is split and needs to be freed up
17000c9211fSPeter Ujfalusi * @static_sg: Static scatterlist entry for overriding data
17100c9211fSPeter Ujfalusi * @sgt: scatterlist table for DMA API use
17200c9211fSPeter Ujfalusi */
17300c9211fSPeter Ujfalusi struct sa_mapped_sg {
17400c9211fSPeter Ujfalusi bool mapped;
17500c9211fSPeter Ujfalusi enum dma_data_direction dir;
17600c9211fSPeter Ujfalusi struct scatterlist static_sg;
17700c9211fSPeter Ujfalusi struct scatterlist *split_sg;
17800c9211fSPeter Ujfalusi struct sg_table sgt;
17900c9211fSPeter Ujfalusi };
18000c9211fSPeter Ujfalusi /**
1817694b6caSKeerthy * struct sa_rx_data: RX Packet miscellaneous data place holder
1827694b6caSKeerthy * @req: crypto request data pointer
1837694b6caSKeerthy * @ddev: pointer to the DMA device
1847694b6caSKeerthy * @tx_in: dma_async_tx_descriptor pointer for rx channel
18500c9211fSPeter Ujfalusi * @mapped_sg: Information on tx (0) and rx (1) scatterlist DMA mapping
1867694b6caSKeerthy * @enc: Flag indicating either encryption or decryption
1877694b6caSKeerthy * @enc_iv_size: Initialisation vector size
1887694b6caSKeerthy * @iv_idx: Initialisation vector index
1897694b6caSKeerthy */
1907694b6caSKeerthy struct sa_rx_data {
1917694b6caSKeerthy void *req;
1927694b6caSKeerthy struct device *ddev;
1937694b6caSKeerthy struct dma_async_tx_descriptor *tx_in;
19400c9211fSPeter Ujfalusi struct sa_mapped_sg mapped_sg[2];
1957694b6caSKeerthy u8 enc;
1967694b6caSKeerthy u8 enc_iv_size;
1977694b6caSKeerthy u8 iv_idx;
1987694b6caSKeerthy };
1997694b6caSKeerthy
2007694b6caSKeerthy /**
2017694b6caSKeerthy * struct sa_req: SA request definition
2027694b6caSKeerthy * @dev: device for the request
2037694b6caSKeerthy * @size: total data to the xmitted via DMA
2047694b6caSKeerthy * @enc_offset: offset of cipher data
2057694b6caSKeerthy * @enc_size: data to be passed to cipher engine
2067694b6caSKeerthy * @enc_iv: cipher IV
2072dc53d00SKeerthy * @auth_offset: offset of the authentication data
2082dc53d00SKeerthy * @auth_size: size of the authentication data
2092dc53d00SKeerthy * @auth_iv: authentication IV
2107694b6caSKeerthy * @type: algorithm type for the request
2117694b6caSKeerthy * @cmdl: command label pointer
2127694b6caSKeerthy * @base: pointer to the base request
2137694b6caSKeerthy * @ctx: pointer to the algorithm context data
2147694b6caSKeerthy * @enc: true if this is an encode request
2157694b6caSKeerthy * @src: source data
2167694b6caSKeerthy * @dst: destination data
2177694b6caSKeerthy * @callback: DMA callback for the request
2187694b6caSKeerthy * @mdata_size: metadata size passed to DMA
2197694b6caSKeerthy */
2207694b6caSKeerthy struct sa_req {
2217694b6caSKeerthy struct device *dev;
2227694b6caSKeerthy u16 size;
2237694b6caSKeerthy u8 enc_offset;
2247694b6caSKeerthy u16 enc_size;
2257694b6caSKeerthy u8 *enc_iv;
2262dc53d00SKeerthy u8 auth_offset;
2272dc53d00SKeerthy u16 auth_size;
2282dc53d00SKeerthy u8 *auth_iv;
2297694b6caSKeerthy u32 type;
2307694b6caSKeerthy u32 *cmdl;
2317694b6caSKeerthy struct crypto_async_request *base;
2327694b6caSKeerthy struct sa_tfm_ctx *ctx;
2337694b6caSKeerthy bool enc;
2347694b6caSKeerthy struct scatterlist *src;
2357694b6caSKeerthy struct scatterlist *dst;
2367694b6caSKeerthy dma_async_tx_callback callback;
2377694b6caSKeerthy u16 mdata_size;
2387694b6caSKeerthy };
2397694b6caSKeerthy
2407694b6caSKeerthy /*
2417694b6caSKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256
2427694b6caSKeerthy * For CBC (Cipher Block Chaining) mode for encryption
2437694b6caSKeerthy */
2447694b6caSKeerthy static u8 mci_cbc_enc_array[3][MODE_CONTROL_BYTES] = {
2457694b6caSKeerthy { 0x61, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
2467694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2477694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
2487694b6caSKeerthy { 0x61, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
2497694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2507694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
2517694b6caSKeerthy { 0x61, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
2527694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2537694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
2547694b6caSKeerthy };
2557694b6caSKeerthy
2567694b6caSKeerthy /*
2577694b6caSKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256
2587694b6caSKeerthy * For CBC (Cipher Block Chaining) mode for decryption
2597694b6caSKeerthy */
2607694b6caSKeerthy static u8 mci_cbc_dec_array[3][MODE_CONTROL_BYTES] = {
2617694b6caSKeerthy { 0x71, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
2627694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2637694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
2647694b6caSKeerthy { 0x71, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
2657694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2667694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
2677694b6caSKeerthy { 0x71, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
2687694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
2697694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
2707694b6caSKeerthy };
2717694b6caSKeerthy
2727694b6caSKeerthy /*
2737694b6caSKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256
274d2c8ac18SKeerthy * For CBC (Cipher Block Chaining) mode for encryption
275d2c8ac18SKeerthy */
276d2c8ac18SKeerthy static u8 mci_cbc_enc_no_iv_array[3][MODE_CONTROL_BYTES] = {
277d2c8ac18SKeerthy { 0x21, 0x00, 0x00, 0x18, 0x88, 0x0a, 0xaa, 0x4b, 0x7e, 0x00,
278d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
279d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
280d2c8ac18SKeerthy { 0x21, 0x00, 0x00, 0x18, 0x88, 0x4a, 0xaa, 0x4b, 0x7e, 0x00,
281d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
282d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
283d2c8ac18SKeerthy { 0x21, 0x00, 0x00, 0x18, 0x88, 0x8a, 0xaa, 0x4b, 0x7e, 0x00,
284d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
285d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
286d2c8ac18SKeerthy };
287d2c8ac18SKeerthy
288d2c8ac18SKeerthy /*
289d2c8ac18SKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256
290d2c8ac18SKeerthy * For CBC (Cipher Block Chaining) mode for decryption
291d2c8ac18SKeerthy */
292d2c8ac18SKeerthy static u8 mci_cbc_dec_no_iv_array[3][MODE_CONTROL_BYTES] = {
293d2c8ac18SKeerthy { 0x31, 0x00, 0x00, 0x80, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
294d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
295d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
296d2c8ac18SKeerthy { 0x31, 0x00, 0x00, 0x84, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
297d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
298d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
299d2c8ac18SKeerthy { 0x31, 0x00, 0x00, 0x88, 0x8a, 0xca, 0x98, 0xf4, 0x40, 0xc0,
300d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
301d2c8ac18SKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
302d2c8ac18SKeerthy };
303d2c8ac18SKeerthy
304d2c8ac18SKeerthy /*
305d2c8ac18SKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256
3067694b6caSKeerthy * For ECB (Electronic Code Book) mode for encryption
3077694b6caSKeerthy */
3087694b6caSKeerthy static u8 mci_ecb_enc_array[3][27] = {
3097694b6caSKeerthy { 0x21, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
3107694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3117694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
3127694b6caSKeerthy { 0x21, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
3137694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3147694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
3157694b6caSKeerthy { 0x21, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
3167694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3177694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
3187694b6caSKeerthy };
3197694b6caSKeerthy
3207694b6caSKeerthy /*
3217694b6caSKeerthy * Mode Control Instructions for various Key lengths 128, 192, 256
3227694b6caSKeerthy * For ECB (Electronic Code Book) mode for decryption
3237694b6caSKeerthy */
3247694b6caSKeerthy static u8 mci_ecb_dec_array[3][27] = {
3257694b6caSKeerthy { 0x31, 0x00, 0x00, 0x80, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
3267694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3277694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
3287694b6caSKeerthy { 0x31, 0x00, 0x00, 0x84, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
3297694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3307694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
3317694b6caSKeerthy { 0x31, 0x00, 0x00, 0x88, 0x8a, 0x04, 0xb7, 0x90, 0x00, 0x00,
3327694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3337694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
3347694b6caSKeerthy };
3357694b6caSKeerthy
3367694b6caSKeerthy /*
3377694b6caSKeerthy * Mode Control Instructions for DES algorithm
3387694b6caSKeerthy * For CBC (Cipher Block Chaining) mode and ECB mode
3397694b6caSKeerthy * encryption and for decryption respectively
3407694b6caSKeerthy */
3417694b6caSKeerthy static u8 mci_cbc_3des_enc_array[MODE_CONTROL_BYTES] = {
3427694b6caSKeerthy 0x60, 0x00, 0x00, 0x18, 0x88, 0x52, 0xaa, 0x4b, 0x7e, 0x00, 0x00, 0x00,
3437694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3447694b6caSKeerthy 0x00, 0x00, 0x00,
3457694b6caSKeerthy };
3467694b6caSKeerthy
3477694b6caSKeerthy static u8 mci_cbc_3des_dec_array[MODE_CONTROL_BYTES] = {
3487694b6caSKeerthy 0x70, 0x00, 0x00, 0x85, 0x0a, 0xca, 0x98, 0xf4, 0x40, 0xc0, 0x00, 0x00,
3497694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3507694b6caSKeerthy 0x00, 0x00, 0x00,
3517694b6caSKeerthy };
3527694b6caSKeerthy
3537694b6caSKeerthy static u8 mci_ecb_3des_enc_array[MODE_CONTROL_BYTES] = {
3547694b6caSKeerthy 0x20, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
3557694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3567694b6caSKeerthy 0x00, 0x00, 0x00,
3577694b6caSKeerthy };
3587694b6caSKeerthy
3597694b6caSKeerthy static u8 mci_ecb_3des_dec_array[MODE_CONTROL_BYTES] = {
3607694b6caSKeerthy 0x30, 0x00, 0x00, 0x85, 0x0a, 0x04, 0xb7, 0x90, 0x00, 0x00, 0x00, 0x00,
3617694b6caSKeerthy 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
3627694b6caSKeerthy 0x00, 0x00, 0x00,
3637694b6caSKeerthy };
3647694b6caSKeerthy
3657694b6caSKeerthy /*
3667694b6caSKeerthy * Perform 16 byte or 128 bit swizzling
3677694b6caSKeerthy * The SA2UL Expects the security context to
3687694b6caSKeerthy * be in little Endian and the bus width is 128 bits or 16 bytes
3697694b6caSKeerthy * Hence swap 16 bytes at a time from higher to lower address
3707694b6caSKeerthy */
sa_swiz_128(u8 * in,u16 len)3717694b6caSKeerthy static void sa_swiz_128(u8 *in, u16 len)
3727694b6caSKeerthy {
3737694b6caSKeerthy u8 data[16];
3747694b6caSKeerthy int i, j;
3757694b6caSKeerthy
3767694b6caSKeerthy for (i = 0; i < len; i += 16) {
3777694b6caSKeerthy memcpy(data, &in[i], 16);
3787694b6caSKeerthy for (j = 0; j < 16; j++)
3797694b6caSKeerthy in[i + j] = data[15 - j];
3807694b6caSKeerthy }
3817694b6caSKeerthy }
3827694b6caSKeerthy
383d2c8ac18SKeerthy /* Prepare the ipad and opad from key as per SHA algorithm step 1*/
prepare_kipad(u8 * k_ipad,const u8 * key,u16 key_sz)384ad0bb4e4SHerbert Xu static void prepare_kipad(u8 *k_ipad, const u8 *key, u16 key_sz)
385d2c8ac18SKeerthy {
386d2c8ac18SKeerthy int i;
387d2c8ac18SKeerthy
388ad0bb4e4SHerbert Xu for (i = 0; i < key_sz; i++)
389d2c8ac18SKeerthy k_ipad[i] = key[i] ^ 0x36;
390d2c8ac18SKeerthy
391d2c8ac18SKeerthy /* Instead of XOR with 0 */
392ad0bb4e4SHerbert Xu for (; i < SHA1_BLOCK_SIZE; i++)
393d2c8ac18SKeerthy k_ipad[i] = 0x36;
394d2c8ac18SKeerthy }
395d2c8ac18SKeerthy
prepare_kopad(u8 * k_opad,const u8 * key,u16 key_sz)396ad0bb4e4SHerbert Xu static void prepare_kopad(u8 *k_opad, const u8 *key, u16 key_sz)
397ad0bb4e4SHerbert Xu {
398ad0bb4e4SHerbert Xu int i;
399ad0bb4e4SHerbert Xu
400ad0bb4e4SHerbert Xu for (i = 0; i < key_sz; i++)
401ad0bb4e4SHerbert Xu k_opad[i] = key[i] ^ 0x5c;
402ad0bb4e4SHerbert Xu
403ad0bb4e4SHerbert Xu /* Instead of XOR with 0 */
404ad0bb4e4SHerbert Xu for (; i < SHA1_BLOCK_SIZE; i++)
405ad0bb4e4SHerbert Xu k_opad[i] = 0x5c;
406ad0bb4e4SHerbert Xu }
407ad0bb4e4SHerbert Xu
sa_export_shash(void * state,struct shash_desc * hash,int digest_size,__be32 * out)408ad0bb4e4SHerbert Xu static void sa_export_shash(void *state, struct shash_desc *hash,
409d2c8ac18SKeerthy int digest_size, __be32 *out)
410d2c8ac18SKeerthy {
411ad0bb4e4SHerbert Xu struct sha1_state *sha1;
412ad0bb4e4SHerbert Xu struct sha256_state *sha256;
413d2c8ac18SKeerthy u32 *result;
414d2c8ac18SKeerthy
415d2c8ac18SKeerthy switch (digest_size) {
416d2c8ac18SKeerthy case SHA1_DIGEST_SIZE:
417ad0bb4e4SHerbert Xu sha1 = state;
418ad0bb4e4SHerbert Xu result = sha1->state;
419d2c8ac18SKeerthy break;
420d2c8ac18SKeerthy case SHA256_DIGEST_SIZE:
421ad0bb4e4SHerbert Xu sha256 = state;
422ad0bb4e4SHerbert Xu result = sha256->state;
423d2c8ac18SKeerthy break;
424d2c8ac18SKeerthy default:
425d2c8ac18SKeerthy dev_err(sa_k3_dev, "%s: bad digest_size=%d\n", __func__,
426d2c8ac18SKeerthy digest_size);
427d2c8ac18SKeerthy return;
428d2c8ac18SKeerthy }
429d2c8ac18SKeerthy
430d2c8ac18SKeerthy crypto_shash_export(hash, state);
431d2c8ac18SKeerthy
432ad0bb4e4SHerbert Xu cpu_to_be32_array(out, result, digest_size / 4);
433d2c8ac18SKeerthy }
434d2c8ac18SKeerthy
sa_prepare_iopads(struct algo_data * data,const u8 * key,u16 key_sz,__be32 * ipad,__be32 * opad)435d2c8ac18SKeerthy static void sa_prepare_iopads(struct algo_data *data, const u8 *key,
436d2c8ac18SKeerthy u16 key_sz, __be32 *ipad, __be32 *opad)
437d2c8ac18SKeerthy {
438d2c8ac18SKeerthy SHASH_DESC_ON_STACK(shash, data->ctx->shash);
439d2c8ac18SKeerthy int block_size = crypto_shash_blocksize(data->ctx->shash);
440d2c8ac18SKeerthy int digest_size = crypto_shash_digestsize(data->ctx->shash);
441ad0bb4e4SHerbert Xu union {
442ad0bb4e4SHerbert Xu struct sha1_state sha1;
443ad0bb4e4SHerbert Xu struct sha256_state sha256;
444ad0bb4e4SHerbert Xu u8 k_pad[SHA1_BLOCK_SIZE];
445ad0bb4e4SHerbert Xu } sha;
446d2c8ac18SKeerthy
447d2c8ac18SKeerthy shash->tfm = data->ctx->shash;
448d2c8ac18SKeerthy
449ad0bb4e4SHerbert Xu prepare_kipad(sha.k_pad, key, key_sz);
450d2c8ac18SKeerthy
451d2c8ac18SKeerthy crypto_shash_init(shash);
452ad0bb4e4SHerbert Xu crypto_shash_update(shash, sha.k_pad, block_size);
453ad0bb4e4SHerbert Xu sa_export_shash(&sha, shash, digest_size, ipad);
454ad0bb4e4SHerbert Xu
455ad0bb4e4SHerbert Xu prepare_kopad(sha.k_pad, key, key_sz);
456d2c8ac18SKeerthy
457d2c8ac18SKeerthy crypto_shash_init(shash);
458ad0bb4e4SHerbert Xu crypto_shash_update(shash, sha.k_pad, block_size);
459d2c8ac18SKeerthy
460ad0bb4e4SHerbert Xu sa_export_shash(&sha, shash, digest_size, opad);
461ad0bb4e4SHerbert Xu
462ad0bb4e4SHerbert Xu memzero_explicit(&sha, sizeof(sha));
463d2c8ac18SKeerthy }
464d2c8ac18SKeerthy
4657694b6caSKeerthy /* Derive the inverse key used in AES-CBC decryption operation */
sa_aes_inv_key(u8 * inv_key,const u8 * key,u16 key_sz)4667694b6caSKeerthy static inline int sa_aes_inv_key(u8 *inv_key, const u8 *key, u16 key_sz)
4677694b6caSKeerthy {
4687694b6caSKeerthy struct crypto_aes_ctx ctx;
4697694b6caSKeerthy int key_pos;
4707694b6caSKeerthy
4717694b6caSKeerthy if (aes_expandkey(&ctx, key, key_sz)) {
4727694b6caSKeerthy dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
4737694b6caSKeerthy return -EINVAL;
4747694b6caSKeerthy }
4757694b6caSKeerthy
4767694b6caSKeerthy /* work around to get the right inverse for AES_KEYSIZE_192 size keys */
4777694b6caSKeerthy if (key_sz == AES_KEYSIZE_192) {
4787694b6caSKeerthy ctx.key_enc[52] = ctx.key_enc[51] ^ ctx.key_enc[46];
4797694b6caSKeerthy ctx.key_enc[53] = ctx.key_enc[52] ^ ctx.key_enc[47];
4807694b6caSKeerthy }
4817694b6caSKeerthy
4827694b6caSKeerthy /* Based crypto_aes_expand_key logic */
4837694b6caSKeerthy switch (key_sz) {
4847694b6caSKeerthy case AES_KEYSIZE_128:
4857694b6caSKeerthy case AES_KEYSIZE_192:
4867694b6caSKeerthy key_pos = key_sz + 24;
4877694b6caSKeerthy break;
4887694b6caSKeerthy
4897694b6caSKeerthy case AES_KEYSIZE_256:
4907694b6caSKeerthy key_pos = key_sz + 24 - 4;
4917694b6caSKeerthy break;
4927694b6caSKeerthy
4937694b6caSKeerthy default:
4947694b6caSKeerthy dev_err(sa_k3_dev, "%s: bad key len(%d)\n", __func__, key_sz);
4957694b6caSKeerthy return -EINVAL;
4967694b6caSKeerthy }
4977694b6caSKeerthy
4987694b6caSKeerthy memcpy(inv_key, &ctx.key_enc[key_pos], key_sz);
4997694b6caSKeerthy return 0;
5007694b6caSKeerthy }
5017694b6caSKeerthy
5027694b6caSKeerthy /* Set Security context for the encryption engine */
sa_set_sc_enc(struct algo_data * ad,const u8 * key,u16 key_sz,u8 enc,u8 * sc_buf)5037694b6caSKeerthy static int sa_set_sc_enc(struct algo_data *ad, const u8 *key, u16 key_sz,
5047694b6caSKeerthy u8 enc, u8 *sc_buf)
5057694b6caSKeerthy {
5067694b6caSKeerthy const u8 *mci = NULL;
5077694b6caSKeerthy
5087694b6caSKeerthy /* Set Encryption mode selector to crypto processing */
5097694b6caSKeerthy sc_buf[0] = SA_CRYPTO_PROCESSING;
5107694b6caSKeerthy
5117694b6caSKeerthy if (enc)
5127694b6caSKeerthy mci = ad->mci_enc;
5137694b6caSKeerthy else
5147694b6caSKeerthy mci = ad->mci_dec;
5157694b6caSKeerthy /* Set the mode control instructions in security context */
5167694b6caSKeerthy if (mci)
5177694b6caSKeerthy memcpy(&sc_buf[1], mci, MODE_CONTROL_BYTES);
5187694b6caSKeerthy
5197694b6caSKeerthy /* For AES-CBC decryption get the inverse key */
5207694b6caSKeerthy if (ad->inv_key && !enc) {
5217694b6caSKeerthy if (sa_aes_inv_key(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz))
5227694b6caSKeerthy return -EINVAL;
5237694b6caSKeerthy /* For all other cases: key is used */
5247694b6caSKeerthy } else {
5257694b6caSKeerthy memcpy(&sc_buf[SC_ENC_KEY_OFFSET], key, key_sz);
5267694b6caSKeerthy }
5277694b6caSKeerthy
5287694b6caSKeerthy return 0;
5297694b6caSKeerthy }
5307694b6caSKeerthy
5312dc53d00SKeerthy /* Set Security context for the authentication engine */
sa_set_sc_auth(struct algo_data * ad,const u8 * key,u16 key_sz,u8 * sc_buf)5322dc53d00SKeerthy static void sa_set_sc_auth(struct algo_data *ad, const u8 *key, u16 key_sz,
5332dc53d00SKeerthy u8 *sc_buf)
5342dc53d00SKeerthy {
535ad0bb4e4SHerbert Xu __be32 *ipad = (void *)(sc_buf + 32);
536ad0bb4e4SHerbert Xu __be32 *opad = (void *)(sc_buf + 64);
537d2c8ac18SKeerthy
5382dc53d00SKeerthy /* Set Authentication mode selector to hash processing */
5392dc53d00SKeerthy sc_buf[0] = SA_HASH_PROCESSING;
5402dc53d00SKeerthy /* Auth SW ctrl word: bit[6]=1 (upload computed hash to TLR section) */
5412dc53d00SKeerthy sc_buf[1] = SA_UPLOAD_HASH_TO_TLR;
5422dc53d00SKeerthy sc_buf[1] |= ad->auth_ctrl;
5432dc53d00SKeerthy
544d2c8ac18SKeerthy /* Copy the keys or ipad/opad */
545ad0bb4e4SHerbert Xu if (ad->keyed_mac)
546d2c8ac18SKeerthy ad->prep_iopad(ad, key, key_sz, ipad, opad);
547ad0bb4e4SHerbert Xu else {
5482dc53d00SKeerthy /* basic hash */
5492dc53d00SKeerthy sc_buf[1] |= SA_BASIC_HASH;
5502dc53d00SKeerthy }
551d2c8ac18SKeerthy }
5522dc53d00SKeerthy
sa_copy_iv(__be32 * out,const u8 * iv,bool size16)5537694b6caSKeerthy static inline void sa_copy_iv(__be32 *out, const u8 *iv, bool size16)
5547694b6caSKeerthy {
5557694b6caSKeerthy int j;
5567694b6caSKeerthy
5577694b6caSKeerthy for (j = 0; j < ((size16) ? 4 : 2); j++) {
5587694b6caSKeerthy *out = cpu_to_be32(*((u32 *)iv));
5597694b6caSKeerthy iv += 4;
5607694b6caSKeerthy out++;
5617694b6caSKeerthy }
5627694b6caSKeerthy }
5637694b6caSKeerthy
5647694b6caSKeerthy /* Format general command label */
sa_format_cmdl_gen(struct sa_cmdl_cfg * cfg,u8 * cmdl,struct sa_cmdl_upd_info * upd_info)5657694b6caSKeerthy static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl,
5667694b6caSKeerthy struct sa_cmdl_upd_info *upd_info)
5677694b6caSKeerthy {
5682dc53d00SKeerthy u8 enc_offset = 0, auth_offset = 0, total = 0;
5697694b6caSKeerthy u8 enc_next_eng = SA_ENG_ID_OUTPORT2;
5702dc53d00SKeerthy u8 auth_next_eng = SA_ENG_ID_OUTPORT2;
5717694b6caSKeerthy u32 *word_ptr = (u32 *)cmdl;
5727694b6caSKeerthy int i;
5737694b6caSKeerthy
5747694b6caSKeerthy /* Clear the command label */
5757694b6caSKeerthy memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32)));
5767694b6caSKeerthy
5777694b6caSKeerthy /* Iniialize the command update structure */
5787694b6caSKeerthy memzero_explicit(upd_info, sizeof(*upd_info));
5797694b6caSKeerthy
580d2c8ac18SKeerthy if (cfg->enc_eng_id && cfg->auth_eng_id) {
581d2c8ac18SKeerthy if (cfg->enc) {
582d2c8ac18SKeerthy auth_offset = SA_CMDL_HEADER_SIZE_BYTES;
583d2c8ac18SKeerthy enc_next_eng = cfg->auth_eng_id;
5847694b6caSKeerthy
5857694b6caSKeerthy if (cfg->iv_size)
586d2c8ac18SKeerthy auth_offset += cfg->iv_size;
587d2c8ac18SKeerthy } else {
588d2c8ac18SKeerthy enc_offset = SA_CMDL_HEADER_SIZE_BYTES;
589d2c8ac18SKeerthy auth_next_eng = cfg->enc_eng_id;
590d2c8ac18SKeerthy }
591d2c8ac18SKeerthy }
5927694b6caSKeerthy
5932dc53d00SKeerthy if (cfg->enc_eng_id) {
5947694b6caSKeerthy upd_info->flags |= SA_CMDL_UPD_ENC;
5957694b6caSKeerthy upd_info->enc_size.index = enc_offset >> 2;
5967694b6caSKeerthy upd_info->enc_offset.index = upd_info->enc_size.index + 1;
5977694b6caSKeerthy /* Encryption command label */
5987694b6caSKeerthy cmdl[enc_offset + SA_CMDL_OFFSET_NESC] = enc_next_eng;
5997694b6caSKeerthy
6007694b6caSKeerthy /* Encryption modes requiring IV */
6017694b6caSKeerthy if (cfg->iv_size) {
6027694b6caSKeerthy upd_info->flags |= SA_CMDL_UPD_ENC_IV;
6037694b6caSKeerthy upd_info->enc_iv.index =
6047694b6caSKeerthy (enc_offset + SA_CMDL_HEADER_SIZE_BYTES) >> 2;
6057694b6caSKeerthy upd_info->enc_iv.size = cfg->iv_size;
6067694b6caSKeerthy
6077694b6caSKeerthy cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
6087694b6caSKeerthy SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
6097694b6caSKeerthy
6107694b6caSKeerthy cmdl[enc_offset + SA_CMDL_OFFSET_OPTION_CTRL1] =
6117694b6caSKeerthy (SA_CTX_ENC_AUX2_OFFSET | (cfg->iv_size >> 3));
612d2c8ac18SKeerthy total += SA_CMDL_HEADER_SIZE_BYTES + cfg->iv_size;
6137694b6caSKeerthy } else {
6147694b6caSKeerthy cmdl[enc_offset + SA_CMDL_OFFSET_LABEL_LEN] =
6157694b6caSKeerthy SA_CMDL_HEADER_SIZE_BYTES;
616d2c8ac18SKeerthy total += SA_CMDL_HEADER_SIZE_BYTES;
6177694b6caSKeerthy }
6187694b6caSKeerthy }
6197694b6caSKeerthy
6202dc53d00SKeerthy if (cfg->auth_eng_id) {
6212dc53d00SKeerthy upd_info->flags |= SA_CMDL_UPD_AUTH;
6222dc53d00SKeerthy upd_info->auth_size.index = auth_offset >> 2;
6232dc53d00SKeerthy upd_info->auth_offset.index = upd_info->auth_size.index + 1;
6242dc53d00SKeerthy cmdl[auth_offset + SA_CMDL_OFFSET_NESC] = auth_next_eng;
6252dc53d00SKeerthy cmdl[auth_offset + SA_CMDL_OFFSET_LABEL_LEN] =
6262dc53d00SKeerthy SA_CMDL_HEADER_SIZE_BYTES;
6272dc53d00SKeerthy total += SA_CMDL_HEADER_SIZE_BYTES;
6282dc53d00SKeerthy }
6292dc53d00SKeerthy
6307694b6caSKeerthy total = roundup(total, 8);
6317694b6caSKeerthy
6327694b6caSKeerthy for (i = 0; i < total / 4; i++)
6337694b6caSKeerthy word_ptr[i] = swab32(word_ptr[i]);
6347694b6caSKeerthy
6357694b6caSKeerthy return total;
6367694b6caSKeerthy }
6377694b6caSKeerthy
6387694b6caSKeerthy /* Update Command label */
sa_update_cmdl(struct sa_req * req,u32 * cmdl,struct sa_cmdl_upd_info * upd_info)6397694b6caSKeerthy static inline void sa_update_cmdl(struct sa_req *req, u32 *cmdl,
6407694b6caSKeerthy struct sa_cmdl_upd_info *upd_info)
6417694b6caSKeerthy {
6427694b6caSKeerthy int i = 0, j;
6437694b6caSKeerthy
6447694b6caSKeerthy if (likely(upd_info->flags & SA_CMDL_UPD_ENC)) {
6457694b6caSKeerthy cmdl[upd_info->enc_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
6467694b6caSKeerthy cmdl[upd_info->enc_size.index] |= req->enc_size;
6477694b6caSKeerthy cmdl[upd_info->enc_offset.index] &=
6487694b6caSKeerthy ~SA_CMDL_SOP_BYPASS_LEN_MASK;
6497694b6caSKeerthy cmdl[upd_info->enc_offset.index] |=
6505876b0cbSGeert Uytterhoeven FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK,
6515876b0cbSGeert Uytterhoeven req->enc_offset);
6527694b6caSKeerthy
6537694b6caSKeerthy if (likely(upd_info->flags & SA_CMDL_UPD_ENC_IV)) {
6547694b6caSKeerthy __be32 *data = (__be32 *)&cmdl[upd_info->enc_iv.index];
6557694b6caSKeerthy u32 *enc_iv = (u32 *)req->enc_iv;
6567694b6caSKeerthy
6577694b6caSKeerthy for (j = 0; i < upd_info->enc_iv.size; i += 4, j++) {
6587694b6caSKeerthy data[j] = cpu_to_be32(*enc_iv);
6597694b6caSKeerthy enc_iv++;
6607694b6caSKeerthy }
6617694b6caSKeerthy }
6627694b6caSKeerthy }
6632dc53d00SKeerthy
6642dc53d00SKeerthy if (likely(upd_info->flags & SA_CMDL_UPD_AUTH)) {
6652dc53d00SKeerthy cmdl[upd_info->auth_size.index] &= ~SA_CMDL_PAYLOAD_LENGTH_MASK;
6662dc53d00SKeerthy cmdl[upd_info->auth_size.index] |= req->auth_size;
6672dc53d00SKeerthy cmdl[upd_info->auth_offset.index] &=
6682dc53d00SKeerthy ~SA_CMDL_SOP_BYPASS_LEN_MASK;
6692dc53d00SKeerthy cmdl[upd_info->auth_offset.index] |=
6705876b0cbSGeert Uytterhoeven FIELD_PREP(SA_CMDL_SOP_BYPASS_LEN_MASK,
6715876b0cbSGeert Uytterhoeven req->auth_offset);
6722dc53d00SKeerthy if (upd_info->flags & SA_CMDL_UPD_AUTH_IV) {
6732dc53d00SKeerthy sa_copy_iv((void *)&cmdl[upd_info->auth_iv.index],
6742dc53d00SKeerthy req->auth_iv,
6752dc53d00SKeerthy (upd_info->auth_iv.size > 8));
6762dc53d00SKeerthy }
6772dc53d00SKeerthy if (upd_info->flags & SA_CMDL_UPD_AUX_KEY) {
6782dc53d00SKeerthy int offset = (req->auth_size & 0xF) ? 4 : 0;
6792dc53d00SKeerthy
6802dc53d00SKeerthy memcpy(&cmdl[upd_info->aux_key_info.index],
6812dc53d00SKeerthy &upd_info->aux_key[offset], 16);
6822dc53d00SKeerthy }
6832dc53d00SKeerthy }
6847694b6caSKeerthy }
6857694b6caSKeerthy
6867694b6caSKeerthy /* Format SWINFO words to be sent to SA */
6877694b6caSKeerthy static
sa_set_swinfo(u8 eng_id,u16 sc_id,dma_addr_t sc_phys,u8 cmdl_present,u8 cmdl_offset,u8 flags,u8 hash_size,u32 * swinfo)6887694b6caSKeerthy void sa_set_swinfo(u8 eng_id, u16 sc_id, dma_addr_t sc_phys,
6897694b6caSKeerthy u8 cmdl_present, u8 cmdl_offset, u8 flags,
6907694b6caSKeerthy u8 hash_size, u32 *swinfo)
6917694b6caSKeerthy {
6927694b6caSKeerthy swinfo[0] = sc_id;
6935876b0cbSGeert Uytterhoeven swinfo[0] |= FIELD_PREP(SA_SW0_FLAGS_MASK, flags);
6947694b6caSKeerthy if (likely(cmdl_present))
6955876b0cbSGeert Uytterhoeven swinfo[0] |= FIELD_PREP(SA_SW0_CMDL_INFO_MASK,
6965876b0cbSGeert Uytterhoeven cmdl_offset | SA_SW0_CMDL_PRESENT);
6975876b0cbSGeert Uytterhoeven swinfo[0] |= FIELD_PREP(SA_SW0_ENG_ID_MASK, eng_id);
6987694b6caSKeerthy
6997694b6caSKeerthy swinfo[0] |= SA_SW0_DEST_INFO_PRESENT;
7007694b6caSKeerthy swinfo[1] = (u32)(sc_phys & 0xFFFFFFFFULL);
7017694b6caSKeerthy swinfo[2] = (u32)((sc_phys & 0xFFFFFFFF00000000ULL) >> 32);
7025876b0cbSGeert Uytterhoeven swinfo[2] |= FIELD_PREP(SA_SW2_EGRESS_LENGTH, hash_size);
7037694b6caSKeerthy }
7047694b6caSKeerthy
7057694b6caSKeerthy /* Dump the security context */
sa_dump_sc(u8 * buf,dma_addr_t dma_addr)7067694b6caSKeerthy static void sa_dump_sc(u8 *buf, dma_addr_t dma_addr)
7077694b6caSKeerthy {
7087694b6caSKeerthy #ifdef DEBUG
7097694b6caSKeerthy dev_info(sa_k3_dev, "Security context dump:: 0x%pad\n", &dma_addr);
7107694b6caSKeerthy print_hex_dump(KERN_CONT, "", DUMP_PREFIX_OFFSET,
7117694b6caSKeerthy 16, 1, buf, SA_CTX_MAX_SZ, false);
7127694b6caSKeerthy #endif
7137694b6caSKeerthy }
7147694b6caSKeerthy
7157694b6caSKeerthy static
sa_init_sc(struct sa_ctx_info * ctx,const struct sa_match_data * match_data,const u8 * enc_key,u16 enc_key_sz,const u8 * auth_key,u16 auth_key_sz,struct algo_data * ad,u8 enc,u32 * swinfo)7160bc42311SPeter Ujfalusi int sa_init_sc(struct sa_ctx_info *ctx, const struct sa_match_data *match_data,
7170bc42311SPeter Ujfalusi const u8 *enc_key, u16 enc_key_sz,
7180bc42311SPeter Ujfalusi const u8 *auth_key, u16 auth_key_sz,
7192dc53d00SKeerthy struct algo_data *ad, u8 enc, u32 *swinfo)
7207694b6caSKeerthy {
7217694b6caSKeerthy int enc_sc_offset = 0;
7222dc53d00SKeerthy int auth_sc_offset = 0;
7237694b6caSKeerthy u8 *sc_buf = ctx->sc;
7247694b6caSKeerthy u16 sc_id = ctx->sc_id;
725d2c8ac18SKeerthy u8 first_engine = 0;
7267694b6caSKeerthy
7277694b6caSKeerthy memzero_explicit(sc_buf, SA_CTX_MAX_SZ);
7287694b6caSKeerthy
729d2c8ac18SKeerthy if (ad->auth_eng.eng_id) {
730d2c8ac18SKeerthy if (enc)
7312dc53d00SKeerthy first_engine = ad->enc_eng.eng_id;
732d2c8ac18SKeerthy else
733d2c8ac18SKeerthy first_engine = ad->auth_eng.eng_id;
734d2c8ac18SKeerthy
7352dc53d00SKeerthy enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
7362dc53d00SKeerthy auth_sc_offset = enc_sc_offset + ad->enc_eng.sc_size;
7372dc53d00SKeerthy sc_buf[1] = SA_SCCTL_FE_AUTH_ENC;
7382dc53d00SKeerthy if (!ad->hash_size)
7392dc53d00SKeerthy return -EINVAL;
7402dc53d00SKeerthy ad->hash_size = roundup(ad->hash_size, 8);
741d2c8ac18SKeerthy
742d2c8ac18SKeerthy } else if (ad->enc_eng.eng_id && !ad->auth_eng.eng_id) {
743d2c8ac18SKeerthy enc_sc_offset = SA_CTX_PHP_PE_CTX_SZ;
744d2c8ac18SKeerthy first_engine = ad->enc_eng.eng_id;
745d2c8ac18SKeerthy sc_buf[1] = SA_SCCTL_FE_ENC;
746d2c8ac18SKeerthy ad->hash_size = ad->iv_out_size;
7472dc53d00SKeerthy }
7487694b6caSKeerthy
7497694b6caSKeerthy /* SCCTL Owner info: 0=host, 1=CP_ACE */
7507694b6caSKeerthy sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0;
7517694b6caSKeerthy memcpy(&sc_buf[2], &sc_id, 2);
7527694b6caSKeerthy sc_buf[4] = 0x0;
7530bc42311SPeter Ujfalusi sc_buf[5] = match_data->priv_id;
7540bc42311SPeter Ujfalusi sc_buf[6] = match_data->priv;
7557694b6caSKeerthy sc_buf[7] = 0x0;
7567694b6caSKeerthy
7577694b6caSKeerthy /* Prepare context for encryption engine */
7587694b6caSKeerthy if (ad->enc_eng.sc_size) {
7597694b6caSKeerthy if (sa_set_sc_enc(ad, enc_key, enc_key_sz, enc,
7607694b6caSKeerthy &sc_buf[enc_sc_offset]))
7617694b6caSKeerthy return -EINVAL;
7627694b6caSKeerthy }
7637694b6caSKeerthy
7642dc53d00SKeerthy /* Prepare context for authentication engine */
7652dc53d00SKeerthy if (ad->auth_eng.sc_size)
7662dc53d00SKeerthy sa_set_sc_auth(ad, auth_key, auth_key_sz,
7672dc53d00SKeerthy &sc_buf[auth_sc_offset]);
7682dc53d00SKeerthy
7697694b6caSKeerthy /* Set the ownership of context to CP_ACE */
7707694b6caSKeerthy sc_buf[SA_CTX_SCCTL_OWNER_OFFSET] = 0x80;
7717694b6caSKeerthy
7727694b6caSKeerthy /* swizzle the security context */
7737694b6caSKeerthy sa_swiz_128(sc_buf, SA_CTX_MAX_SZ);
7747694b6caSKeerthy
7757694b6caSKeerthy sa_set_swinfo(first_engine, ctx->sc_id, ctx->sc_phys, 1, 0,
7762dc53d00SKeerthy SA_SW_INFO_FLAG_EVICT, ad->hash_size, swinfo);
7777694b6caSKeerthy
7787694b6caSKeerthy sa_dump_sc(sc_buf, ctx->sc_phys);
7797694b6caSKeerthy
7807694b6caSKeerthy return 0;
7817694b6caSKeerthy }
7827694b6caSKeerthy
7837694b6caSKeerthy /* Free the per direction context memory */
sa_free_ctx_info(struct sa_ctx_info * ctx,struct sa_crypto_data * data)7847694b6caSKeerthy static void sa_free_ctx_info(struct sa_ctx_info *ctx,
7857694b6caSKeerthy struct sa_crypto_data *data)
7867694b6caSKeerthy {
7877694b6caSKeerthy unsigned long bn;
7887694b6caSKeerthy
7897694b6caSKeerthy bn = ctx->sc_id - data->sc_id_start;
7907694b6caSKeerthy spin_lock(&data->scid_lock);
7917694b6caSKeerthy __clear_bit(bn, data->ctx_bm);
7927694b6caSKeerthy data->sc_id--;
7937694b6caSKeerthy spin_unlock(&data->scid_lock);
7947694b6caSKeerthy
7957694b6caSKeerthy if (ctx->sc) {
7967694b6caSKeerthy dma_pool_free(data->sc_pool, ctx->sc, ctx->sc_phys);
7977694b6caSKeerthy ctx->sc = NULL;
7987694b6caSKeerthy }
7997694b6caSKeerthy }
8007694b6caSKeerthy
sa_init_ctx_info(struct sa_ctx_info * ctx,struct sa_crypto_data * data)8017694b6caSKeerthy static int sa_init_ctx_info(struct sa_ctx_info *ctx,
8027694b6caSKeerthy struct sa_crypto_data *data)
8037694b6caSKeerthy {
8047694b6caSKeerthy unsigned long bn;
8057694b6caSKeerthy int err;
8067694b6caSKeerthy
8077694b6caSKeerthy spin_lock(&data->scid_lock);
8087694b6caSKeerthy bn = find_first_zero_bit(data->ctx_bm, SA_MAX_NUM_CTX);
8097694b6caSKeerthy __set_bit(bn, data->ctx_bm);
8107694b6caSKeerthy data->sc_id++;
8117694b6caSKeerthy spin_unlock(&data->scid_lock);
8127694b6caSKeerthy
8137694b6caSKeerthy ctx->sc_id = (u16)(data->sc_id_start + bn);
8147694b6caSKeerthy
8157694b6caSKeerthy ctx->sc = dma_pool_alloc(data->sc_pool, GFP_KERNEL, &ctx->sc_phys);
8167694b6caSKeerthy if (!ctx->sc) {
8177694b6caSKeerthy dev_err(&data->pdev->dev, "Failed to allocate SC memory\n");
8187694b6caSKeerthy err = -ENOMEM;
8197694b6caSKeerthy goto scid_rollback;
8207694b6caSKeerthy }
8217694b6caSKeerthy
8227694b6caSKeerthy return 0;
8237694b6caSKeerthy
8247694b6caSKeerthy scid_rollback:
8257694b6caSKeerthy spin_lock(&data->scid_lock);
8267694b6caSKeerthy __clear_bit(bn, data->ctx_bm);
8277694b6caSKeerthy data->sc_id--;
8287694b6caSKeerthy spin_unlock(&data->scid_lock);
8297694b6caSKeerthy
8307694b6caSKeerthy return err;
8317694b6caSKeerthy }
8327694b6caSKeerthy
sa_cipher_cra_exit(struct crypto_skcipher * tfm)8337694b6caSKeerthy static void sa_cipher_cra_exit(struct crypto_skcipher *tfm)
8347694b6caSKeerthy {
8357694b6caSKeerthy struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
8367694b6caSKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
8377694b6caSKeerthy
8387694b6caSKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
8397694b6caSKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
8407694b6caSKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys);
8417694b6caSKeerthy
8427694b6caSKeerthy sa_free_ctx_info(&ctx->enc, data);
8437694b6caSKeerthy sa_free_ctx_info(&ctx->dec, data);
8447694b6caSKeerthy
845ad0bb4e4SHerbert Xu crypto_free_skcipher(ctx->fallback.skcipher);
8467694b6caSKeerthy }
8477694b6caSKeerthy
sa_cipher_cra_init(struct crypto_skcipher * tfm)8487694b6caSKeerthy static int sa_cipher_cra_init(struct crypto_skcipher *tfm)
8497694b6caSKeerthy {
8507694b6caSKeerthy struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
8517694b6caSKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
8527694b6caSKeerthy const char *name = crypto_tfm_alg_name(&tfm->base);
853ad0bb4e4SHerbert Xu struct crypto_skcipher *child;
8547694b6caSKeerthy int ret;
8557694b6caSKeerthy
8567694b6caSKeerthy memzero_explicit(ctx, sizeof(*ctx));
8577694b6caSKeerthy ctx->dev_data = data;
8587694b6caSKeerthy
8597694b6caSKeerthy ret = sa_init_ctx_info(&ctx->enc, data);
8607694b6caSKeerthy if (ret)
8617694b6caSKeerthy return ret;
8627694b6caSKeerthy ret = sa_init_ctx_info(&ctx->dec, data);
8637694b6caSKeerthy if (ret) {
8647694b6caSKeerthy sa_free_ctx_info(&ctx->enc, data);
8657694b6caSKeerthy return ret;
8667694b6caSKeerthy }
8677694b6caSKeerthy
868ad0bb4e4SHerbert Xu child = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK);
8697694b6caSKeerthy
870ad0bb4e4SHerbert Xu if (IS_ERR(child)) {
8717694b6caSKeerthy dev_err(sa_k3_dev, "Error allocating fallback algo %s\n", name);
872ad0bb4e4SHerbert Xu return PTR_ERR(child);
8737694b6caSKeerthy }
8747694b6caSKeerthy
875ad0bb4e4SHerbert Xu ctx->fallback.skcipher = child;
876ad0bb4e4SHerbert Xu crypto_skcipher_set_reqsize(tfm, crypto_skcipher_reqsize(child) +
877ad0bb4e4SHerbert Xu sizeof(struct skcipher_request));
878ad0bb4e4SHerbert Xu
8797694b6caSKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
8807694b6caSKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
8817694b6caSKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys);
8827694b6caSKeerthy return 0;
8837694b6caSKeerthy }
8847694b6caSKeerthy
sa_cipher_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen,struct algo_data * ad)8857694b6caSKeerthy static int sa_cipher_setkey(struct crypto_skcipher *tfm, const u8 *key,
8867694b6caSKeerthy unsigned int keylen, struct algo_data *ad)
8877694b6caSKeerthy {
8887694b6caSKeerthy struct sa_tfm_ctx *ctx = crypto_skcipher_ctx(tfm);
889ad0bb4e4SHerbert Xu struct crypto_skcipher *child = ctx->fallback.skcipher;
8907694b6caSKeerthy int cmdl_len;
8917694b6caSKeerthy struct sa_cmdl_cfg cfg;
8927694b6caSKeerthy int ret;
8937694b6caSKeerthy
8947694b6caSKeerthy if (keylen != AES_KEYSIZE_128 && keylen != AES_KEYSIZE_192 &&
8957694b6caSKeerthy keylen != AES_KEYSIZE_256)
8967694b6caSKeerthy return -EINVAL;
8977694b6caSKeerthy
8987694b6caSKeerthy ad->enc_eng.eng_id = SA_ENG_ID_EM1;
8997694b6caSKeerthy ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
9007694b6caSKeerthy
9017694b6caSKeerthy memzero_explicit(&cfg, sizeof(cfg));
9027694b6caSKeerthy cfg.enc_eng_id = ad->enc_eng.eng_id;
9037694b6caSKeerthy cfg.iv_size = crypto_skcipher_ivsize(tfm);
9047694b6caSKeerthy
905ad0bb4e4SHerbert Xu crypto_skcipher_clear_flags(child, CRYPTO_TFM_REQ_MASK);
906ad0bb4e4SHerbert Xu crypto_skcipher_set_flags(child, tfm->base.crt_flags &
9077694b6caSKeerthy CRYPTO_TFM_REQ_MASK);
908ad0bb4e4SHerbert Xu ret = crypto_skcipher_setkey(child, key, keylen);
9097694b6caSKeerthy if (ret)
9107694b6caSKeerthy return ret;
9117694b6caSKeerthy
9127694b6caSKeerthy /* Setup Encryption Security Context & Command label template */
9130bc42311SPeter Ujfalusi if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, key, keylen, NULL, 0,
9140bc42311SPeter Ujfalusi ad, 1, &ctx->enc.epib[1]))
9157694b6caSKeerthy goto badkey;
9167694b6caSKeerthy
9177694b6caSKeerthy cmdl_len = sa_format_cmdl_gen(&cfg,
9187694b6caSKeerthy (u8 *)ctx->enc.cmdl,
9197694b6caSKeerthy &ctx->enc.cmdl_upd_info);
9207694b6caSKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
9217694b6caSKeerthy goto badkey;
9227694b6caSKeerthy
9237694b6caSKeerthy ctx->enc.cmdl_size = cmdl_len;
9247694b6caSKeerthy
9257694b6caSKeerthy /* Setup Decryption Security Context & Command label template */
9260bc42311SPeter Ujfalusi if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, key, keylen, NULL, 0,
9270bc42311SPeter Ujfalusi ad, 0, &ctx->dec.epib[1]))
9287694b6caSKeerthy goto badkey;
9297694b6caSKeerthy
9307694b6caSKeerthy cfg.enc_eng_id = ad->enc_eng.eng_id;
9317694b6caSKeerthy cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
9327694b6caSKeerthy &ctx->dec.cmdl_upd_info);
9337694b6caSKeerthy
9347694b6caSKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
9357694b6caSKeerthy goto badkey;
9367694b6caSKeerthy
9377694b6caSKeerthy ctx->dec.cmdl_size = cmdl_len;
9387694b6caSKeerthy ctx->iv_idx = ad->iv_idx;
9397694b6caSKeerthy
9407694b6caSKeerthy return 0;
9417694b6caSKeerthy
9427694b6caSKeerthy badkey:
9437694b6caSKeerthy dev_err(sa_k3_dev, "%s: badkey\n", __func__);
9447694b6caSKeerthy return -EINVAL;
9457694b6caSKeerthy }
9467694b6caSKeerthy
sa_aes_cbc_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)9477694b6caSKeerthy static int sa_aes_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
9487694b6caSKeerthy unsigned int keylen)
9497694b6caSKeerthy {
9507694b6caSKeerthy struct algo_data ad = { 0 };
9517694b6caSKeerthy /* Convert the key size (16/24/32) to the key size index (0/1/2) */
9527694b6caSKeerthy int key_idx = (keylen >> 3) - 2;
9537694b6caSKeerthy
9547694b6caSKeerthy if (key_idx >= 3)
9557694b6caSKeerthy return -EINVAL;
9567694b6caSKeerthy
9577694b6caSKeerthy ad.mci_enc = mci_cbc_enc_array[key_idx];
9587694b6caSKeerthy ad.mci_dec = mci_cbc_dec_array[key_idx];
9597694b6caSKeerthy ad.inv_key = true;
9607694b6caSKeerthy ad.ealg_id = SA_EALG_ID_AES_CBC;
9617694b6caSKeerthy ad.iv_idx = 4;
9627694b6caSKeerthy ad.iv_out_size = 16;
9637694b6caSKeerthy
9647694b6caSKeerthy return sa_cipher_setkey(tfm, key, keylen, &ad);
9657694b6caSKeerthy }
9667694b6caSKeerthy
sa_aes_ecb_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)9677694b6caSKeerthy static int sa_aes_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
9687694b6caSKeerthy unsigned int keylen)
9697694b6caSKeerthy {
9707694b6caSKeerthy struct algo_data ad = { 0 };
9717694b6caSKeerthy /* Convert the key size (16/24/32) to the key size index (0/1/2) */
9727694b6caSKeerthy int key_idx = (keylen >> 3) - 2;
9737694b6caSKeerthy
9747694b6caSKeerthy if (key_idx >= 3)
9757694b6caSKeerthy return -EINVAL;
9767694b6caSKeerthy
9777694b6caSKeerthy ad.mci_enc = mci_ecb_enc_array[key_idx];
9787694b6caSKeerthy ad.mci_dec = mci_ecb_dec_array[key_idx];
9797694b6caSKeerthy ad.inv_key = true;
9807694b6caSKeerthy ad.ealg_id = SA_EALG_ID_AES_ECB;
9817694b6caSKeerthy
9827694b6caSKeerthy return sa_cipher_setkey(tfm, key, keylen, &ad);
9837694b6caSKeerthy }
9847694b6caSKeerthy
sa_3des_cbc_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)9857694b6caSKeerthy static int sa_3des_cbc_setkey(struct crypto_skcipher *tfm, const u8 *key,
9867694b6caSKeerthy unsigned int keylen)
9877694b6caSKeerthy {
9887694b6caSKeerthy struct algo_data ad = { 0 };
9897694b6caSKeerthy
9907694b6caSKeerthy ad.mci_enc = mci_cbc_3des_enc_array;
9917694b6caSKeerthy ad.mci_dec = mci_cbc_3des_dec_array;
9927694b6caSKeerthy ad.ealg_id = SA_EALG_ID_3DES_CBC;
9937694b6caSKeerthy ad.iv_idx = 6;
9947694b6caSKeerthy ad.iv_out_size = 8;
9957694b6caSKeerthy
9967694b6caSKeerthy return sa_cipher_setkey(tfm, key, keylen, &ad);
9977694b6caSKeerthy }
9987694b6caSKeerthy
sa_3des_ecb_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)9997694b6caSKeerthy static int sa_3des_ecb_setkey(struct crypto_skcipher *tfm, const u8 *key,
10007694b6caSKeerthy unsigned int keylen)
10017694b6caSKeerthy {
10027694b6caSKeerthy struct algo_data ad = { 0 };
10037694b6caSKeerthy
10047694b6caSKeerthy ad.mci_enc = mci_ecb_3des_enc_array;
10057694b6caSKeerthy ad.mci_dec = mci_ecb_3des_dec_array;
10067694b6caSKeerthy
10077694b6caSKeerthy return sa_cipher_setkey(tfm, key, keylen, &ad);
10087694b6caSKeerthy }
10097694b6caSKeerthy
sa_sync_from_device(struct sa_rx_data * rxd)101000c9211fSPeter Ujfalusi static void sa_sync_from_device(struct sa_rx_data *rxd)
101100c9211fSPeter Ujfalusi {
101200c9211fSPeter Ujfalusi struct sg_table *sgt;
101300c9211fSPeter Ujfalusi
101400c9211fSPeter Ujfalusi if (rxd->mapped_sg[0].dir == DMA_BIDIRECTIONAL)
101500c9211fSPeter Ujfalusi sgt = &rxd->mapped_sg[0].sgt;
101600c9211fSPeter Ujfalusi else
101700c9211fSPeter Ujfalusi sgt = &rxd->mapped_sg[1].sgt;
101800c9211fSPeter Ujfalusi
101900c9211fSPeter Ujfalusi dma_sync_sgtable_for_cpu(rxd->ddev, sgt, DMA_FROM_DEVICE);
102000c9211fSPeter Ujfalusi }
102100c9211fSPeter Ujfalusi
sa_free_sa_rx_data(struct sa_rx_data * rxd)102200c9211fSPeter Ujfalusi static void sa_free_sa_rx_data(struct sa_rx_data *rxd)
102300c9211fSPeter Ujfalusi {
102400c9211fSPeter Ujfalusi int i;
102500c9211fSPeter Ujfalusi
102600c9211fSPeter Ujfalusi for (i = 0; i < ARRAY_SIZE(rxd->mapped_sg); i++) {
102700c9211fSPeter Ujfalusi struct sa_mapped_sg *mapped_sg = &rxd->mapped_sg[i];
102800c9211fSPeter Ujfalusi
102900c9211fSPeter Ujfalusi if (mapped_sg->mapped) {
103000c9211fSPeter Ujfalusi dma_unmap_sgtable(rxd->ddev, &mapped_sg->sgt,
103100c9211fSPeter Ujfalusi mapped_sg->dir, 0);
103200c9211fSPeter Ujfalusi kfree(mapped_sg->split_sg);
103300c9211fSPeter Ujfalusi }
103400c9211fSPeter Ujfalusi }
103500c9211fSPeter Ujfalusi
103600c9211fSPeter Ujfalusi kfree(rxd);
103700c9211fSPeter Ujfalusi }
103800c9211fSPeter Ujfalusi
sa_aes_dma_in_callback(void * data)10397694b6caSKeerthy static void sa_aes_dma_in_callback(void *data)
10407694b6caSKeerthy {
1041aedf818bSYu Zhe struct sa_rx_data *rxd = data;
10427694b6caSKeerthy struct skcipher_request *req;
10437694b6caSKeerthy u32 *result;
10447694b6caSKeerthy __be32 *mdptr;
10457694b6caSKeerthy size_t ml, pl;
10467694b6caSKeerthy int i;
10477694b6caSKeerthy
104800c9211fSPeter Ujfalusi sa_sync_from_device(rxd);
10497694b6caSKeerthy req = container_of(rxd->req, struct skcipher_request, base);
10507694b6caSKeerthy
10517694b6caSKeerthy if (req->iv) {
10527694b6caSKeerthy mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl,
10537694b6caSKeerthy &ml);
10547694b6caSKeerthy result = (u32 *)req->iv;
10557694b6caSKeerthy
10567694b6caSKeerthy for (i = 0; i < (rxd->enc_iv_size / 4); i++)
10577694b6caSKeerthy result[i] = be32_to_cpu(mdptr[i + rxd->iv_idx]);
10587694b6caSKeerthy }
10597694b6caSKeerthy
106000c9211fSPeter Ujfalusi sa_free_sa_rx_data(rxd);
10617694b6caSKeerthy
10627694b6caSKeerthy skcipher_request_complete(req, 0);
10637694b6caSKeerthy }
10647694b6caSKeerthy
10657694b6caSKeerthy static void
sa_prepare_tx_desc(u32 * mdptr,u32 pslen,u32 * psdata,u32 epiblen,u32 * epib)10667694b6caSKeerthy sa_prepare_tx_desc(u32 *mdptr, u32 pslen, u32 *psdata, u32 epiblen, u32 *epib)
10677694b6caSKeerthy {
10687694b6caSKeerthy u32 *out, *in;
10697694b6caSKeerthy int i;
10707694b6caSKeerthy
10717694b6caSKeerthy for (out = mdptr, in = epib, i = 0; i < epiblen / sizeof(u32); i++)
10727694b6caSKeerthy *out++ = *in++;
10737694b6caSKeerthy
10747694b6caSKeerthy mdptr[4] = (0xFFFF << 16);
10757694b6caSKeerthy for (out = &mdptr[5], in = psdata, i = 0;
10767694b6caSKeerthy i < pslen / sizeof(u32); i++)
10777694b6caSKeerthy *out++ = *in++;
10787694b6caSKeerthy }
10797694b6caSKeerthy
sa_run(struct sa_req * req)10807694b6caSKeerthy static int sa_run(struct sa_req *req)
10817694b6caSKeerthy {
10827694b6caSKeerthy struct sa_rx_data *rxd;
10837694b6caSKeerthy gfp_t gfp_flags;
10847694b6caSKeerthy u32 cmdl[SA_MAX_CMDL_WORDS];
10857694b6caSKeerthy struct sa_crypto_data *pdata = dev_get_drvdata(sa_k3_dev);
10867694b6caSKeerthy struct device *ddev;
10877694b6caSKeerthy struct dma_chan *dma_rx;
10887694b6caSKeerthy int sg_nents, src_nents, dst_nents;
10897694b6caSKeerthy struct scatterlist *src, *dst;
10907694b6caSKeerthy size_t pl, ml, split_size;
10917694b6caSKeerthy struct sa_ctx_info *sa_ctx = req->enc ? &req->ctx->enc : &req->ctx->dec;
10927694b6caSKeerthy int ret;
10937694b6caSKeerthy struct dma_async_tx_descriptor *tx_out;
10947694b6caSKeerthy u32 *mdptr;
10957694b6caSKeerthy bool diff_dst;
10967694b6caSKeerthy enum dma_data_direction dir_src;
109700c9211fSPeter Ujfalusi struct sa_mapped_sg *mapped_sg;
10987694b6caSKeerthy
10997694b6caSKeerthy gfp_flags = req->base->flags & CRYPTO_TFM_REQ_MAY_SLEEP ?
11007694b6caSKeerthy GFP_KERNEL : GFP_ATOMIC;
11017694b6caSKeerthy
11027694b6caSKeerthy rxd = kzalloc(sizeof(*rxd), gfp_flags);
11037694b6caSKeerthy if (!rxd)
11047694b6caSKeerthy return -ENOMEM;
11057694b6caSKeerthy
11067694b6caSKeerthy if (req->src != req->dst) {
11077694b6caSKeerthy diff_dst = true;
11087694b6caSKeerthy dir_src = DMA_TO_DEVICE;
11097694b6caSKeerthy } else {
11107694b6caSKeerthy diff_dst = false;
11117694b6caSKeerthy dir_src = DMA_BIDIRECTIONAL;
11127694b6caSKeerthy }
11137694b6caSKeerthy
11147694b6caSKeerthy /*
11157694b6caSKeerthy * SA2UL has an interesting feature where the receive DMA channel
11167694b6caSKeerthy * is selected based on the data passed to the engine. Within the
11177694b6caSKeerthy * transition range, there is also a space where it is impossible
11187694b6caSKeerthy * to determine where the data will end up, and this should be
11197694b6caSKeerthy * avoided. This will be handled by the SW fallback mechanism by
11207694b6caSKeerthy * the individual algorithm implementations.
11217694b6caSKeerthy */
11227694b6caSKeerthy if (req->size >= 256)
11237694b6caSKeerthy dma_rx = pdata->dma_rx2;
11247694b6caSKeerthy else
11257694b6caSKeerthy dma_rx = pdata->dma_rx1;
11267694b6caSKeerthy
1127e8a4529dSPeter Ujfalusi ddev = dmaengine_get_dma_device(pdata->dma_tx);
112800c9211fSPeter Ujfalusi rxd->ddev = ddev;
11297694b6caSKeerthy
11307694b6caSKeerthy memcpy(cmdl, sa_ctx->cmdl, sa_ctx->cmdl_size);
11317694b6caSKeerthy
11327694b6caSKeerthy sa_update_cmdl(req, cmdl, &sa_ctx->cmdl_upd_info);
11337694b6caSKeerthy
11347694b6caSKeerthy if (req->type != CRYPTO_ALG_TYPE_AHASH) {
11357694b6caSKeerthy if (req->enc)
11367694b6caSKeerthy req->type |=
11377694b6caSKeerthy (SA_REQ_SUBTYPE_ENC << SA_REQ_SUBTYPE_SHIFT);
11387694b6caSKeerthy else
11397694b6caSKeerthy req->type |=
11407694b6caSKeerthy (SA_REQ_SUBTYPE_DEC << SA_REQ_SUBTYPE_SHIFT);
11417694b6caSKeerthy }
11427694b6caSKeerthy
11437694b6caSKeerthy cmdl[sa_ctx->cmdl_size / sizeof(u32)] = req->type;
11447694b6caSKeerthy
11457694b6caSKeerthy /*
11467694b6caSKeerthy * Map the packets, first we check if the data fits into a single
11477694b6caSKeerthy * sg entry and use that if possible. If it does not fit, we check
11487694b6caSKeerthy * if we need to do sg_split to align the scatterlist data on the
11497694b6caSKeerthy * actual data size being processed by the crypto engine.
11507694b6caSKeerthy */
11517694b6caSKeerthy src = req->src;
11527694b6caSKeerthy sg_nents = sg_nents_for_len(src, req->size);
11537694b6caSKeerthy
11547694b6caSKeerthy split_size = req->size;
11557694b6caSKeerthy
115600c9211fSPeter Ujfalusi mapped_sg = &rxd->mapped_sg[0];
11577694b6caSKeerthy if (sg_nents == 1 && split_size <= req->src->length) {
115800c9211fSPeter Ujfalusi src = &mapped_sg->static_sg;
115900c9211fSPeter Ujfalusi src_nents = 1;
11607694b6caSKeerthy sg_init_table(src, 1);
11617694b6caSKeerthy sg_set_page(src, sg_page(req->src), split_size,
11627694b6caSKeerthy req->src->offset);
116300c9211fSPeter Ujfalusi
116400c9211fSPeter Ujfalusi mapped_sg->sgt.sgl = src;
116500c9211fSPeter Ujfalusi mapped_sg->sgt.orig_nents = src_nents;
116600c9211fSPeter Ujfalusi ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1167854b7737SColin Ian King if (ret) {
1168854b7737SColin Ian King kfree(rxd);
116900c9211fSPeter Ujfalusi return ret;
1170854b7737SColin Ian King }
117100c9211fSPeter Ujfalusi
117200c9211fSPeter Ujfalusi mapped_sg->dir = dir_src;
117300c9211fSPeter Ujfalusi mapped_sg->mapped = true;
11747694b6caSKeerthy } else {
117500c9211fSPeter Ujfalusi mapped_sg->sgt.sgl = req->src;
117600c9211fSPeter Ujfalusi mapped_sg->sgt.orig_nents = sg_nents;
117700c9211fSPeter Ujfalusi ret = dma_map_sgtable(ddev, &mapped_sg->sgt, dir_src, 0);
1178854b7737SColin Ian King if (ret) {
1179854b7737SColin Ian King kfree(rxd);
118000c9211fSPeter Ujfalusi return ret;
1181854b7737SColin Ian King }
118200c9211fSPeter Ujfalusi
118300c9211fSPeter Ujfalusi mapped_sg->dir = dir_src;
118400c9211fSPeter Ujfalusi mapped_sg->mapped = true;
118500c9211fSPeter Ujfalusi
118600c9211fSPeter Ujfalusi ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents, 0, 1,
118700c9211fSPeter Ujfalusi &split_size, &src, &src_nents, gfp_flags);
11887694b6caSKeerthy if (ret) {
118900c9211fSPeter Ujfalusi src_nents = mapped_sg->sgt.nents;
119000c9211fSPeter Ujfalusi src = mapped_sg->sgt.sgl;
11917694b6caSKeerthy } else {
119200c9211fSPeter Ujfalusi mapped_sg->split_sg = src;
11937694b6caSKeerthy }
11947694b6caSKeerthy }
11957694b6caSKeerthy
119600c9211fSPeter Ujfalusi dma_sync_sgtable_for_device(ddev, &mapped_sg->sgt, DMA_TO_DEVICE);
119700c9211fSPeter Ujfalusi
11987694b6caSKeerthy if (!diff_dst) {
11997694b6caSKeerthy dst_nents = src_nents;
12007694b6caSKeerthy dst = src;
12017694b6caSKeerthy } else {
12027694b6caSKeerthy dst_nents = sg_nents_for_len(req->dst, req->size);
120300c9211fSPeter Ujfalusi mapped_sg = &rxd->mapped_sg[1];
12047694b6caSKeerthy
12057694b6caSKeerthy if (dst_nents == 1 && split_size <= req->dst->length) {
120600c9211fSPeter Ujfalusi dst = &mapped_sg->static_sg;
120700c9211fSPeter Ujfalusi dst_nents = 1;
12087694b6caSKeerthy sg_init_table(dst, 1);
12097694b6caSKeerthy sg_set_page(dst, sg_page(req->dst), split_size,
12107694b6caSKeerthy req->dst->offset);
12117694b6caSKeerthy
121200c9211fSPeter Ujfalusi mapped_sg->sgt.sgl = dst;
121300c9211fSPeter Ujfalusi mapped_sg->sgt.orig_nents = dst_nents;
121400c9211fSPeter Ujfalusi ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
121500c9211fSPeter Ujfalusi DMA_FROM_DEVICE, 0);
121600c9211fSPeter Ujfalusi if (ret)
12177694b6caSKeerthy goto err_cleanup;
121800c9211fSPeter Ujfalusi
121900c9211fSPeter Ujfalusi mapped_sg->dir = DMA_FROM_DEVICE;
122000c9211fSPeter Ujfalusi mapped_sg->mapped = true;
122100c9211fSPeter Ujfalusi } else {
122200c9211fSPeter Ujfalusi mapped_sg->sgt.sgl = req->dst;
122300c9211fSPeter Ujfalusi mapped_sg->sgt.orig_nents = dst_nents;
122400c9211fSPeter Ujfalusi ret = dma_map_sgtable(ddev, &mapped_sg->sgt,
122500c9211fSPeter Ujfalusi DMA_FROM_DEVICE, 0);
122600c9211fSPeter Ujfalusi if (ret)
122700c9211fSPeter Ujfalusi goto err_cleanup;
122800c9211fSPeter Ujfalusi
122900c9211fSPeter Ujfalusi mapped_sg->dir = DMA_FROM_DEVICE;
123000c9211fSPeter Ujfalusi mapped_sg->mapped = true;
123100c9211fSPeter Ujfalusi
123200c9211fSPeter Ujfalusi ret = sg_split(mapped_sg->sgt.sgl, mapped_sg->sgt.nents,
123300c9211fSPeter Ujfalusi 0, 1, &split_size, &dst, &dst_nents,
123400c9211fSPeter Ujfalusi gfp_flags);
123500c9211fSPeter Ujfalusi if (ret) {
123600c9211fSPeter Ujfalusi dst_nents = mapped_sg->sgt.nents;
123700c9211fSPeter Ujfalusi dst = mapped_sg->sgt.sgl;
123800c9211fSPeter Ujfalusi } else {
123900c9211fSPeter Ujfalusi mapped_sg->split_sg = dst;
124000c9211fSPeter Ujfalusi }
124100c9211fSPeter Ujfalusi }
12427694b6caSKeerthy }
12437694b6caSKeerthy
12447694b6caSKeerthy rxd->tx_in = dmaengine_prep_slave_sg(dma_rx, dst, dst_nents,
12457694b6caSKeerthy DMA_DEV_TO_MEM,
12467694b6caSKeerthy DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
12477694b6caSKeerthy if (!rxd->tx_in) {
12487694b6caSKeerthy dev_err(pdata->dev, "IN prep_slave_sg() failed\n");
12497694b6caSKeerthy ret = -EINVAL;
12507694b6caSKeerthy goto err_cleanup;
12517694b6caSKeerthy }
12527694b6caSKeerthy
12537694b6caSKeerthy rxd->req = (void *)req->base;
12547694b6caSKeerthy rxd->enc = req->enc;
12557694b6caSKeerthy rxd->iv_idx = req->ctx->iv_idx;
12567694b6caSKeerthy rxd->enc_iv_size = sa_ctx->cmdl_upd_info.enc_iv.size;
12577694b6caSKeerthy rxd->tx_in->callback = req->callback;
12587694b6caSKeerthy rxd->tx_in->callback_param = rxd;
12597694b6caSKeerthy
12607694b6caSKeerthy tx_out = dmaengine_prep_slave_sg(pdata->dma_tx, src,
12617694b6caSKeerthy src_nents, DMA_MEM_TO_DEV,
12627694b6caSKeerthy DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
12637694b6caSKeerthy
12647694b6caSKeerthy if (!tx_out) {
12657694b6caSKeerthy dev_err(pdata->dev, "OUT prep_slave_sg() failed\n");
12667694b6caSKeerthy ret = -EINVAL;
12677694b6caSKeerthy goto err_cleanup;
12687694b6caSKeerthy }
12697694b6caSKeerthy
12707694b6caSKeerthy /*
12717694b6caSKeerthy * Prepare metadata for DMA engine. This essentially describes the
12727694b6caSKeerthy * crypto algorithm to be used, data sizes, different keys etc.
12737694b6caSKeerthy */
12747694b6caSKeerthy mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(tx_out, &pl, &ml);
12757694b6caSKeerthy
12767694b6caSKeerthy sa_prepare_tx_desc(mdptr, (sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS *
12777694b6caSKeerthy sizeof(u32))), cmdl, sizeof(sa_ctx->epib),
12787694b6caSKeerthy sa_ctx->epib);
12797694b6caSKeerthy
12807694b6caSKeerthy ml = sa_ctx->cmdl_size + (SA_PSDATA_CTX_WORDS * sizeof(u32));
12817694b6caSKeerthy dmaengine_desc_set_metadata_len(tx_out, req->mdata_size);
12827694b6caSKeerthy
12837694b6caSKeerthy dmaengine_submit(tx_out);
12847694b6caSKeerthy dmaengine_submit(rxd->tx_in);
12857694b6caSKeerthy
12867694b6caSKeerthy dma_async_issue_pending(dma_rx);
12877694b6caSKeerthy dma_async_issue_pending(pdata->dma_tx);
12887694b6caSKeerthy
12897694b6caSKeerthy return -EINPROGRESS;
12907694b6caSKeerthy
12917694b6caSKeerthy err_cleanup:
129200c9211fSPeter Ujfalusi sa_free_sa_rx_data(rxd);
12937694b6caSKeerthy
12947694b6caSKeerthy return ret;
12957694b6caSKeerthy }
12967694b6caSKeerthy
sa_cipher_run(struct skcipher_request * req,u8 * iv,int enc)12977694b6caSKeerthy static int sa_cipher_run(struct skcipher_request *req, u8 *iv, int enc)
12987694b6caSKeerthy {
12997694b6caSKeerthy struct sa_tfm_ctx *ctx =
13007694b6caSKeerthy crypto_skcipher_ctx(crypto_skcipher_reqtfm(req));
13017694b6caSKeerthy struct crypto_alg *alg = req->base.tfm->__crt_alg;
13027694b6caSKeerthy struct sa_req sa_req = { 0 };
13037694b6caSKeerthy
13047694b6caSKeerthy if (!req->cryptlen)
13057694b6caSKeerthy return 0;
13067694b6caSKeerthy
13077694b6caSKeerthy if (req->cryptlen % alg->cra_blocksize)
13087694b6caSKeerthy return -EINVAL;
13097694b6caSKeerthy
13107694b6caSKeerthy /* Use SW fallback if the data size is not supported */
13117694b6caSKeerthy if (req->cryptlen > SA_MAX_DATA_SZ ||
13127694b6caSKeerthy (req->cryptlen >= SA_UNSAFE_DATA_SZ_MIN &&
13137694b6caSKeerthy req->cryptlen <= SA_UNSAFE_DATA_SZ_MAX)) {
1314ad0bb4e4SHerbert Xu struct skcipher_request *subreq = skcipher_request_ctx(req);
13157694b6caSKeerthy
1316ad0bb4e4SHerbert Xu skcipher_request_set_tfm(subreq, ctx->fallback.skcipher);
13177694b6caSKeerthy skcipher_request_set_callback(subreq, req->base.flags,
1318ad0bb4e4SHerbert Xu req->base.complete,
1319ad0bb4e4SHerbert Xu req->base.data);
13207694b6caSKeerthy skcipher_request_set_crypt(subreq, req->src, req->dst,
13217694b6caSKeerthy req->cryptlen, req->iv);
13227694b6caSKeerthy if (enc)
1323ad0bb4e4SHerbert Xu return crypto_skcipher_encrypt(subreq);
13247694b6caSKeerthy else
1325ad0bb4e4SHerbert Xu return crypto_skcipher_decrypt(subreq);
13267694b6caSKeerthy }
13277694b6caSKeerthy
13287694b6caSKeerthy sa_req.size = req->cryptlen;
13297694b6caSKeerthy sa_req.enc_size = req->cryptlen;
13307694b6caSKeerthy sa_req.src = req->src;
13317694b6caSKeerthy sa_req.dst = req->dst;
13327694b6caSKeerthy sa_req.enc_iv = iv;
13337694b6caSKeerthy sa_req.type = CRYPTO_ALG_TYPE_SKCIPHER;
13347694b6caSKeerthy sa_req.enc = enc;
13357694b6caSKeerthy sa_req.callback = sa_aes_dma_in_callback;
13367694b6caSKeerthy sa_req.mdata_size = 44;
13377694b6caSKeerthy sa_req.base = &req->base;
13387694b6caSKeerthy sa_req.ctx = ctx;
13397694b6caSKeerthy
13407694b6caSKeerthy return sa_run(&sa_req);
13417694b6caSKeerthy }
13427694b6caSKeerthy
sa_encrypt(struct skcipher_request * req)13437694b6caSKeerthy static int sa_encrypt(struct skcipher_request *req)
13447694b6caSKeerthy {
13457694b6caSKeerthy return sa_cipher_run(req, req->iv, 1);
13467694b6caSKeerthy }
13477694b6caSKeerthy
sa_decrypt(struct skcipher_request * req)13487694b6caSKeerthy static int sa_decrypt(struct skcipher_request *req)
13497694b6caSKeerthy {
13507694b6caSKeerthy return sa_cipher_run(req, req->iv, 0);
13517694b6caSKeerthy }
13527694b6caSKeerthy
sa_sha_dma_in_callback(void * data)13532dc53d00SKeerthy static void sa_sha_dma_in_callback(void *data)
13542dc53d00SKeerthy {
1355aedf818bSYu Zhe struct sa_rx_data *rxd = data;
13562dc53d00SKeerthy struct ahash_request *req;
13572dc53d00SKeerthy struct crypto_ahash *tfm;
13582dc53d00SKeerthy unsigned int authsize;
135900c9211fSPeter Ujfalusi int i;
13602dc53d00SKeerthy size_t ml, pl;
13612dc53d00SKeerthy u32 *result;
13622dc53d00SKeerthy __be32 *mdptr;
13632dc53d00SKeerthy
136400c9211fSPeter Ujfalusi sa_sync_from_device(rxd);
13652dc53d00SKeerthy req = container_of(rxd->req, struct ahash_request, base);
13662dc53d00SKeerthy tfm = crypto_ahash_reqtfm(req);
13672dc53d00SKeerthy authsize = crypto_ahash_digestsize(tfm);
13682dc53d00SKeerthy
13692dc53d00SKeerthy mdptr = (__be32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
13702dc53d00SKeerthy result = (u32 *)req->result;
13712dc53d00SKeerthy
13722dc53d00SKeerthy for (i = 0; i < (authsize / 4); i++)
13732dc53d00SKeerthy result[i] = be32_to_cpu(mdptr[i + 4]);
13742dc53d00SKeerthy
137500c9211fSPeter Ujfalusi sa_free_sa_rx_data(rxd);
13762dc53d00SKeerthy
13772dc53d00SKeerthy ahash_request_complete(req, 0);
13782dc53d00SKeerthy }
13792dc53d00SKeerthy
zero_message_process(struct ahash_request * req)13802dc53d00SKeerthy static int zero_message_process(struct ahash_request *req)
13812dc53d00SKeerthy {
13822dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
13832dc53d00SKeerthy int sa_digest_size = crypto_ahash_digestsize(tfm);
13842dc53d00SKeerthy
13852dc53d00SKeerthy switch (sa_digest_size) {
13862dc53d00SKeerthy case SHA1_DIGEST_SIZE:
13872dc53d00SKeerthy memcpy(req->result, sha1_zero_message_hash, sa_digest_size);
13882dc53d00SKeerthy break;
13892dc53d00SKeerthy case SHA256_DIGEST_SIZE:
13902dc53d00SKeerthy memcpy(req->result, sha256_zero_message_hash, sa_digest_size);
13912dc53d00SKeerthy break;
13922dc53d00SKeerthy case SHA512_DIGEST_SIZE:
13932dc53d00SKeerthy memcpy(req->result, sha512_zero_message_hash, sa_digest_size);
13942dc53d00SKeerthy break;
13952dc53d00SKeerthy default:
13962dc53d00SKeerthy return -EINVAL;
13972dc53d00SKeerthy }
13982dc53d00SKeerthy
13992dc53d00SKeerthy return 0;
14002dc53d00SKeerthy }
14012dc53d00SKeerthy
sa_sha_run(struct ahash_request * req)14022dc53d00SKeerthy static int sa_sha_run(struct ahash_request *req)
14032dc53d00SKeerthy {
14042dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(crypto_ahash_reqtfm(req));
14052dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
14062dc53d00SKeerthy struct sa_req sa_req = { 0 };
14072dc53d00SKeerthy size_t auth_len;
14082dc53d00SKeerthy
14092dc53d00SKeerthy auth_len = req->nbytes;
14102dc53d00SKeerthy
14112dc53d00SKeerthy if (!auth_len)
14122dc53d00SKeerthy return zero_message_process(req);
14132dc53d00SKeerthy
14142dc53d00SKeerthy if (auth_len > SA_MAX_DATA_SZ ||
14152dc53d00SKeerthy (auth_len >= SA_UNSAFE_DATA_SZ_MIN &&
14162dc53d00SKeerthy auth_len <= SA_UNSAFE_DATA_SZ_MAX)) {
14172dc53d00SKeerthy struct ahash_request *subreq = &rctx->fallback_req;
14182dc53d00SKeerthy int ret = 0;
14192dc53d00SKeerthy
14202dc53d00SKeerthy ahash_request_set_tfm(subreq, ctx->fallback.ahash);
14212dc53d00SKeerthy subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
14222dc53d00SKeerthy
14232dc53d00SKeerthy crypto_ahash_init(subreq);
14242dc53d00SKeerthy
14252dc53d00SKeerthy subreq->nbytes = auth_len;
14262dc53d00SKeerthy subreq->src = req->src;
14272dc53d00SKeerthy subreq->result = req->result;
14282dc53d00SKeerthy
14292dc53d00SKeerthy ret |= crypto_ahash_update(subreq);
14302dc53d00SKeerthy
14312dc53d00SKeerthy subreq->nbytes = 0;
14322dc53d00SKeerthy
14332dc53d00SKeerthy ret |= crypto_ahash_final(subreq);
14342dc53d00SKeerthy
14352dc53d00SKeerthy return ret;
14362dc53d00SKeerthy }
14372dc53d00SKeerthy
14382dc53d00SKeerthy sa_req.size = auth_len;
14392dc53d00SKeerthy sa_req.auth_size = auth_len;
14402dc53d00SKeerthy sa_req.src = req->src;
14412dc53d00SKeerthy sa_req.dst = req->src;
14422dc53d00SKeerthy sa_req.enc = true;
14432dc53d00SKeerthy sa_req.type = CRYPTO_ALG_TYPE_AHASH;
14442dc53d00SKeerthy sa_req.callback = sa_sha_dma_in_callback;
14452dc53d00SKeerthy sa_req.mdata_size = 28;
14462dc53d00SKeerthy sa_req.ctx = ctx;
14472dc53d00SKeerthy sa_req.base = &req->base;
14482dc53d00SKeerthy
14492dc53d00SKeerthy return sa_run(&sa_req);
14502dc53d00SKeerthy }
14512dc53d00SKeerthy
sa_sha_setup(struct sa_tfm_ctx * ctx,struct algo_data * ad)14522dc53d00SKeerthy static int sa_sha_setup(struct sa_tfm_ctx *ctx, struct algo_data *ad)
14532dc53d00SKeerthy {
14542dc53d00SKeerthy int bs = crypto_shash_blocksize(ctx->shash);
14552dc53d00SKeerthy int cmdl_len;
14562dc53d00SKeerthy struct sa_cmdl_cfg cfg;
14572dc53d00SKeerthy
14582dc53d00SKeerthy ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
14592dc53d00SKeerthy ad->auth_eng.eng_id = SA_ENG_ID_AM1;
14602dc53d00SKeerthy ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
14612dc53d00SKeerthy
14622dc53d00SKeerthy memset(ctx->authkey, 0, bs);
14632dc53d00SKeerthy memset(&cfg, 0, sizeof(cfg));
14642dc53d00SKeerthy cfg.aalg = ad->aalg_id;
14652dc53d00SKeerthy cfg.enc_eng_id = ad->enc_eng.eng_id;
14662dc53d00SKeerthy cfg.auth_eng_id = ad->auth_eng.eng_id;
14672dc53d00SKeerthy cfg.iv_size = 0;
14682dc53d00SKeerthy cfg.akey = NULL;
14692dc53d00SKeerthy cfg.akey_len = 0;
14702dc53d00SKeerthy
14710bc42311SPeter Ujfalusi ctx->dev_data = dev_get_drvdata(sa_k3_dev);
14722dc53d00SKeerthy /* Setup Encryption Security Context & Command label template */
14730bc42311SPeter Ujfalusi if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, NULL, 0, NULL, 0,
14740bc42311SPeter Ujfalusi ad, 0, &ctx->enc.epib[1]))
14752dc53d00SKeerthy goto badkey;
14762dc53d00SKeerthy
14772dc53d00SKeerthy cmdl_len = sa_format_cmdl_gen(&cfg,
14782dc53d00SKeerthy (u8 *)ctx->enc.cmdl,
14792dc53d00SKeerthy &ctx->enc.cmdl_upd_info);
14802dc53d00SKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
14812dc53d00SKeerthy goto badkey;
14822dc53d00SKeerthy
14832dc53d00SKeerthy ctx->enc.cmdl_size = cmdl_len;
14842dc53d00SKeerthy
14852dc53d00SKeerthy return 0;
14862dc53d00SKeerthy
14872dc53d00SKeerthy badkey:
14882dc53d00SKeerthy dev_err(sa_k3_dev, "%s: badkey\n", __func__);
14892dc53d00SKeerthy return -EINVAL;
14902dc53d00SKeerthy }
14912dc53d00SKeerthy
sa_sha_cra_init_alg(struct crypto_tfm * tfm,const char * alg_base)14922dc53d00SKeerthy static int sa_sha_cra_init_alg(struct crypto_tfm *tfm, const char *alg_base)
14932dc53d00SKeerthy {
14942dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
14952dc53d00SKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
14962dc53d00SKeerthy int ret;
14972dc53d00SKeerthy
14982dc53d00SKeerthy memset(ctx, 0, sizeof(*ctx));
14992dc53d00SKeerthy ctx->dev_data = data;
15002dc53d00SKeerthy ret = sa_init_ctx_info(&ctx->enc, data);
15012dc53d00SKeerthy if (ret)
15022dc53d00SKeerthy return ret;
15032dc53d00SKeerthy
15042dc53d00SKeerthy if (alg_base) {
15052dc53d00SKeerthy ctx->shash = crypto_alloc_shash(alg_base, 0,
15062dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK);
15072dc53d00SKeerthy if (IS_ERR(ctx->shash)) {
15082dc53d00SKeerthy dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n",
15092dc53d00SKeerthy alg_base);
15102dc53d00SKeerthy return PTR_ERR(ctx->shash);
15112dc53d00SKeerthy }
15122dc53d00SKeerthy /* for fallback */
15132dc53d00SKeerthy ctx->fallback.ahash =
15142dc53d00SKeerthy crypto_alloc_ahash(alg_base, 0,
15152dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK);
15162dc53d00SKeerthy if (IS_ERR(ctx->fallback.ahash)) {
15172dc53d00SKeerthy dev_err(ctx->dev_data->dev,
15182dc53d00SKeerthy "Could not load fallback driver\n");
15192dc53d00SKeerthy return PTR_ERR(ctx->fallback.ahash);
15202dc53d00SKeerthy }
15212dc53d00SKeerthy }
15222dc53d00SKeerthy
15232dc53d00SKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
15242dc53d00SKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
15252dc53d00SKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys);
15262dc53d00SKeerthy
15272dc53d00SKeerthy crypto_ahash_set_reqsize(__crypto_ahash_cast(tfm),
15282dc53d00SKeerthy sizeof(struct sa_sha_req_ctx) +
15292dc53d00SKeerthy crypto_ahash_reqsize(ctx->fallback.ahash));
15302dc53d00SKeerthy
15312dc53d00SKeerthy return 0;
15322dc53d00SKeerthy }
15332dc53d00SKeerthy
sa_sha_digest(struct ahash_request * req)15342dc53d00SKeerthy static int sa_sha_digest(struct ahash_request *req)
15352dc53d00SKeerthy {
15362dc53d00SKeerthy return sa_sha_run(req);
15372dc53d00SKeerthy }
15382dc53d00SKeerthy
sa_sha_init(struct ahash_request * req)15392dc53d00SKeerthy static int sa_sha_init(struct ahash_request *req)
15402dc53d00SKeerthy {
15412dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
15422dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
15432dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
15442dc53d00SKeerthy
1545ea066b7aSYueHaibing dev_dbg(sa_k3_dev, "init: digest size: %u, rctx=%p\n",
1546ea066b7aSYueHaibing crypto_ahash_digestsize(tfm), rctx);
15472dc53d00SKeerthy
15482dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
15492dc53d00SKeerthy rctx->fallback_req.base.flags =
15502dc53d00SKeerthy req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
15512dc53d00SKeerthy
15522dc53d00SKeerthy return crypto_ahash_init(&rctx->fallback_req);
15532dc53d00SKeerthy }
15542dc53d00SKeerthy
sa_sha_update(struct ahash_request * req)15552dc53d00SKeerthy static int sa_sha_update(struct ahash_request *req)
15562dc53d00SKeerthy {
15572dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
15582dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
15592dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
15602dc53d00SKeerthy
15612dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
15622dc53d00SKeerthy rctx->fallback_req.base.flags =
15632dc53d00SKeerthy req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
15642dc53d00SKeerthy rctx->fallback_req.nbytes = req->nbytes;
15652dc53d00SKeerthy rctx->fallback_req.src = req->src;
15662dc53d00SKeerthy
15672dc53d00SKeerthy return crypto_ahash_update(&rctx->fallback_req);
15682dc53d00SKeerthy }
15692dc53d00SKeerthy
sa_sha_final(struct ahash_request * req)15702dc53d00SKeerthy static int sa_sha_final(struct ahash_request *req)
15712dc53d00SKeerthy {
15722dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
15732dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
15742dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
15752dc53d00SKeerthy
15762dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
15772dc53d00SKeerthy rctx->fallback_req.base.flags =
15782dc53d00SKeerthy req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
15792dc53d00SKeerthy rctx->fallback_req.result = req->result;
15802dc53d00SKeerthy
15812dc53d00SKeerthy return crypto_ahash_final(&rctx->fallback_req);
15822dc53d00SKeerthy }
15832dc53d00SKeerthy
sa_sha_finup(struct ahash_request * req)15842dc53d00SKeerthy static int sa_sha_finup(struct ahash_request *req)
15852dc53d00SKeerthy {
15862dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
15872dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
15882dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
15892dc53d00SKeerthy
15902dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
15912dc53d00SKeerthy rctx->fallback_req.base.flags =
15922dc53d00SKeerthy req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
15932dc53d00SKeerthy
15942dc53d00SKeerthy rctx->fallback_req.nbytes = req->nbytes;
15952dc53d00SKeerthy rctx->fallback_req.src = req->src;
15962dc53d00SKeerthy rctx->fallback_req.result = req->result;
15972dc53d00SKeerthy
15982dc53d00SKeerthy return crypto_ahash_finup(&rctx->fallback_req);
15992dc53d00SKeerthy }
16002dc53d00SKeerthy
sa_sha_import(struct ahash_request * req,const void * in)16012dc53d00SKeerthy static int sa_sha_import(struct ahash_request *req, const void *in)
16022dc53d00SKeerthy {
16032dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
16042dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
16052dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
16062dc53d00SKeerthy
16072dc53d00SKeerthy ahash_request_set_tfm(&rctx->fallback_req, ctx->fallback.ahash);
16082dc53d00SKeerthy rctx->fallback_req.base.flags = req->base.flags &
16092dc53d00SKeerthy CRYPTO_TFM_REQ_MAY_SLEEP;
16102dc53d00SKeerthy
16112dc53d00SKeerthy return crypto_ahash_import(&rctx->fallback_req, in);
16122dc53d00SKeerthy }
16132dc53d00SKeerthy
sa_sha_export(struct ahash_request * req,void * out)16142dc53d00SKeerthy static int sa_sha_export(struct ahash_request *req, void *out)
16152dc53d00SKeerthy {
16162dc53d00SKeerthy struct sa_sha_req_ctx *rctx = ahash_request_ctx(req);
16172dc53d00SKeerthy struct crypto_ahash *tfm = crypto_ahash_reqtfm(req);
16182dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_ahash_ctx(tfm);
16192dc53d00SKeerthy struct ahash_request *subreq = &rctx->fallback_req;
16202dc53d00SKeerthy
16212dc53d00SKeerthy ahash_request_set_tfm(subreq, ctx->fallback.ahash);
16222dc53d00SKeerthy subreq->base.flags = req->base.flags & CRYPTO_TFM_REQ_MAY_SLEEP;
16232dc53d00SKeerthy
16242dc53d00SKeerthy return crypto_ahash_export(subreq, out);
16252dc53d00SKeerthy }
16262dc53d00SKeerthy
sa_sha1_cra_init(struct crypto_tfm * tfm)16272dc53d00SKeerthy static int sa_sha1_cra_init(struct crypto_tfm *tfm)
16282dc53d00SKeerthy {
16292dc53d00SKeerthy struct algo_data ad = { 0 };
16302dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
16312dc53d00SKeerthy
16322dc53d00SKeerthy sa_sha_cra_init_alg(tfm, "sha1");
16332dc53d00SKeerthy
16342dc53d00SKeerthy ad.aalg_id = SA_AALG_ID_SHA1;
16352dc53d00SKeerthy ad.hash_size = SHA1_DIGEST_SIZE;
16362dc53d00SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
16372dc53d00SKeerthy
16382dc53d00SKeerthy sa_sha_setup(ctx, &ad);
16392dc53d00SKeerthy
16402dc53d00SKeerthy return 0;
16412dc53d00SKeerthy }
16422dc53d00SKeerthy
sa_sha256_cra_init(struct crypto_tfm * tfm)16432dc53d00SKeerthy static int sa_sha256_cra_init(struct crypto_tfm *tfm)
16442dc53d00SKeerthy {
16452dc53d00SKeerthy struct algo_data ad = { 0 };
16462dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
16472dc53d00SKeerthy
16482dc53d00SKeerthy sa_sha_cra_init_alg(tfm, "sha256");
16492dc53d00SKeerthy
16502dc53d00SKeerthy ad.aalg_id = SA_AALG_ID_SHA2_256;
16512dc53d00SKeerthy ad.hash_size = SHA256_DIGEST_SIZE;
16522dc53d00SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
16532dc53d00SKeerthy
16542dc53d00SKeerthy sa_sha_setup(ctx, &ad);
16552dc53d00SKeerthy
16562dc53d00SKeerthy return 0;
16572dc53d00SKeerthy }
16582dc53d00SKeerthy
sa_sha512_cra_init(struct crypto_tfm * tfm)16592dc53d00SKeerthy static int sa_sha512_cra_init(struct crypto_tfm *tfm)
16602dc53d00SKeerthy {
16612dc53d00SKeerthy struct algo_data ad = { 0 };
16622dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
16632dc53d00SKeerthy
16642dc53d00SKeerthy sa_sha_cra_init_alg(tfm, "sha512");
16652dc53d00SKeerthy
16662dc53d00SKeerthy ad.aalg_id = SA_AALG_ID_SHA2_512;
16672dc53d00SKeerthy ad.hash_size = SHA512_DIGEST_SIZE;
16682dc53d00SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA512;
16692dc53d00SKeerthy
16702dc53d00SKeerthy sa_sha_setup(ctx, &ad);
16712dc53d00SKeerthy
16722dc53d00SKeerthy return 0;
16732dc53d00SKeerthy }
16742dc53d00SKeerthy
sa_sha_cra_exit(struct crypto_tfm * tfm)16752dc53d00SKeerthy static void sa_sha_cra_exit(struct crypto_tfm *tfm)
16762dc53d00SKeerthy {
16772dc53d00SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(tfm);
16782dc53d00SKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
16792dc53d00SKeerthy
16802dc53d00SKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
16812dc53d00SKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
16822dc53d00SKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys);
16832dc53d00SKeerthy
16842dc53d00SKeerthy if (crypto_tfm_alg_type(tfm) == CRYPTO_ALG_TYPE_AHASH)
16852dc53d00SKeerthy sa_free_ctx_info(&ctx->enc, data);
16862dc53d00SKeerthy
16872dc53d00SKeerthy crypto_free_shash(ctx->shash);
16882dc53d00SKeerthy crypto_free_ahash(ctx->fallback.ahash);
16892dc53d00SKeerthy }
16902dc53d00SKeerthy
sa_aead_dma_in_callback(void * data)1691d2c8ac18SKeerthy static void sa_aead_dma_in_callback(void *data)
1692d2c8ac18SKeerthy {
1693aedf818bSYu Zhe struct sa_rx_data *rxd = data;
1694d2c8ac18SKeerthy struct aead_request *req;
1695d2c8ac18SKeerthy struct crypto_aead *tfm;
1696d2c8ac18SKeerthy unsigned int start;
1697d2c8ac18SKeerthy unsigned int authsize;
1698d2c8ac18SKeerthy u8 auth_tag[SA_MAX_AUTH_TAG_SZ];
1699d2c8ac18SKeerthy size_t pl, ml;
170000c9211fSPeter Ujfalusi int i;
1701d2c8ac18SKeerthy int err = 0;
1702d2c8ac18SKeerthy u32 *mdptr;
1703d2c8ac18SKeerthy
170400c9211fSPeter Ujfalusi sa_sync_from_device(rxd);
1705d2c8ac18SKeerthy req = container_of(rxd->req, struct aead_request, base);
1706d2c8ac18SKeerthy tfm = crypto_aead_reqtfm(req);
1707d2c8ac18SKeerthy start = req->assoclen + req->cryptlen;
1708d2c8ac18SKeerthy authsize = crypto_aead_authsize(tfm);
1709d2c8ac18SKeerthy
1710d2c8ac18SKeerthy mdptr = (u32 *)dmaengine_desc_get_metadata_ptr(rxd->tx_in, &pl, &ml);
1711d2c8ac18SKeerthy for (i = 0; i < (authsize / 4); i++)
1712d2c8ac18SKeerthy mdptr[i + 4] = swab32(mdptr[i + 4]);
1713d2c8ac18SKeerthy
1714d2c8ac18SKeerthy if (rxd->enc) {
1715d2c8ac18SKeerthy scatterwalk_map_and_copy(&mdptr[4], req->dst, start, authsize,
1716d2c8ac18SKeerthy 1);
1717d2c8ac18SKeerthy } else {
1718d2c8ac18SKeerthy start -= authsize;
1719d2c8ac18SKeerthy scatterwalk_map_and_copy(auth_tag, req->src, start, authsize,
1720d2c8ac18SKeerthy 0);
1721d2c8ac18SKeerthy
1722d2c8ac18SKeerthy err = memcmp(&mdptr[4], auth_tag, authsize) ? -EBADMSG : 0;
1723d2c8ac18SKeerthy }
1724d2c8ac18SKeerthy
172500c9211fSPeter Ujfalusi sa_free_sa_rx_data(rxd);
1726d2c8ac18SKeerthy
1727d2c8ac18SKeerthy aead_request_complete(req, err);
1728d2c8ac18SKeerthy }
1729d2c8ac18SKeerthy
sa_cra_init_aead(struct crypto_aead * tfm,const char * hash,const char * fallback)1730d2c8ac18SKeerthy static int sa_cra_init_aead(struct crypto_aead *tfm, const char *hash,
1731d2c8ac18SKeerthy const char *fallback)
1732d2c8ac18SKeerthy {
1733d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1734d2c8ac18SKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1735d2c8ac18SKeerthy int ret;
1736d2c8ac18SKeerthy
1737d2c8ac18SKeerthy memzero_explicit(ctx, sizeof(*ctx));
17380bc42311SPeter Ujfalusi ctx->dev_data = data;
1739d2c8ac18SKeerthy
1740d2c8ac18SKeerthy ctx->shash = crypto_alloc_shash(hash, 0, CRYPTO_ALG_NEED_FALLBACK);
1741d2c8ac18SKeerthy if (IS_ERR(ctx->shash)) {
1742d2c8ac18SKeerthy dev_err(sa_k3_dev, "base driver %s couldn't be loaded\n", hash);
1743d2c8ac18SKeerthy return PTR_ERR(ctx->shash);
1744d2c8ac18SKeerthy }
1745d2c8ac18SKeerthy
1746d2c8ac18SKeerthy ctx->fallback.aead = crypto_alloc_aead(fallback, 0,
1747d2c8ac18SKeerthy CRYPTO_ALG_NEED_FALLBACK);
1748d2c8ac18SKeerthy
1749d2c8ac18SKeerthy if (IS_ERR(ctx->fallback.aead)) {
1750d2c8ac18SKeerthy dev_err(sa_k3_dev, "fallback driver %s couldn't be loaded\n",
1751d2c8ac18SKeerthy fallback);
1752d2c8ac18SKeerthy return PTR_ERR(ctx->fallback.aead);
1753d2c8ac18SKeerthy }
1754d2c8ac18SKeerthy
1755d2c8ac18SKeerthy crypto_aead_set_reqsize(tfm, sizeof(struct aead_request) +
1756d2c8ac18SKeerthy crypto_aead_reqsize(ctx->fallback.aead));
1757d2c8ac18SKeerthy
1758d2c8ac18SKeerthy ret = sa_init_ctx_info(&ctx->enc, data);
1759d2c8ac18SKeerthy if (ret)
1760d2c8ac18SKeerthy return ret;
1761d2c8ac18SKeerthy
1762d2c8ac18SKeerthy ret = sa_init_ctx_info(&ctx->dec, data);
1763d2c8ac18SKeerthy if (ret) {
1764d2c8ac18SKeerthy sa_free_ctx_info(&ctx->enc, data);
1765d2c8ac18SKeerthy return ret;
1766d2c8ac18SKeerthy }
1767d2c8ac18SKeerthy
1768d2c8ac18SKeerthy dev_dbg(sa_k3_dev, "%s(0x%p) sc-ids(0x%x(0x%pad), 0x%x(0x%pad))\n",
1769d2c8ac18SKeerthy __func__, tfm, ctx->enc.sc_id, &ctx->enc.sc_phys,
1770d2c8ac18SKeerthy ctx->dec.sc_id, &ctx->dec.sc_phys);
1771d2c8ac18SKeerthy
1772d2c8ac18SKeerthy return ret;
1773d2c8ac18SKeerthy }
1774d2c8ac18SKeerthy
sa_cra_init_aead_sha1(struct crypto_aead * tfm)1775d2c8ac18SKeerthy static int sa_cra_init_aead_sha1(struct crypto_aead *tfm)
1776d2c8ac18SKeerthy {
1777d2c8ac18SKeerthy return sa_cra_init_aead(tfm, "sha1",
1778d2c8ac18SKeerthy "authenc(hmac(sha1-ce),cbc(aes-ce))");
1779d2c8ac18SKeerthy }
1780d2c8ac18SKeerthy
sa_cra_init_aead_sha256(struct crypto_aead * tfm)1781d2c8ac18SKeerthy static int sa_cra_init_aead_sha256(struct crypto_aead *tfm)
1782d2c8ac18SKeerthy {
1783d2c8ac18SKeerthy return sa_cra_init_aead(tfm, "sha256",
1784d2c8ac18SKeerthy "authenc(hmac(sha256-ce),cbc(aes-ce))");
1785d2c8ac18SKeerthy }
1786d2c8ac18SKeerthy
sa_exit_tfm_aead(struct crypto_aead * tfm)1787d2c8ac18SKeerthy static void sa_exit_tfm_aead(struct crypto_aead *tfm)
1788d2c8ac18SKeerthy {
1789d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1790d2c8ac18SKeerthy struct sa_crypto_data *data = dev_get_drvdata(sa_k3_dev);
1791d2c8ac18SKeerthy
1792d2c8ac18SKeerthy crypto_free_shash(ctx->shash);
1793d2c8ac18SKeerthy crypto_free_aead(ctx->fallback.aead);
1794d2c8ac18SKeerthy
1795d2c8ac18SKeerthy sa_free_ctx_info(&ctx->enc, data);
1796d2c8ac18SKeerthy sa_free_ctx_info(&ctx->dec, data);
1797d2c8ac18SKeerthy }
1798d2c8ac18SKeerthy
1799d2c8ac18SKeerthy /* AEAD algorithm configuration interface function */
sa_aead_setkey(struct crypto_aead * authenc,const u8 * key,unsigned int keylen,struct algo_data * ad)1800d2c8ac18SKeerthy static int sa_aead_setkey(struct crypto_aead *authenc,
1801d2c8ac18SKeerthy const u8 *key, unsigned int keylen,
1802d2c8ac18SKeerthy struct algo_data *ad)
1803d2c8ac18SKeerthy {
1804d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_aead_ctx(authenc);
1805d2c8ac18SKeerthy struct crypto_authenc_keys keys;
1806d2c8ac18SKeerthy int cmdl_len;
1807d2c8ac18SKeerthy struct sa_cmdl_cfg cfg;
1808d2c8ac18SKeerthy int key_idx;
1809d2c8ac18SKeerthy
1810d2c8ac18SKeerthy if (crypto_authenc_extractkeys(&keys, key, keylen) != 0)
1811d2c8ac18SKeerthy return -EINVAL;
1812d2c8ac18SKeerthy
1813d2c8ac18SKeerthy /* Convert the key size (16/24/32) to the key size index (0/1/2) */
1814d2c8ac18SKeerthy key_idx = (keys.enckeylen >> 3) - 2;
1815d2c8ac18SKeerthy if (key_idx >= 3)
1816d2c8ac18SKeerthy return -EINVAL;
1817d2c8ac18SKeerthy
1818d2c8ac18SKeerthy ad->ctx = ctx;
1819d2c8ac18SKeerthy ad->enc_eng.eng_id = SA_ENG_ID_EM1;
1820d2c8ac18SKeerthy ad->enc_eng.sc_size = SA_CTX_ENC_TYPE1_SZ;
1821d2c8ac18SKeerthy ad->auth_eng.eng_id = SA_ENG_ID_AM1;
1822d2c8ac18SKeerthy ad->auth_eng.sc_size = SA_CTX_AUTH_TYPE2_SZ;
1823d2c8ac18SKeerthy ad->mci_enc = mci_cbc_enc_no_iv_array[key_idx];
1824d2c8ac18SKeerthy ad->mci_dec = mci_cbc_dec_no_iv_array[key_idx];
1825d2c8ac18SKeerthy ad->inv_key = true;
1826d2c8ac18SKeerthy ad->keyed_mac = true;
1827d2c8ac18SKeerthy ad->ealg_id = SA_EALG_ID_AES_CBC;
1828d2c8ac18SKeerthy ad->prep_iopad = sa_prepare_iopads;
1829d2c8ac18SKeerthy
1830d2c8ac18SKeerthy memset(&cfg, 0, sizeof(cfg));
1831d2c8ac18SKeerthy cfg.enc = true;
1832d2c8ac18SKeerthy cfg.aalg = ad->aalg_id;
1833d2c8ac18SKeerthy cfg.enc_eng_id = ad->enc_eng.eng_id;
1834d2c8ac18SKeerthy cfg.auth_eng_id = ad->auth_eng.eng_id;
1835d2c8ac18SKeerthy cfg.iv_size = crypto_aead_ivsize(authenc);
1836d2c8ac18SKeerthy cfg.akey = keys.authkey;
1837d2c8ac18SKeerthy cfg.akey_len = keys.authkeylen;
1838d2c8ac18SKeerthy
1839d2c8ac18SKeerthy /* Setup Encryption Security Context & Command label template */
18400bc42311SPeter Ujfalusi if (sa_init_sc(&ctx->enc, ctx->dev_data->match_data, keys.enckey,
18410bc42311SPeter Ujfalusi keys.enckeylen, keys.authkey, keys.authkeylen,
1842d2c8ac18SKeerthy ad, 1, &ctx->enc.epib[1]))
1843d2c8ac18SKeerthy return -EINVAL;
1844d2c8ac18SKeerthy
1845d2c8ac18SKeerthy cmdl_len = sa_format_cmdl_gen(&cfg,
1846d2c8ac18SKeerthy (u8 *)ctx->enc.cmdl,
1847d2c8ac18SKeerthy &ctx->enc.cmdl_upd_info);
1848d2c8ac18SKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1849d2c8ac18SKeerthy return -EINVAL;
1850d2c8ac18SKeerthy
1851d2c8ac18SKeerthy ctx->enc.cmdl_size = cmdl_len;
1852d2c8ac18SKeerthy
1853d2c8ac18SKeerthy /* Setup Decryption Security Context & Command label template */
18540bc42311SPeter Ujfalusi if (sa_init_sc(&ctx->dec, ctx->dev_data->match_data, keys.enckey,
18550bc42311SPeter Ujfalusi keys.enckeylen, keys.authkey, keys.authkeylen,
1856d2c8ac18SKeerthy ad, 0, &ctx->dec.epib[1]))
1857d2c8ac18SKeerthy return -EINVAL;
1858d2c8ac18SKeerthy
1859d2c8ac18SKeerthy cfg.enc = false;
1860d2c8ac18SKeerthy cmdl_len = sa_format_cmdl_gen(&cfg, (u8 *)ctx->dec.cmdl,
1861d2c8ac18SKeerthy &ctx->dec.cmdl_upd_info);
1862d2c8ac18SKeerthy
1863d2c8ac18SKeerthy if (cmdl_len <= 0 || (cmdl_len > SA_MAX_CMDL_WORDS * sizeof(u32)))
1864d2c8ac18SKeerthy return -EINVAL;
1865d2c8ac18SKeerthy
1866d2c8ac18SKeerthy ctx->dec.cmdl_size = cmdl_len;
1867d2c8ac18SKeerthy
1868d2c8ac18SKeerthy crypto_aead_clear_flags(ctx->fallback.aead, CRYPTO_TFM_REQ_MASK);
1869d2c8ac18SKeerthy crypto_aead_set_flags(ctx->fallback.aead,
1870d2c8ac18SKeerthy crypto_aead_get_flags(authenc) &
1871d2c8ac18SKeerthy CRYPTO_TFM_REQ_MASK);
1872d2c8ac18SKeerthy
1873*b94f7e34SChen Ni return crypto_aead_setkey(ctx->fallback.aead, key, keylen);
1874d2c8ac18SKeerthy }
1875d2c8ac18SKeerthy
sa_aead_setauthsize(struct crypto_aead * tfm,unsigned int authsize)1876d2c8ac18SKeerthy static int sa_aead_setauthsize(struct crypto_aead *tfm, unsigned int authsize)
1877d2c8ac18SKeerthy {
1878d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_tfm_ctx(crypto_aead_tfm(tfm));
1879d2c8ac18SKeerthy
1880d2c8ac18SKeerthy return crypto_aead_setauthsize(ctx->fallback.aead, authsize);
1881d2c8ac18SKeerthy }
1882d2c8ac18SKeerthy
sa_aead_cbc_sha1_setkey(struct crypto_aead * authenc,const u8 * key,unsigned int keylen)1883d2c8ac18SKeerthy static int sa_aead_cbc_sha1_setkey(struct crypto_aead *authenc,
1884d2c8ac18SKeerthy const u8 *key, unsigned int keylen)
1885d2c8ac18SKeerthy {
1886d2c8ac18SKeerthy struct algo_data ad = { 0 };
1887d2c8ac18SKeerthy
1888d2c8ac18SKeerthy ad.ealg_id = SA_EALG_ID_AES_CBC;
1889d2c8ac18SKeerthy ad.aalg_id = SA_AALG_ID_HMAC_SHA1;
1890d2c8ac18SKeerthy ad.hash_size = SHA1_DIGEST_SIZE;
1891d2c8ac18SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA1;
1892d2c8ac18SKeerthy
1893d2c8ac18SKeerthy return sa_aead_setkey(authenc, key, keylen, &ad);
1894d2c8ac18SKeerthy }
1895d2c8ac18SKeerthy
sa_aead_cbc_sha256_setkey(struct crypto_aead * authenc,const u8 * key,unsigned int keylen)1896d2c8ac18SKeerthy static int sa_aead_cbc_sha256_setkey(struct crypto_aead *authenc,
1897d2c8ac18SKeerthy const u8 *key, unsigned int keylen)
1898d2c8ac18SKeerthy {
1899d2c8ac18SKeerthy struct algo_data ad = { 0 };
1900d2c8ac18SKeerthy
1901d2c8ac18SKeerthy ad.ealg_id = SA_EALG_ID_AES_CBC;
1902d2c8ac18SKeerthy ad.aalg_id = SA_AALG_ID_HMAC_SHA2_256;
1903d2c8ac18SKeerthy ad.hash_size = SHA256_DIGEST_SIZE;
1904d2c8ac18SKeerthy ad.auth_ctrl = SA_AUTH_SW_CTRL_SHA256;
1905d2c8ac18SKeerthy
1906d2c8ac18SKeerthy return sa_aead_setkey(authenc, key, keylen, &ad);
1907d2c8ac18SKeerthy }
1908d2c8ac18SKeerthy
sa_aead_run(struct aead_request * req,u8 * iv,int enc)1909d2c8ac18SKeerthy static int sa_aead_run(struct aead_request *req, u8 *iv, int enc)
1910d2c8ac18SKeerthy {
1911d2c8ac18SKeerthy struct crypto_aead *tfm = crypto_aead_reqtfm(req);
1912d2c8ac18SKeerthy struct sa_tfm_ctx *ctx = crypto_aead_ctx(tfm);
1913d2c8ac18SKeerthy struct sa_req sa_req = { 0 };
1914d2c8ac18SKeerthy size_t auth_size, enc_size;
1915d2c8ac18SKeerthy
1916d2c8ac18SKeerthy enc_size = req->cryptlen;
1917d2c8ac18SKeerthy auth_size = req->assoclen + req->cryptlen;
1918d2c8ac18SKeerthy
1919d2c8ac18SKeerthy if (!enc) {
1920d2c8ac18SKeerthy enc_size -= crypto_aead_authsize(tfm);
1921d2c8ac18SKeerthy auth_size -= crypto_aead_authsize(tfm);
1922d2c8ac18SKeerthy }
1923d2c8ac18SKeerthy
1924d2c8ac18SKeerthy if (auth_size > SA_MAX_DATA_SZ ||
1925d2c8ac18SKeerthy (auth_size >= SA_UNSAFE_DATA_SZ_MIN &&
1926d2c8ac18SKeerthy auth_size <= SA_UNSAFE_DATA_SZ_MAX)) {
1927d2c8ac18SKeerthy struct aead_request *subreq = aead_request_ctx(req);
1928d2c8ac18SKeerthy int ret;
1929d2c8ac18SKeerthy
1930d2c8ac18SKeerthy aead_request_set_tfm(subreq, ctx->fallback.aead);
1931d2c8ac18SKeerthy aead_request_set_callback(subreq, req->base.flags,
1932d2c8ac18SKeerthy req->base.complete, req->base.data);
1933d2c8ac18SKeerthy aead_request_set_crypt(subreq, req->src, req->dst,
1934d2c8ac18SKeerthy req->cryptlen, req->iv);
1935d2c8ac18SKeerthy aead_request_set_ad(subreq, req->assoclen);
1936d2c8ac18SKeerthy
1937d2c8ac18SKeerthy ret = enc ? crypto_aead_encrypt(subreq) :
1938d2c8ac18SKeerthy crypto_aead_decrypt(subreq);
1939d2c8ac18SKeerthy return ret;
1940d2c8ac18SKeerthy }
1941d2c8ac18SKeerthy
1942d2c8ac18SKeerthy sa_req.enc_offset = req->assoclen;
1943d2c8ac18SKeerthy sa_req.enc_size = enc_size;
1944d2c8ac18SKeerthy sa_req.auth_size = auth_size;
1945d2c8ac18SKeerthy sa_req.size = auth_size;
1946d2c8ac18SKeerthy sa_req.enc_iv = iv;
1947d2c8ac18SKeerthy sa_req.type = CRYPTO_ALG_TYPE_AEAD;
1948d2c8ac18SKeerthy sa_req.enc = enc;
1949d2c8ac18SKeerthy sa_req.callback = sa_aead_dma_in_callback;
1950d2c8ac18SKeerthy sa_req.mdata_size = 52;
1951d2c8ac18SKeerthy sa_req.base = &req->base;
1952d2c8ac18SKeerthy sa_req.ctx = ctx;
1953d2c8ac18SKeerthy sa_req.src = req->src;
1954d2c8ac18SKeerthy sa_req.dst = req->dst;
1955d2c8ac18SKeerthy
1956d2c8ac18SKeerthy return sa_run(&sa_req);
1957d2c8ac18SKeerthy }
1958d2c8ac18SKeerthy
1959d2c8ac18SKeerthy /* AEAD algorithm encrypt interface function */
sa_aead_encrypt(struct aead_request * req)1960d2c8ac18SKeerthy static int sa_aead_encrypt(struct aead_request *req)
1961d2c8ac18SKeerthy {
1962d2c8ac18SKeerthy return sa_aead_run(req, req->iv, 1);
1963d2c8ac18SKeerthy }
1964d2c8ac18SKeerthy
1965d2c8ac18SKeerthy /* AEAD algorithm decrypt interface function */
sa_aead_decrypt(struct aead_request * req)1966d2c8ac18SKeerthy static int sa_aead_decrypt(struct aead_request *req)
1967d2c8ac18SKeerthy {
1968d2c8ac18SKeerthy return sa_aead_run(req, req->iv, 0);
1969d2c8ac18SKeerthy }
1970d2c8ac18SKeerthy
19717694b6caSKeerthy static struct sa_alg_tmpl sa_algs[] = {
19720bc42311SPeter Ujfalusi [SA_ALG_CBC_AES] = {
19737694b6caSKeerthy .type = CRYPTO_ALG_TYPE_SKCIPHER,
19747694b6caSKeerthy .alg.skcipher = {
19757694b6caSKeerthy .base.cra_name = "cbc(aes)",
19767694b6caSKeerthy .base.cra_driver_name = "cbc-aes-sa2ul",
19777694b6caSKeerthy .base.cra_priority = 30000,
19787694b6caSKeerthy .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
19797694b6caSKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY |
19807694b6caSKeerthy CRYPTO_ALG_ASYNC |
19817694b6caSKeerthy CRYPTO_ALG_NEED_FALLBACK,
19827694b6caSKeerthy .base.cra_blocksize = AES_BLOCK_SIZE,
19837694b6caSKeerthy .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
19847694b6caSKeerthy .base.cra_module = THIS_MODULE,
19857694b6caSKeerthy .init = sa_cipher_cra_init,
19867694b6caSKeerthy .exit = sa_cipher_cra_exit,
19877694b6caSKeerthy .min_keysize = AES_MIN_KEY_SIZE,
19887694b6caSKeerthy .max_keysize = AES_MAX_KEY_SIZE,
19897694b6caSKeerthy .ivsize = AES_BLOCK_SIZE,
19907694b6caSKeerthy .setkey = sa_aes_cbc_setkey,
19917694b6caSKeerthy .encrypt = sa_encrypt,
19927694b6caSKeerthy .decrypt = sa_decrypt,
19937694b6caSKeerthy }
19947694b6caSKeerthy },
19950bc42311SPeter Ujfalusi [SA_ALG_EBC_AES] = {
19967694b6caSKeerthy .type = CRYPTO_ALG_TYPE_SKCIPHER,
19977694b6caSKeerthy .alg.skcipher = {
19987694b6caSKeerthy .base.cra_name = "ecb(aes)",
19997694b6caSKeerthy .base.cra_driver_name = "ecb-aes-sa2ul",
20007694b6caSKeerthy .base.cra_priority = 30000,
20017694b6caSKeerthy .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
20027694b6caSKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY |
20037694b6caSKeerthy CRYPTO_ALG_ASYNC |
20047694b6caSKeerthy CRYPTO_ALG_NEED_FALLBACK,
20057694b6caSKeerthy .base.cra_blocksize = AES_BLOCK_SIZE,
20067694b6caSKeerthy .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
20077694b6caSKeerthy .base.cra_module = THIS_MODULE,
20087694b6caSKeerthy .init = sa_cipher_cra_init,
20097694b6caSKeerthy .exit = sa_cipher_cra_exit,
20107694b6caSKeerthy .min_keysize = AES_MIN_KEY_SIZE,
20117694b6caSKeerthy .max_keysize = AES_MAX_KEY_SIZE,
20127694b6caSKeerthy .setkey = sa_aes_ecb_setkey,
20137694b6caSKeerthy .encrypt = sa_encrypt,
20147694b6caSKeerthy .decrypt = sa_decrypt,
20157694b6caSKeerthy }
20167694b6caSKeerthy },
20170bc42311SPeter Ujfalusi [SA_ALG_CBC_DES3] = {
20187694b6caSKeerthy .type = CRYPTO_ALG_TYPE_SKCIPHER,
20197694b6caSKeerthy .alg.skcipher = {
20207694b6caSKeerthy .base.cra_name = "cbc(des3_ede)",
20217694b6caSKeerthy .base.cra_driver_name = "cbc-des3-sa2ul",
20227694b6caSKeerthy .base.cra_priority = 30000,
20237694b6caSKeerthy .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
20247694b6caSKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY |
20257694b6caSKeerthy CRYPTO_ALG_ASYNC |
20267694b6caSKeerthy CRYPTO_ALG_NEED_FALLBACK,
20277694b6caSKeerthy .base.cra_blocksize = DES_BLOCK_SIZE,
20287694b6caSKeerthy .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
20297694b6caSKeerthy .base.cra_module = THIS_MODULE,
20307694b6caSKeerthy .init = sa_cipher_cra_init,
20317694b6caSKeerthy .exit = sa_cipher_cra_exit,
20327694b6caSKeerthy .min_keysize = 3 * DES_KEY_SIZE,
20337694b6caSKeerthy .max_keysize = 3 * DES_KEY_SIZE,
20347694b6caSKeerthy .ivsize = DES_BLOCK_SIZE,
20357694b6caSKeerthy .setkey = sa_3des_cbc_setkey,
20367694b6caSKeerthy .encrypt = sa_encrypt,
20377694b6caSKeerthy .decrypt = sa_decrypt,
20387694b6caSKeerthy }
20397694b6caSKeerthy },
20400bc42311SPeter Ujfalusi [SA_ALG_ECB_DES3] = {
20417694b6caSKeerthy .type = CRYPTO_ALG_TYPE_SKCIPHER,
20427694b6caSKeerthy .alg.skcipher = {
20437694b6caSKeerthy .base.cra_name = "ecb(des3_ede)",
20447694b6caSKeerthy .base.cra_driver_name = "ecb-des3-sa2ul",
20457694b6caSKeerthy .base.cra_priority = 30000,
20467694b6caSKeerthy .base.cra_flags = CRYPTO_ALG_TYPE_SKCIPHER |
20477694b6caSKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY |
20487694b6caSKeerthy CRYPTO_ALG_ASYNC |
20497694b6caSKeerthy CRYPTO_ALG_NEED_FALLBACK,
20507694b6caSKeerthy .base.cra_blocksize = DES_BLOCK_SIZE,
20517694b6caSKeerthy .base.cra_ctxsize = sizeof(struct sa_tfm_ctx),
20527694b6caSKeerthy .base.cra_module = THIS_MODULE,
20537694b6caSKeerthy .init = sa_cipher_cra_init,
20547694b6caSKeerthy .exit = sa_cipher_cra_exit,
20557694b6caSKeerthy .min_keysize = 3 * DES_KEY_SIZE,
20567694b6caSKeerthy .max_keysize = 3 * DES_KEY_SIZE,
20577694b6caSKeerthy .setkey = sa_3des_ecb_setkey,
20587694b6caSKeerthy .encrypt = sa_encrypt,
20597694b6caSKeerthy .decrypt = sa_decrypt,
20607694b6caSKeerthy }
20617694b6caSKeerthy },
20620bc42311SPeter Ujfalusi [SA_ALG_SHA1] = {
20632dc53d00SKeerthy .type = CRYPTO_ALG_TYPE_AHASH,
20642dc53d00SKeerthy .alg.ahash = {
20652dc53d00SKeerthy .halg.base = {
20662dc53d00SKeerthy .cra_name = "sha1",
20672dc53d00SKeerthy .cra_driver_name = "sha1-sa2ul",
20682dc53d00SKeerthy .cra_priority = 400,
20692dc53d00SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AHASH |
20702dc53d00SKeerthy CRYPTO_ALG_ASYNC |
20712dc53d00SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY |
20722dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK,
20732dc53d00SKeerthy .cra_blocksize = SHA1_BLOCK_SIZE,
20742dc53d00SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx),
20752dc53d00SKeerthy .cra_module = THIS_MODULE,
20762dc53d00SKeerthy .cra_init = sa_sha1_cra_init,
20772dc53d00SKeerthy .cra_exit = sa_sha_cra_exit,
20782dc53d00SKeerthy },
20792dc53d00SKeerthy .halg.digestsize = SHA1_DIGEST_SIZE,
20802dc53d00SKeerthy .halg.statesize = sizeof(struct sa_sha_req_ctx) +
20812dc53d00SKeerthy sizeof(struct sha1_state),
20822dc53d00SKeerthy .init = sa_sha_init,
20832dc53d00SKeerthy .update = sa_sha_update,
20842dc53d00SKeerthy .final = sa_sha_final,
20852dc53d00SKeerthy .finup = sa_sha_finup,
20862dc53d00SKeerthy .digest = sa_sha_digest,
20872dc53d00SKeerthy .export = sa_sha_export,
20882dc53d00SKeerthy .import = sa_sha_import,
20892dc53d00SKeerthy },
20902dc53d00SKeerthy },
20910bc42311SPeter Ujfalusi [SA_ALG_SHA256] = {
20922dc53d00SKeerthy .type = CRYPTO_ALG_TYPE_AHASH,
20932dc53d00SKeerthy .alg.ahash = {
20942dc53d00SKeerthy .halg.base = {
20952dc53d00SKeerthy .cra_name = "sha256",
20962dc53d00SKeerthy .cra_driver_name = "sha256-sa2ul",
20972dc53d00SKeerthy .cra_priority = 400,
20982dc53d00SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AHASH |
20992dc53d00SKeerthy CRYPTO_ALG_ASYNC |
21002dc53d00SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY |
21012dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK,
21022dc53d00SKeerthy .cra_blocksize = SHA256_BLOCK_SIZE,
21032dc53d00SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx),
21042dc53d00SKeerthy .cra_module = THIS_MODULE,
21052dc53d00SKeerthy .cra_init = sa_sha256_cra_init,
21062dc53d00SKeerthy .cra_exit = sa_sha_cra_exit,
21072dc53d00SKeerthy },
21082dc53d00SKeerthy .halg.digestsize = SHA256_DIGEST_SIZE,
21092dc53d00SKeerthy .halg.statesize = sizeof(struct sa_sha_req_ctx) +
21102dc53d00SKeerthy sizeof(struct sha256_state),
21112dc53d00SKeerthy .init = sa_sha_init,
21122dc53d00SKeerthy .update = sa_sha_update,
21132dc53d00SKeerthy .final = sa_sha_final,
21142dc53d00SKeerthy .finup = sa_sha_finup,
21152dc53d00SKeerthy .digest = sa_sha_digest,
21162dc53d00SKeerthy .export = sa_sha_export,
21172dc53d00SKeerthy .import = sa_sha_import,
21182dc53d00SKeerthy },
21192dc53d00SKeerthy },
21200bc42311SPeter Ujfalusi [SA_ALG_SHA512] = {
21212dc53d00SKeerthy .type = CRYPTO_ALG_TYPE_AHASH,
21222dc53d00SKeerthy .alg.ahash = {
21232dc53d00SKeerthy .halg.base = {
21242dc53d00SKeerthy .cra_name = "sha512",
21252dc53d00SKeerthy .cra_driver_name = "sha512-sa2ul",
21262dc53d00SKeerthy .cra_priority = 400,
21272dc53d00SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AHASH |
21282dc53d00SKeerthy CRYPTO_ALG_ASYNC |
21292dc53d00SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY |
21302dc53d00SKeerthy CRYPTO_ALG_NEED_FALLBACK,
21312dc53d00SKeerthy .cra_blocksize = SHA512_BLOCK_SIZE,
21322dc53d00SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx),
21332dc53d00SKeerthy .cra_module = THIS_MODULE,
21342dc53d00SKeerthy .cra_init = sa_sha512_cra_init,
21352dc53d00SKeerthy .cra_exit = sa_sha_cra_exit,
21362dc53d00SKeerthy },
21372dc53d00SKeerthy .halg.digestsize = SHA512_DIGEST_SIZE,
21382dc53d00SKeerthy .halg.statesize = sizeof(struct sa_sha_req_ctx) +
21392dc53d00SKeerthy sizeof(struct sha512_state),
21402dc53d00SKeerthy .init = sa_sha_init,
21412dc53d00SKeerthy .update = sa_sha_update,
21422dc53d00SKeerthy .final = sa_sha_final,
21432dc53d00SKeerthy .finup = sa_sha_finup,
21442dc53d00SKeerthy .digest = sa_sha_digest,
21452dc53d00SKeerthy .export = sa_sha_export,
21462dc53d00SKeerthy .import = sa_sha_import,
21472dc53d00SKeerthy },
21482dc53d00SKeerthy },
21490bc42311SPeter Ujfalusi [SA_ALG_AUTHENC_SHA1_AES] = {
2150d2c8ac18SKeerthy .type = CRYPTO_ALG_TYPE_AEAD,
2151d2c8ac18SKeerthy .alg.aead = {
2152d2c8ac18SKeerthy .base = {
2153d2c8ac18SKeerthy .cra_name = "authenc(hmac(sha1),cbc(aes))",
2154d2c8ac18SKeerthy .cra_driver_name =
2155d2c8ac18SKeerthy "authenc(hmac(sha1),cbc(aes))-sa2ul",
2156d2c8ac18SKeerthy .cra_blocksize = AES_BLOCK_SIZE,
2157d2c8ac18SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2158d2c8ac18SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY |
2159d2c8ac18SKeerthy CRYPTO_ALG_ASYNC |
2160d2c8ac18SKeerthy CRYPTO_ALG_NEED_FALLBACK,
2161d2c8ac18SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2162d2c8ac18SKeerthy .cra_module = THIS_MODULE,
2163d2c8ac18SKeerthy .cra_priority = 3000,
2164d2c8ac18SKeerthy },
2165d2c8ac18SKeerthy .ivsize = AES_BLOCK_SIZE,
2166d2c8ac18SKeerthy .maxauthsize = SHA1_DIGEST_SIZE,
2167d2c8ac18SKeerthy
2168d2c8ac18SKeerthy .init = sa_cra_init_aead_sha1,
2169d2c8ac18SKeerthy .exit = sa_exit_tfm_aead,
2170d2c8ac18SKeerthy .setkey = sa_aead_cbc_sha1_setkey,
2171d2c8ac18SKeerthy .setauthsize = sa_aead_setauthsize,
2172d2c8ac18SKeerthy .encrypt = sa_aead_encrypt,
2173d2c8ac18SKeerthy .decrypt = sa_aead_decrypt,
2174d2c8ac18SKeerthy },
2175d2c8ac18SKeerthy },
21760bc42311SPeter Ujfalusi [SA_ALG_AUTHENC_SHA256_AES] = {
2177d2c8ac18SKeerthy .type = CRYPTO_ALG_TYPE_AEAD,
2178d2c8ac18SKeerthy .alg.aead = {
2179d2c8ac18SKeerthy .base = {
2180d2c8ac18SKeerthy .cra_name = "authenc(hmac(sha256),cbc(aes))",
2181d2c8ac18SKeerthy .cra_driver_name =
2182d2c8ac18SKeerthy "authenc(hmac(sha256),cbc(aes))-sa2ul",
2183d2c8ac18SKeerthy .cra_blocksize = AES_BLOCK_SIZE,
2184d2c8ac18SKeerthy .cra_flags = CRYPTO_ALG_TYPE_AEAD |
2185d2c8ac18SKeerthy CRYPTO_ALG_KERN_DRIVER_ONLY |
2186d2c8ac18SKeerthy CRYPTO_ALG_ASYNC |
2187d2c8ac18SKeerthy CRYPTO_ALG_NEED_FALLBACK,
2188d2c8ac18SKeerthy .cra_ctxsize = sizeof(struct sa_tfm_ctx),
2189d2c8ac18SKeerthy .cra_module = THIS_MODULE,
2190d2c8ac18SKeerthy .cra_alignmask = 0,
2191d2c8ac18SKeerthy .cra_priority = 3000,
2192d2c8ac18SKeerthy },
2193d2c8ac18SKeerthy .ivsize = AES_BLOCK_SIZE,
2194d2c8ac18SKeerthy .maxauthsize = SHA256_DIGEST_SIZE,
2195d2c8ac18SKeerthy
2196d2c8ac18SKeerthy .init = sa_cra_init_aead_sha256,
2197d2c8ac18SKeerthy .exit = sa_exit_tfm_aead,
2198d2c8ac18SKeerthy .setkey = sa_aead_cbc_sha256_setkey,
2199d2c8ac18SKeerthy .setauthsize = sa_aead_setauthsize,
2200d2c8ac18SKeerthy .encrypt = sa_aead_encrypt,
2201d2c8ac18SKeerthy .decrypt = sa_aead_decrypt,
2202d2c8ac18SKeerthy },
2203d2c8ac18SKeerthy },
22047694b6caSKeerthy };
22057694b6caSKeerthy
22067694b6caSKeerthy /* Register the algorithms in crypto framework */
sa_register_algos(struct sa_crypto_data * dev_data)22070bc42311SPeter Ujfalusi static void sa_register_algos(struct sa_crypto_data *dev_data)
22087694b6caSKeerthy {
22090bc42311SPeter Ujfalusi const struct sa_match_data *match_data = dev_data->match_data;
22100bc42311SPeter Ujfalusi struct device *dev = dev_data->dev;
22117694b6caSKeerthy char *alg_name;
22127694b6caSKeerthy u32 type;
22137694b6caSKeerthy int i, err;
22147694b6caSKeerthy
22157694b6caSKeerthy for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
22160bc42311SPeter Ujfalusi /* Skip unsupported algos */
22170bc42311SPeter Ujfalusi if (!(match_data->supported_algos & BIT(i)))
22180bc42311SPeter Ujfalusi continue;
22190bc42311SPeter Ujfalusi
22207694b6caSKeerthy type = sa_algs[i].type;
22217694b6caSKeerthy if (type == CRYPTO_ALG_TYPE_SKCIPHER) {
22227694b6caSKeerthy alg_name = sa_algs[i].alg.skcipher.base.cra_name;
22237694b6caSKeerthy err = crypto_register_skcipher(&sa_algs[i].alg.skcipher);
22242dc53d00SKeerthy } else if (type == CRYPTO_ALG_TYPE_AHASH) {
22252dc53d00SKeerthy alg_name = sa_algs[i].alg.ahash.halg.base.cra_name;
22262dc53d00SKeerthy err = crypto_register_ahash(&sa_algs[i].alg.ahash);
2227d2c8ac18SKeerthy } else if (type == CRYPTO_ALG_TYPE_AEAD) {
2228d2c8ac18SKeerthy alg_name = sa_algs[i].alg.aead.base.cra_name;
2229d2c8ac18SKeerthy err = crypto_register_aead(&sa_algs[i].alg.aead);
22307694b6caSKeerthy } else {
22317694b6caSKeerthy dev_err(dev,
22327694b6caSKeerthy "un-supported crypto algorithm (%d)",
22337694b6caSKeerthy sa_algs[i].type);
22347694b6caSKeerthy continue;
22357694b6caSKeerthy }
22367694b6caSKeerthy
22377694b6caSKeerthy if (err)
22387694b6caSKeerthy dev_err(dev, "Failed to register '%s'\n", alg_name);
22397694b6caSKeerthy else
22407694b6caSKeerthy sa_algs[i].registered = true;
22417694b6caSKeerthy }
22427694b6caSKeerthy }
22437694b6caSKeerthy
22447694b6caSKeerthy /* Unregister the algorithms in crypto framework */
sa_unregister_algos(const struct device * dev)22457694b6caSKeerthy static void sa_unregister_algos(const struct device *dev)
22467694b6caSKeerthy {
22477694b6caSKeerthy u32 type;
22487694b6caSKeerthy int i;
22497694b6caSKeerthy
22507694b6caSKeerthy for (i = 0; i < ARRAY_SIZE(sa_algs); i++) {
22517694b6caSKeerthy type = sa_algs[i].type;
22527694b6caSKeerthy if (!sa_algs[i].registered)
22537694b6caSKeerthy continue;
22547694b6caSKeerthy if (type == CRYPTO_ALG_TYPE_SKCIPHER)
22557694b6caSKeerthy crypto_unregister_skcipher(&sa_algs[i].alg.skcipher);
22562dc53d00SKeerthy else if (type == CRYPTO_ALG_TYPE_AHASH)
22572dc53d00SKeerthy crypto_unregister_ahash(&sa_algs[i].alg.ahash);
2258d2c8ac18SKeerthy else if (type == CRYPTO_ALG_TYPE_AEAD)
2259d2c8ac18SKeerthy crypto_unregister_aead(&sa_algs[i].alg.aead);
22607694b6caSKeerthy
22617694b6caSKeerthy sa_algs[i].registered = false;
22627694b6caSKeerthy }
22637694b6caSKeerthy }
22647694b6caSKeerthy
sa_init_mem(struct sa_crypto_data * dev_data)22657694b6caSKeerthy static int sa_init_mem(struct sa_crypto_data *dev_data)
22667694b6caSKeerthy {
22677694b6caSKeerthy struct device *dev = &dev_data->pdev->dev;
22687694b6caSKeerthy /* Setup dma pool for security context buffers */
22697694b6caSKeerthy dev_data->sc_pool = dma_pool_create("keystone-sc", dev,
22707694b6caSKeerthy SA_CTX_MAX_SZ, 64, 0);
22717694b6caSKeerthy if (!dev_data->sc_pool) {
22727694b6caSKeerthy dev_err(dev, "Failed to create dma pool");
22737694b6caSKeerthy return -ENOMEM;
22747694b6caSKeerthy }
22757694b6caSKeerthy
22767694b6caSKeerthy return 0;
22777694b6caSKeerthy }
22787694b6caSKeerthy
sa_dma_init(struct sa_crypto_data * dd)22797694b6caSKeerthy static int sa_dma_init(struct sa_crypto_data *dd)
22807694b6caSKeerthy {
22817694b6caSKeerthy int ret;
22827694b6caSKeerthy struct dma_slave_config cfg;
22837694b6caSKeerthy
22847694b6caSKeerthy dd->dma_rx1 = NULL;
22857694b6caSKeerthy dd->dma_tx = NULL;
22867694b6caSKeerthy dd->dma_rx2 = NULL;
22877694b6caSKeerthy
22887694b6caSKeerthy ret = dma_coerce_mask_and_coherent(dd->dev, DMA_BIT_MASK(48));
22897694b6caSKeerthy if (ret)
22907694b6caSKeerthy return ret;
22917694b6caSKeerthy
22927694b6caSKeerthy dd->dma_rx1 = dma_request_chan(dd->dev, "rx1");
22931b5d3a8bSKrzysztof Kozlowski if (IS_ERR(dd->dma_rx1))
22941b5d3a8bSKrzysztof Kozlowski return dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx1),
22951b5d3a8bSKrzysztof Kozlowski "Unable to request rx1 DMA channel\n");
22967694b6caSKeerthy
22977694b6caSKeerthy dd->dma_rx2 = dma_request_chan(dd->dev, "rx2");
22987694b6caSKeerthy if (IS_ERR(dd->dma_rx2)) {
22994c0716eeSSuman Anna ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_rx2),
23001b5d3a8bSKrzysztof Kozlowski "Unable to request rx2 DMA channel\n");
23014c0716eeSSuman Anna goto err_dma_rx2;
23027694b6caSKeerthy }
23037694b6caSKeerthy
23047694b6caSKeerthy dd->dma_tx = dma_request_chan(dd->dev, "tx");
23057694b6caSKeerthy if (IS_ERR(dd->dma_tx)) {
23061b5d3a8bSKrzysztof Kozlowski ret = dev_err_probe(dd->dev, PTR_ERR(dd->dma_tx),
23071b5d3a8bSKrzysztof Kozlowski "Unable to request tx DMA channel\n");
23087694b6caSKeerthy goto err_dma_tx;
23097694b6caSKeerthy }
23107694b6caSKeerthy
23117694b6caSKeerthy memzero_explicit(&cfg, sizeof(cfg));
23127694b6caSKeerthy
23137694b6caSKeerthy cfg.src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
23147694b6caSKeerthy cfg.dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
23157694b6caSKeerthy cfg.src_maxburst = 4;
23167694b6caSKeerthy cfg.dst_maxburst = 4;
23177694b6caSKeerthy
23187694b6caSKeerthy ret = dmaengine_slave_config(dd->dma_rx1, &cfg);
23197694b6caSKeerthy if (ret) {
23207694b6caSKeerthy dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
23217694b6caSKeerthy ret);
23224c0716eeSSuman Anna goto err_dma_config;
23237694b6caSKeerthy }
23247694b6caSKeerthy
23257694b6caSKeerthy ret = dmaengine_slave_config(dd->dma_rx2, &cfg);
23267694b6caSKeerthy if (ret) {
23277694b6caSKeerthy dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",
23287694b6caSKeerthy ret);
23294c0716eeSSuman Anna goto err_dma_config;
23307694b6caSKeerthy }
23317694b6caSKeerthy
23327694b6caSKeerthy ret = dmaengine_slave_config(dd->dma_tx, &cfg);
23337694b6caSKeerthy if (ret) {
23347694b6caSKeerthy dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",
23357694b6caSKeerthy ret);
23364c0716eeSSuman Anna goto err_dma_config;
23377694b6caSKeerthy }
23387694b6caSKeerthy
23397694b6caSKeerthy return 0;
23407694b6caSKeerthy
23414c0716eeSSuman Anna err_dma_config:
23424c0716eeSSuman Anna dma_release_channel(dd->dma_tx);
23437694b6caSKeerthy err_dma_tx:
23447694b6caSKeerthy dma_release_channel(dd->dma_rx2);
23454c0716eeSSuman Anna err_dma_rx2:
23464c0716eeSSuman Anna dma_release_channel(dd->dma_rx1);
23477694b6caSKeerthy
23487694b6caSKeerthy return ret;
23497694b6caSKeerthy }
23507694b6caSKeerthy
sa_link_child(struct device * dev,void * data)2351fd92028eSTero Kristo static int sa_link_child(struct device *dev, void *data)
2352fd92028eSTero Kristo {
2353fd92028eSTero Kristo struct device *parent = data;
2354fd92028eSTero Kristo
2355fd92028eSTero Kristo device_link_add(dev, parent, DL_FLAG_AUTOPROBE_CONSUMER);
2356fd92028eSTero Kristo
2357fd92028eSTero Kristo return 0;
2358fd92028eSTero Kristo }
2359fd92028eSTero Kristo
23600bc42311SPeter Ujfalusi static struct sa_match_data am654_match_data = {
23610bc42311SPeter Ujfalusi .priv = 1,
23620bc42311SPeter Ujfalusi .priv_id = 1,
2363a65c9a2aSAndrew Davis .supported_algos = BIT(SA_ALG_CBC_AES) |
2364a65c9a2aSAndrew Davis BIT(SA_ALG_EBC_AES) |
2365a65c9a2aSAndrew Davis BIT(SA_ALG_CBC_DES3) |
2366a65c9a2aSAndrew Davis BIT(SA_ALG_ECB_DES3) |
2367a65c9a2aSAndrew Davis BIT(SA_ALG_SHA1) |
2368a65c9a2aSAndrew Davis BIT(SA_ALG_SHA256) |
2369a65c9a2aSAndrew Davis BIT(SA_ALG_SHA512) |
2370a65c9a2aSAndrew Davis BIT(SA_ALG_AUTHENC_SHA1_AES) |
2371a65c9a2aSAndrew Davis BIT(SA_ALG_AUTHENC_SHA256_AES),
23720bc42311SPeter Ujfalusi };
23730bc42311SPeter Ujfalusi
23740bc42311SPeter Ujfalusi static struct sa_match_data am64_match_data = {
23750bc42311SPeter Ujfalusi .priv = 0,
23760bc42311SPeter Ujfalusi .priv_id = 0,
23770bc42311SPeter Ujfalusi .supported_algos = BIT(SA_ALG_CBC_AES) |
23780bc42311SPeter Ujfalusi BIT(SA_ALG_EBC_AES) |
23790bc42311SPeter Ujfalusi BIT(SA_ALG_SHA256) |
23800bc42311SPeter Ujfalusi BIT(SA_ALG_SHA512) |
23810bc42311SPeter Ujfalusi BIT(SA_ALG_AUTHENC_SHA256_AES),
23820bc42311SPeter Ujfalusi };
23830bc42311SPeter Ujfalusi
23840bc42311SPeter Ujfalusi static const struct of_device_id of_match[] = {
23850bc42311SPeter Ujfalusi { .compatible = "ti,j721e-sa2ul", .data = &am654_match_data, },
23860bc42311SPeter Ujfalusi { .compatible = "ti,am654-sa2ul", .data = &am654_match_data, },
23870bc42311SPeter Ujfalusi { .compatible = "ti,am64-sa2ul", .data = &am64_match_data, },
23885a6477eaSJayesh Choudhary { .compatible = "ti,am62-sa3ul", .data = &am64_match_data, },
23890bc42311SPeter Ujfalusi {},
23900bc42311SPeter Ujfalusi };
23910bc42311SPeter Ujfalusi MODULE_DEVICE_TABLE(of, of_match);
23920bc42311SPeter Ujfalusi
sa_ul_probe(struct platform_device * pdev)23937694b6caSKeerthy static int sa_ul_probe(struct platform_device *pdev)
23947694b6caSKeerthy {
23957694b6caSKeerthy struct device *dev = &pdev->dev;
23967694b6caSKeerthy struct device_node *node = dev->of_node;
23977694b6caSKeerthy static void __iomem *saul_base;
23987694b6caSKeerthy struct sa_crypto_data *dev_data;
2399b77e34f5SAndrew Davis u32 status, val;
24007694b6caSKeerthy int ret;
24017694b6caSKeerthy
24027694b6caSKeerthy dev_data = devm_kzalloc(dev, sizeof(*dev_data), GFP_KERNEL);
24037694b6caSKeerthy if (!dev_data)
24047694b6caSKeerthy return -ENOMEM;
24057694b6caSKeerthy
2406d699c5d0SSuman Anna dev_data->match_data = of_device_get_match_data(dev);
2407d699c5d0SSuman Anna if (!dev_data->match_data)
2408d699c5d0SSuman Anna return -ENODEV;
2409d699c5d0SSuman Anna
2410daeec738SSuman Anna saul_base = devm_platform_ioremap_resource(pdev, 0);
2411daeec738SSuman Anna if (IS_ERR(saul_base))
2412daeec738SSuman Anna return PTR_ERR(saul_base);
2413daeec738SSuman Anna
24147694b6caSKeerthy sa_k3_dev = dev;
24157694b6caSKeerthy dev_data->dev = dev;
24167694b6caSKeerthy dev_data->pdev = pdev;
2417daeec738SSuman Anna dev_data->base = saul_base;
24187694b6caSKeerthy platform_set_drvdata(pdev, dev_data);
24197694b6caSKeerthy dev_set_drvdata(sa_k3_dev, dev_data);
24207694b6caSKeerthy
24217694b6caSKeerthy pm_runtime_enable(dev);
242213343badSShixin Liu ret = pm_runtime_resume_and_get(dev);
24232baace5fSDan Carpenter if (ret < 0) {
2424284340a3STang Bin dev_err(dev, "%s: failed to get sync: %d\n", __func__, ret);
24255c855232SSuman Anna pm_runtime_disable(dev);
24267694b6caSKeerthy return ret;
24277694b6caSKeerthy }
24287694b6caSKeerthy
24297694b6caSKeerthy sa_init_mem(dev_data);
24307694b6caSKeerthy ret = sa_dma_init(dev_data);
24317694b6caSKeerthy if (ret)
24324c0716eeSSuman Anna goto destroy_dma_pool;
24337694b6caSKeerthy
24347694b6caSKeerthy spin_lock_init(&dev_data->scid_lock);
24350bc42311SPeter Ujfalusi
2436b77e34f5SAndrew Davis val = SA_EEC_ENCSS_EN | SA_EEC_AUTHSS_EN | SA_EEC_CTXCACH_EN |
24377694b6caSKeerthy SA_EEC_CPPI_PORT_IN_EN | SA_EEC_CPPI_PORT_OUT_EN |
24387694b6caSKeerthy SA_EEC_TRNG_EN;
2439b77e34f5SAndrew Davis status = readl_relaxed(saul_base + SA_ENGINE_STATUS);
2440b77e34f5SAndrew Davis /* Only enable engines if all are not already enabled */
2441b77e34f5SAndrew Davis if (val & ~status)
24427694b6caSKeerthy writel_relaxed(val, saul_base + SA_ENGINE_ENABLE_CONTROL);
24437694b6caSKeerthy
24440bc42311SPeter Ujfalusi sa_register_algos(dev_data);
24457694b6caSKeerthy
2446284340a3STang Bin ret = of_platform_populate(node, NULL, NULL, dev);
24477694b6caSKeerthy if (ret)
24487694b6caSKeerthy goto release_dma;
24497694b6caSKeerthy
2450284340a3STang Bin device_for_each_child(dev, dev, sa_link_child);
2451fd92028eSTero Kristo
24527694b6caSKeerthy return 0;
24537694b6caSKeerthy
24547694b6caSKeerthy release_dma:
2455284340a3STang Bin sa_unregister_algos(dev);
24567694b6caSKeerthy
24577694b6caSKeerthy dma_release_channel(dev_data->dma_rx2);
24587694b6caSKeerthy dma_release_channel(dev_data->dma_rx1);
24597694b6caSKeerthy dma_release_channel(dev_data->dma_tx);
24607694b6caSKeerthy
24614c0716eeSSuman Anna destroy_dma_pool:
24627694b6caSKeerthy dma_pool_destroy(dev_data->sc_pool);
24637694b6caSKeerthy
2464284340a3STang Bin pm_runtime_put_sync(dev);
2465284340a3STang Bin pm_runtime_disable(dev);
24667694b6caSKeerthy
24677694b6caSKeerthy return ret;
24687694b6caSKeerthy }
24697694b6caSKeerthy
sa_ul_remove(struct platform_device * pdev)24707694b6caSKeerthy static int sa_ul_remove(struct platform_device *pdev)
24717694b6caSKeerthy {
24727694b6caSKeerthy struct sa_crypto_data *dev_data = platform_get_drvdata(pdev);
24737694b6caSKeerthy
2474c858401cSSuman Anna of_platform_depopulate(&pdev->dev);
2475c858401cSSuman Anna
24767694b6caSKeerthy sa_unregister_algos(&pdev->dev);
24777694b6caSKeerthy
24787694b6caSKeerthy dma_release_channel(dev_data->dma_rx2);
24797694b6caSKeerthy dma_release_channel(dev_data->dma_rx1);
24807694b6caSKeerthy dma_release_channel(dev_data->dma_tx);
24817694b6caSKeerthy
24827694b6caSKeerthy dma_pool_destroy(dev_data->sc_pool);
24837694b6caSKeerthy
24847694b6caSKeerthy platform_set_drvdata(pdev, NULL);
24857694b6caSKeerthy
24867694b6caSKeerthy pm_runtime_put_sync(&pdev->dev);
24877694b6caSKeerthy pm_runtime_disable(&pdev->dev);
24887694b6caSKeerthy
24897694b6caSKeerthy return 0;
24907694b6caSKeerthy }
24917694b6caSKeerthy
24927694b6caSKeerthy static struct platform_driver sa_ul_driver = {
24937694b6caSKeerthy .probe = sa_ul_probe,
24947694b6caSKeerthy .remove = sa_ul_remove,
24957694b6caSKeerthy .driver = {
24967694b6caSKeerthy .name = "saul-crypto",
24977694b6caSKeerthy .of_match_table = of_match,
24987694b6caSKeerthy },
24997694b6caSKeerthy };
25007694b6caSKeerthy module_platform_driver(sa_ul_driver);
25017694b6caSKeerthy MODULE_LICENSE("GPL v2");
2502