1*fbf31dd5STom Zanussi // SPDX-License-Identifier: GPL-2.0-only
2*fbf31dd5STom Zanussi /*
3*fbf31dd5STom Zanussi * Intel Keem Bay OCS AES Crypto Driver.
4*fbf31dd5STom Zanussi *
5*fbf31dd5STom Zanussi * Copyright (C) 2018-2020 Intel Corporation
6*fbf31dd5STom Zanussi */
7*fbf31dd5STom Zanussi
8*fbf31dd5STom Zanussi #include <linux/dma-mapping.h>
9*fbf31dd5STom Zanussi #include <linux/interrupt.h>
10*fbf31dd5STom Zanussi #include <linux/platform_device.h>
11*fbf31dd5STom Zanussi #include <linux/slab.h>
12*fbf31dd5STom Zanussi #include <linux/swab.h>
13*fbf31dd5STom Zanussi
14*fbf31dd5STom Zanussi #include <asm/byteorder.h>
15*fbf31dd5STom Zanussi #include <asm/errno.h>
16*fbf31dd5STom Zanussi
17*fbf31dd5STom Zanussi #include <crypto/aes.h>
18*fbf31dd5STom Zanussi #include <crypto/gcm.h>
19*fbf31dd5STom Zanussi
20*fbf31dd5STom Zanussi #include "ocs-aes.h"
21*fbf31dd5STom Zanussi
22*fbf31dd5STom Zanussi #define AES_COMMAND_OFFSET 0x0000
23*fbf31dd5STom Zanussi #define AES_KEY_0_OFFSET 0x0004
24*fbf31dd5STom Zanussi #define AES_KEY_1_OFFSET 0x0008
25*fbf31dd5STom Zanussi #define AES_KEY_2_OFFSET 0x000C
26*fbf31dd5STom Zanussi #define AES_KEY_3_OFFSET 0x0010
27*fbf31dd5STom Zanussi #define AES_KEY_4_OFFSET 0x0014
28*fbf31dd5STom Zanussi #define AES_KEY_5_OFFSET 0x0018
29*fbf31dd5STom Zanussi #define AES_KEY_6_OFFSET 0x001C
30*fbf31dd5STom Zanussi #define AES_KEY_7_OFFSET 0x0020
31*fbf31dd5STom Zanussi #define AES_IV_0_OFFSET 0x0024
32*fbf31dd5STom Zanussi #define AES_IV_1_OFFSET 0x0028
33*fbf31dd5STom Zanussi #define AES_IV_2_OFFSET 0x002C
34*fbf31dd5STom Zanussi #define AES_IV_3_OFFSET 0x0030
35*fbf31dd5STom Zanussi #define AES_ACTIVE_OFFSET 0x0034
36*fbf31dd5STom Zanussi #define AES_STATUS_OFFSET 0x0038
37*fbf31dd5STom Zanussi #define AES_KEY_SIZE_OFFSET 0x0044
38*fbf31dd5STom Zanussi #define AES_IER_OFFSET 0x0048
39*fbf31dd5STom Zanussi #define AES_ISR_OFFSET 0x005C
40*fbf31dd5STom Zanussi #define AES_MULTIPURPOSE1_0_OFFSET 0x0200
41*fbf31dd5STom Zanussi #define AES_MULTIPURPOSE1_1_OFFSET 0x0204
42*fbf31dd5STom Zanussi #define AES_MULTIPURPOSE1_2_OFFSET 0x0208
43*fbf31dd5STom Zanussi #define AES_MULTIPURPOSE1_3_OFFSET 0x020C
44*fbf31dd5STom Zanussi #define AES_MULTIPURPOSE2_0_OFFSET 0x0220
45*fbf31dd5STom Zanussi #define AES_MULTIPURPOSE2_1_OFFSET 0x0224
46*fbf31dd5STom Zanussi #define AES_MULTIPURPOSE2_2_OFFSET 0x0228
47*fbf31dd5STom Zanussi #define AES_MULTIPURPOSE2_3_OFFSET 0x022C
48*fbf31dd5STom Zanussi #define AES_BYTE_ORDER_CFG_OFFSET 0x02C0
49*fbf31dd5STom Zanussi #define AES_TLEN_OFFSET 0x0300
50*fbf31dd5STom Zanussi #define AES_T_MAC_0_OFFSET 0x0304
51*fbf31dd5STom Zanussi #define AES_T_MAC_1_OFFSET 0x0308
52*fbf31dd5STom Zanussi #define AES_T_MAC_2_OFFSET 0x030C
53*fbf31dd5STom Zanussi #define AES_T_MAC_3_OFFSET 0x0310
54*fbf31dd5STom Zanussi #define AES_PLEN_OFFSET 0x0314
55*fbf31dd5STom Zanussi #define AES_A_DMA_SRC_ADDR_OFFSET 0x0400
56*fbf31dd5STom Zanussi #define AES_A_DMA_DST_ADDR_OFFSET 0x0404
57*fbf31dd5STom Zanussi #define AES_A_DMA_SRC_SIZE_OFFSET 0x0408
58*fbf31dd5STom Zanussi #define AES_A_DMA_DST_SIZE_OFFSET 0x040C
59*fbf31dd5STom Zanussi #define AES_A_DMA_DMA_MODE_OFFSET 0x0410
60*fbf31dd5STom Zanussi #define AES_A_DMA_NEXT_SRC_DESCR_OFFSET 0x0418
61*fbf31dd5STom Zanussi #define AES_A_DMA_NEXT_DST_DESCR_OFFSET 0x041C
62*fbf31dd5STom Zanussi #define AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET 0x0420
63*fbf31dd5STom Zanussi #define AES_A_DMA_LOG_OFFSET 0x0424
64*fbf31dd5STom Zanussi #define AES_A_DMA_STATUS_OFFSET 0x0428
65*fbf31dd5STom Zanussi #define AES_A_DMA_PERF_CNTR_OFFSET 0x042C
66*fbf31dd5STom Zanussi #define AES_A_DMA_MSI_ISR_OFFSET 0x0480
67*fbf31dd5STom Zanussi #define AES_A_DMA_MSI_IER_OFFSET 0x0484
68*fbf31dd5STom Zanussi #define AES_A_DMA_MSI_MASK_OFFSET 0x0488
69*fbf31dd5STom Zanussi #define AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET 0x0600
70*fbf31dd5STom Zanussi #define AES_A_DMA_OUTBUFFER_READ_FIFO_OFFSET 0x0700
71*fbf31dd5STom Zanussi
72*fbf31dd5STom Zanussi /*
73*fbf31dd5STom Zanussi * AES_A_DMA_DMA_MODE register.
74*fbf31dd5STom Zanussi * Default: 0x00000000.
75*fbf31dd5STom Zanussi * bit[31] ACTIVE
76*fbf31dd5STom Zanussi * This bit activates the DMA. When the DMA finishes, it resets
77*fbf31dd5STom Zanussi * this bit to zero.
78*fbf31dd5STom Zanussi * bit[30:26] Unused by this driver.
79*fbf31dd5STom Zanussi * bit[25] SRC_LINK_LIST_EN
80*fbf31dd5STom Zanussi * Source link list enable bit. When the linked list is terminated
81*fbf31dd5STom Zanussi * this bit is reset by the DMA.
82*fbf31dd5STom Zanussi * bit[24] DST_LINK_LIST_EN
83*fbf31dd5STom Zanussi * Destination link list enable bit. When the linked list is
84*fbf31dd5STom Zanussi * terminated this bit is reset by the DMA.
85*fbf31dd5STom Zanussi * bit[23:0] Unused by this driver.
86*fbf31dd5STom Zanussi */
87*fbf31dd5STom Zanussi #define AES_A_DMA_DMA_MODE_ACTIVE BIT(31)
88*fbf31dd5STom Zanussi #define AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN BIT(25)
89*fbf31dd5STom Zanussi #define AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN BIT(24)
90*fbf31dd5STom Zanussi
91*fbf31dd5STom Zanussi /*
92*fbf31dd5STom Zanussi * AES_ACTIVE register
93*fbf31dd5STom Zanussi * default 0x00000000
94*fbf31dd5STom Zanussi * bit[31:10] Reserved
95*fbf31dd5STom Zanussi * bit[9] LAST_ADATA
96*fbf31dd5STom Zanussi * bit[8] LAST_GCX
97*fbf31dd5STom Zanussi * bit[7:2] Reserved
98*fbf31dd5STom Zanussi * bit[1] TERMINATION
99*fbf31dd5STom Zanussi * bit[0] TRIGGER
100*fbf31dd5STom Zanussi */
101*fbf31dd5STom Zanussi #define AES_ACTIVE_LAST_ADATA BIT(9)
102*fbf31dd5STom Zanussi #define AES_ACTIVE_LAST_CCM_GCM BIT(8)
103*fbf31dd5STom Zanussi #define AES_ACTIVE_TERMINATION BIT(1)
104*fbf31dd5STom Zanussi #define AES_ACTIVE_TRIGGER BIT(0)
105*fbf31dd5STom Zanussi
106*fbf31dd5STom Zanussi #define AES_DISABLE_INT 0x00000000
107*fbf31dd5STom Zanussi #define AES_DMA_CPD_ERR_INT BIT(8)
108*fbf31dd5STom Zanussi #define AES_DMA_OUTBUF_RD_ERR_INT BIT(7)
109*fbf31dd5STom Zanussi #define AES_DMA_OUTBUF_WR_ERR_INT BIT(6)
110*fbf31dd5STom Zanussi #define AES_DMA_INBUF_RD_ERR_INT BIT(5)
111*fbf31dd5STom Zanussi #define AES_DMA_INBUF_WR_ERR_INT BIT(4)
112*fbf31dd5STom Zanussi #define AES_DMA_BAD_COMP_INT BIT(3)
113*fbf31dd5STom Zanussi #define AES_DMA_SAI_INT BIT(2)
114*fbf31dd5STom Zanussi #define AES_DMA_SRC_DONE_INT BIT(0)
115*fbf31dd5STom Zanussi #define AES_COMPLETE_INT BIT(1)
116*fbf31dd5STom Zanussi
117*fbf31dd5STom Zanussi #define AES_DMA_MSI_MASK_CLEAR BIT(0)
118*fbf31dd5STom Zanussi
119*fbf31dd5STom Zanussi #define AES_128_BIT_KEY 0x00000000
120*fbf31dd5STom Zanussi #define AES_256_BIT_KEY BIT(0)
121*fbf31dd5STom Zanussi
122*fbf31dd5STom Zanussi #define AES_DEACTIVATE_PERF_CNTR 0x00000000
123*fbf31dd5STom Zanussi #define AES_ACTIVATE_PERF_CNTR BIT(0)
124*fbf31dd5STom Zanussi
125*fbf31dd5STom Zanussi #define AES_MAX_TAG_SIZE_U32 4
126*fbf31dd5STom Zanussi
127*fbf31dd5STom Zanussi #define OCS_LL_DMA_FLAG_TERMINATE BIT(31)
128*fbf31dd5STom Zanussi
129*fbf31dd5STom Zanussi /*
130*fbf31dd5STom Zanussi * There is an inconsistency in the documentation. This is documented as a
131*fbf31dd5STom Zanussi * 11-bit value, but it is actually 10-bits.
132*fbf31dd5STom Zanussi */
133*fbf31dd5STom Zanussi #define AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK 0x3FF
134*fbf31dd5STom Zanussi
135*fbf31dd5STom Zanussi /*
136*fbf31dd5STom Zanussi * During CCM decrypt, the OCS block needs to finish processing the ciphertext
137*fbf31dd5STom Zanussi * before the tag is written. For 128-bit mode this required delay is 28 OCS
138*fbf31dd5STom Zanussi * clock cycles. For 256-bit mode it is 36 OCS clock cycles.
139*fbf31dd5STom Zanussi */
140*fbf31dd5STom Zanussi #define CCM_DECRYPT_DELAY_TAG_CLK_COUNT 36UL
141*fbf31dd5STom Zanussi
142*fbf31dd5STom Zanussi /*
143*fbf31dd5STom Zanussi * During CCM decrypt there must be a delay of at least 42 OCS clock cycles
144*fbf31dd5STom Zanussi * between setting the TRIGGER bit in AES_ACTIVE and setting the LAST_CCM_GCM
145*fbf31dd5STom Zanussi * bit in the same register (as stated in the OCS databook)
146*fbf31dd5STom Zanussi */
147*fbf31dd5STom Zanussi #define CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT 42UL
148*fbf31dd5STom Zanussi
149*fbf31dd5STom Zanussi /* See RFC3610 section 2.2 */
150*fbf31dd5STom Zanussi #define L_PRIME_MIN (1)
151*fbf31dd5STom Zanussi #define L_PRIME_MAX (7)
152*fbf31dd5STom Zanussi /*
153*fbf31dd5STom Zanussi * CCM IV format from RFC 3610 section 2.3
154*fbf31dd5STom Zanussi *
155*fbf31dd5STom Zanussi * Octet Number Contents
156*fbf31dd5STom Zanussi * ------------ ---------
157*fbf31dd5STom Zanussi * 0 Flags
158*fbf31dd5STom Zanussi * 1 ... 15-L Nonce N
159*fbf31dd5STom Zanussi * 16-L ... 15 Counter i
160*fbf31dd5STom Zanussi *
161*fbf31dd5STom Zanussi * Flags = L' = L - 1
162*fbf31dd5STom Zanussi */
163*fbf31dd5STom Zanussi #define L_PRIME_IDX 0
164*fbf31dd5STom Zanussi #define COUNTER_START(lprime) (16 - ((lprime) + 1))
165*fbf31dd5STom Zanussi #define COUNTER_LEN(lprime) ((lprime) + 1)
166*fbf31dd5STom Zanussi
167*fbf31dd5STom Zanussi enum aes_counter_mode {
168*fbf31dd5STom Zanussi AES_CTR_M_NO_INC = 0,
169*fbf31dd5STom Zanussi AES_CTR_M_32_INC = 1,
170*fbf31dd5STom Zanussi AES_CTR_M_64_INC = 2,
171*fbf31dd5STom Zanussi AES_CTR_M_128_INC = 3,
172*fbf31dd5STom Zanussi };
173*fbf31dd5STom Zanussi
174*fbf31dd5STom Zanussi /**
175*fbf31dd5STom Zanussi * struct ocs_dma_linked_list - OCS DMA linked list entry.
176*fbf31dd5STom Zanussi * @src_addr: Source address of the data.
177*fbf31dd5STom Zanussi * @src_len: Length of data to be fetched.
178*fbf31dd5STom Zanussi * @next: Next dma_list to fetch.
179*fbf31dd5STom Zanussi * @ll_flags: Flags (Freeze @ terminate) for the DMA engine.
180*fbf31dd5STom Zanussi */
181*fbf31dd5STom Zanussi struct ocs_dma_linked_list {
182*fbf31dd5STom Zanussi u32 src_addr;
183*fbf31dd5STom Zanussi u32 src_len;
184*fbf31dd5STom Zanussi u32 next;
185*fbf31dd5STom Zanussi u32 ll_flags;
186*fbf31dd5STom Zanussi } __packed;
187*fbf31dd5STom Zanussi
188*fbf31dd5STom Zanussi /*
189*fbf31dd5STom Zanussi * Set endianness of inputs and outputs
190*fbf31dd5STom Zanussi * AES_BYTE_ORDER_CFG
191*fbf31dd5STom Zanussi * default 0x00000000
192*fbf31dd5STom Zanussi * bit [10] - KEY_HI_LO_SWAP
193*fbf31dd5STom Zanussi * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD
194*fbf31dd5STom Zanussi * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD
195*fbf31dd5STom Zanussi * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD
196*fbf31dd5STom Zanussi * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD
197*fbf31dd5STom Zanussi * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD
198*fbf31dd5STom Zanussi * bit [4] - IV_SWAP_BYTES_IN_DWORD
199*fbf31dd5STom Zanussi * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD
200*fbf31dd5STom Zanussi * bit [2] - DOUT_SWAP_BYTES_IN_DWORD
201*fbf31dd5STom Zanussi * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD
202*fbf31dd5STom Zanussi * bit [0] - DOUT_SWAP_BYTES_IN_DWORD
203*fbf31dd5STom Zanussi */
aes_a_set_endianness(const struct ocs_aes_dev * aes_dev)204*fbf31dd5STom Zanussi static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev)
205*fbf31dd5STom Zanussi {
206*fbf31dd5STom Zanussi iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET);
207*fbf31dd5STom Zanussi }
208*fbf31dd5STom Zanussi
209*fbf31dd5STom Zanussi /* Trigger AES process start. */
aes_a_op_trigger(const struct ocs_aes_dev * aes_dev)210*fbf31dd5STom Zanussi static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev)
211*fbf31dd5STom Zanussi {
212*fbf31dd5STom Zanussi iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET);
213*fbf31dd5STom Zanussi }
214*fbf31dd5STom Zanussi
215*fbf31dd5STom Zanussi /* Indicate last bulk of data. */
aes_a_op_termination(const struct ocs_aes_dev * aes_dev)216*fbf31dd5STom Zanussi static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev)
217*fbf31dd5STom Zanussi {
218*fbf31dd5STom Zanussi iowrite32(AES_ACTIVE_TERMINATION,
219*fbf31dd5STom Zanussi aes_dev->base_reg + AES_ACTIVE_OFFSET);
220*fbf31dd5STom Zanussi }
221*fbf31dd5STom Zanussi
222*fbf31dd5STom Zanussi /*
223*fbf31dd5STom Zanussi * Set LAST_CCM_GCM in AES_ACTIVE register and clear all other bits.
224*fbf31dd5STom Zanussi *
225*fbf31dd5STom Zanussi * Called when DMA is programmed to fetch the last batch of data.
226*fbf31dd5STom Zanussi * - For AES-CCM it is called for the last batch of Payload data and Ciphertext
227*fbf31dd5STom Zanussi * data.
228*fbf31dd5STom Zanussi * - For AES-GCM, it is called for the last batch of Plaintext data and
229*fbf31dd5STom Zanussi * Ciphertext data.
230*fbf31dd5STom Zanussi */
aes_a_set_last_gcx(const struct ocs_aes_dev * aes_dev)231*fbf31dd5STom Zanussi static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev)
232*fbf31dd5STom Zanussi {
233*fbf31dd5STom Zanussi iowrite32(AES_ACTIVE_LAST_CCM_GCM,
234*fbf31dd5STom Zanussi aes_dev->base_reg + AES_ACTIVE_OFFSET);
235*fbf31dd5STom Zanussi }
236*fbf31dd5STom Zanussi
237*fbf31dd5STom Zanussi /* Wait for LAST_CCM_GCM bit to be unset. */
aes_a_wait_last_gcx(const struct ocs_aes_dev * aes_dev)238*fbf31dd5STom Zanussi static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev)
239*fbf31dd5STom Zanussi {
240*fbf31dd5STom Zanussi u32 aes_active_reg;
241*fbf31dd5STom Zanussi
242*fbf31dd5STom Zanussi do {
243*fbf31dd5STom Zanussi aes_active_reg = ioread32(aes_dev->base_reg +
244*fbf31dd5STom Zanussi AES_ACTIVE_OFFSET);
245*fbf31dd5STom Zanussi } while (aes_active_reg & AES_ACTIVE_LAST_CCM_GCM);
246*fbf31dd5STom Zanussi }
247*fbf31dd5STom Zanussi
248*fbf31dd5STom Zanussi /* Wait for 10 bits of input occupancy. */
aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev * aes_dev)249*fbf31dd5STom Zanussi static void aes_a_dma_wait_input_buffer_occupancy(const struct ocs_aes_dev *aes_dev)
250*fbf31dd5STom Zanussi {
251*fbf31dd5STom Zanussi u32 reg;
252*fbf31dd5STom Zanussi
253*fbf31dd5STom Zanussi do {
254*fbf31dd5STom Zanussi reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET);
255*fbf31dd5STom Zanussi } while (reg & AES_DMA_STATUS_INPUT_BUFFER_OCCUPANCY_MASK);
256*fbf31dd5STom Zanussi }
257*fbf31dd5STom Zanussi
258*fbf31dd5STom Zanussi /*
259*fbf31dd5STom Zanussi * Set LAST_CCM_GCM and LAST_ADATA bits in AES_ACTIVE register (and clear all
260*fbf31dd5STom Zanussi * other bits).
261*fbf31dd5STom Zanussi *
262*fbf31dd5STom Zanussi * Called when DMA is programmed to fetch the last batch of Associated Data
263*fbf31dd5STom Zanussi * (CCM case) or Additional Authenticated Data (GCM case).
264*fbf31dd5STom Zanussi */
aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev * aes_dev)265*fbf31dd5STom Zanussi static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev)
266*fbf31dd5STom Zanussi {
267*fbf31dd5STom Zanussi iowrite32(AES_ACTIVE_LAST_ADATA | AES_ACTIVE_LAST_CCM_GCM,
268*fbf31dd5STom Zanussi aes_dev->base_reg + AES_ACTIVE_OFFSET);
269*fbf31dd5STom Zanussi }
270*fbf31dd5STom Zanussi
271*fbf31dd5STom Zanussi /* Set DMA src and dst transfer size to 0 */
aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev * aes_dev)272*fbf31dd5STom Zanussi static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev)
273*fbf31dd5STom Zanussi {
274*fbf31dd5STom Zanussi iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
275*fbf31dd5STom Zanussi iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
276*fbf31dd5STom Zanussi }
277*fbf31dd5STom Zanussi
278*fbf31dd5STom Zanussi /* Activate DMA for zero-byte transfer case. */
aes_a_dma_active(const struct ocs_aes_dev * aes_dev)279*fbf31dd5STom Zanussi static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev)
280*fbf31dd5STom Zanussi {
281*fbf31dd5STom Zanussi iowrite32(AES_A_DMA_DMA_MODE_ACTIVE,
282*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
283*fbf31dd5STom Zanussi }
284*fbf31dd5STom Zanussi
285*fbf31dd5STom Zanussi /* Activate DMA and enable src linked list */
aes_a_dma_active_src_ll_en(const struct ocs_aes_dev * aes_dev)286*fbf31dd5STom Zanussi static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev)
287*fbf31dd5STom Zanussi {
288*fbf31dd5STom Zanussi iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
289*fbf31dd5STom Zanussi AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN,
290*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
291*fbf31dd5STom Zanussi }
292*fbf31dd5STom Zanussi
293*fbf31dd5STom Zanussi /* Activate DMA and enable dst linked list */
aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev * aes_dev)294*fbf31dd5STom Zanussi static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev)
295*fbf31dd5STom Zanussi {
296*fbf31dd5STom Zanussi iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
297*fbf31dd5STom Zanussi AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
298*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
299*fbf31dd5STom Zanussi }
300*fbf31dd5STom Zanussi
301*fbf31dd5STom Zanussi /* Activate DMA and enable src and dst linked lists */
aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev * aes_dev)302*fbf31dd5STom Zanussi static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev)
303*fbf31dd5STom Zanussi {
304*fbf31dd5STom Zanussi iowrite32(AES_A_DMA_DMA_MODE_ACTIVE |
305*fbf31dd5STom Zanussi AES_A_DMA_DMA_MODE_SRC_LINK_LIST_EN |
306*fbf31dd5STom Zanussi AES_A_DMA_DMA_MODE_DST_LINK_LIST_EN,
307*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET);
308*fbf31dd5STom Zanussi }
309*fbf31dd5STom Zanussi
310*fbf31dd5STom Zanussi /* Reset PERF_CNTR to 0 and activate it */
aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev * aes_dev)311*fbf31dd5STom Zanussi static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev)
312*fbf31dd5STom Zanussi {
313*fbf31dd5STom Zanussi iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET);
314*fbf31dd5STom Zanussi iowrite32(AES_ACTIVATE_PERF_CNTR,
315*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
316*fbf31dd5STom Zanussi }
317*fbf31dd5STom Zanussi
318*fbf31dd5STom Zanussi /* Wait until PERF_CNTR is > delay, then deactivate it */
aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev * aes_dev,int delay)319*fbf31dd5STom Zanussi static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev,
320*fbf31dd5STom Zanussi int delay)
321*fbf31dd5STom Zanussi {
322*fbf31dd5STom Zanussi while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay)
323*fbf31dd5STom Zanussi ;
324*fbf31dd5STom Zanussi iowrite32(AES_DEACTIVATE_PERF_CNTR,
325*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET);
326*fbf31dd5STom Zanussi }
327*fbf31dd5STom Zanussi
328*fbf31dd5STom Zanussi /* Disable AES and DMA IRQ. */
aes_irq_disable(struct ocs_aes_dev * aes_dev)329*fbf31dd5STom Zanussi static void aes_irq_disable(struct ocs_aes_dev *aes_dev)
330*fbf31dd5STom Zanussi {
331*fbf31dd5STom Zanussi u32 isr_val = 0;
332*fbf31dd5STom Zanussi
333*fbf31dd5STom Zanussi /* Disable interrupts */
334*fbf31dd5STom Zanussi iowrite32(AES_DISABLE_INT,
335*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
336*fbf31dd5STom Zanussi iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
337*fbf31dd5STom Zanussi
338*fbf31dd5STom Zanussi /* Clear any pending interrupt */
339*fbf31dd5STom Zanussi isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
340*fbf31dd5STom Zanussi if (isr_val)
341*fbf31dd5STom Zanussi iowrite32(isr_val,
342*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
343*fbf31dd5STom Zanussi
344*fbf31dd5STom Zanussi isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
345*fbf31dd5STom Zanussi if (isr_val)
346*fbf31dd5STom Zanussi iowrite32(isr_val,
347*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET);
348*fbf31dd5STom Zanussi
349*fbf31dd5STom Zanussi isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET);
350*fbf31dd5STom Zanussi if (isr_val)
351*fbf31dd5STom Zanussi iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET);
352*fbf31dd5STom Zanussi }
353*fbf31dd5STom Zanussi
354*fbf31dd5STom Zanussi /* Enable AES or DMA IRQ. IRQ is disabled once fired. */
aes_irq_enable(struct ocs_aes_dev * aes_dev,u8 irq)355*fbf31dd5STom Zanussi static void aes_irq_enable(struct ocs_aes_dev *aes_dev, u8 irq)
356*fbf31dd5STom Zanussi {
357*fbf31dd5STom Zanussi if (irq == AES_COMPLETE_INT) {
358*fbf31dd5STom Zanussi /* Ensure DMA error interrupts are enabled */
359*fbf31dd5STom Zanussi iowrite32(AES_DMA_CPD_ERR_INT |
360*fbf31dd5STom Zanussi AES_DMA_OUTBUF_RD_ERR_INT |
361*fbf31dd5STom Zanussi AES_DMA_OUTBUF_WR_ERR_INT |
362*fbf31dd5STom Zanussi AES_DMA_INBUF_RD_ERR_INT |
363*fbf31dd5STom Zanussi AES_DMA_INBUF_WR_ERR_INT |
364*fbf31dd5STom Zanussi AES_DMA_BAD_COMP_INT |
365*fbf31dd5STom Zanussi AES_DMA_SAI_INT,
366*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
367*fbf31dd5STom Zanussi /*
368*fbf31dd5STom Zanussi * AES_IER
369*fbf31dd5STom Zanussi * default 0x00000000
370*fbf31dd5STom Zanussi * bits [31:3] - reserved
371*fbf31dd5STom Zanussi * bit [2] - EN_SKS_ERR
372*fbf31dd5STom Zanussi * bit [1] - EN_AES_COMPLETE
373*fbf31dd5STom Zanussi * bit [0] - reserved
374*fbf31dd5STom Zanussi */
375*fbf31dd5STom Zanussi iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET);
376*fbf31dd5STom Zanussi return;
377*fbf31dd5STom Zanussi }
378*fbf31dd5STom Zanussi if (irq == AES_DMA_SRC_DONE_INT) {
379*fbf31dd5STom Zanussi /* Ensure AES interrupts are disabled */
380*fbf31dd5STom Zanussi iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET);
381*fbf31dd5STom Zanussi /*
382*fbf31dd5STom Zanussi * DMA_MSI_IER
383*fbf31dd5STom Zanussi * default 0x00000000
384*fbf31dd5STom Zanussi * bits [31:9] - reserved
385*fbf31dd5STom Zanussi * bit [8] - CPD_ERR_INT_EN
386*fbf31dd5STom Zanussi * bit [7] - OUTBUF_RD_ERR_INT_EN
387*fbf31dd5STom Zanussi * bit [6] - OUTBUF_WR_ERR_INT_EN
388*fbf31dd5STom Zanussi * bit [5] - INBUF_RD_ERR_INT_EN
389*fbf31dd5STom Zanussi * bit [4] - INBUF_WR_ERR_INT_EN
390*fbf31dd5STom Zanussi * bit [3] - BAD_COMP_INT_EN
391*fbf31dd5STom Zanussi * bit [2] - SAI_INT_EN
392*fbf31dd5STom Zanussi * bit [1] - DST_DONE_INT_EN
393*fbf31dd5STom Zanussi * bit [0] - SRC_DONE_INT_EN
394*fbf31dd5STom Zanussi */
395*fbf31dd5STom Zanussi iowrite32(AES_DMA_CPD_ERR_INT |
396*fbf31dd5STom Zanussi AES_DMA_OUTBUF_RD_ERR_INT |
397*fbf31dd5STom Zanussi AES_DMA_OUTBUF_WR_ERR_INT |
398*fbf31dd5STom Zanussi AES_DMA_INBUF_RD_ERR_INT |
399*fbf31dd5STom Zanussi AES_DMA_INBUF_WR_ERR_INT |
400*fbf31dd5STom Zanussi AES_DMA_BAD_COMP_INT |
401*fbf31dd5STom Zanussi AES_DMA_SAI_INT |
402*fbf31dd5STom Zanussi AES_DMA_SRC_DONE_INT,
403*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET);
404*fbf31dd5STom Zanussi }
405*fbf31dd5STom Zanussi }
406*fbf31dd5STom Zanussi
407*fbf31dd5STom Zanussi /* Enable and wait for IRQ (either from OCS AES engine or DMA) */
ocs_aes_irq_enable_and_wait(struct ocs_aes_dev * aes_dev,u8 irq)408*fbf31dd5STom Zanussi static int ocs_aes_irq_enable_and_wait(struct ocs_aes_dev *aes_dev, u8 irq)
409*fbf31dd5STom Zanussi {
410*fbf31dd5STom Zanussi int rc;
411*fbf31dd5STom Zanussi
412*fbf31dd5STom Zanussi reinit_completion(&aes_dev->irq_completion);
413*fbf31dd5STom Zanussi aes_irq_enable(aes_dev, irq);
414*fbf31dd5STom Zanussi rc = wait_for_completion_interruptible(&aes_dev->irq_completion);
415*fbf31dd5STom Zanussi if (rc)
416*fbf31dd5STom Zanussi return rc;
417*fbf31dd5STom Zanussi
418*fbf31dd5STom Zanussi return aes_dev->dma_err_mask ? -EIO : 0;
419*fbf31dd5STom Zanussi }
420*fbf31dd5STom Zanussi
421*fbf31dd5STom Zanussi /* Configure DMA to OCS, linked list mode */
dma_to_ocs_aes_ll(struct ocs_aes_dev * aes_dev,dma_addr_t dma_list)422*fbf31dd5STom Zanussi static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
423*fbf31dd5STom Zanussi dma_addr_t dma_list)
424*fbf31dd5STom Zanussi {
425*fbf31dd5STom Zanussi iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET);
426*fbf31dd5STom Zanussi iowrite32(dma_list,
427*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET);
428*fbf31dd5STom Zanussi }
429*fbf31dd5STom Zanussi
430*fbf31dd5STom Zanussi /* Configure DMA from OCS, linked list mode */
dma_from_ocs_aes_ll(struct ocs_aes_dev * aes_dev,dma_addr_t dma_list)431*fbf31dd5STom Zanussi static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev,
432*fbf31dd5STom Zanussi dma_addr_t dma_list)
433*fbf31dd5STom Zanussi {
434*fbf31dd5STom Zanussi iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET);
435*fbf31dd5STom Zanussi iowrite32(dma_list,
436*fbf31dd5STom Zanussi aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET);
437*fbf31dd5STom Zanussi }
438*fbf31dd5STom Zanussi
ocs_aes_irq_handler(int irq,void * dev_id)439*fbf31dd5STom Zanussi irqreturn_t ocs_aes_irq_handler(int irq, void *dev_id)
440*fbf31dd5STom Zanussi {
441*fbf31dd5STom Zanussi struct ocs_aes_dev *aes_dev = dev_id;
442*fbf31dd5STom Zanussi u32 aes_dma_isr;
443*fbf31dd5STom Zanussi
444*fbf31dd5STom Zanussi /* Read DMA ISR status. */
445*fbf31dd5STom Zanussi aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET);
446*fbf31dd5STom Zanussi
447*fbf31dd5STom Zanussi /* Disable and clear interrupts. */
448*fbf31dd5STom Zanussi aes_irq_disable(aes_dev);
449*fbf31dd5STom Zanussi
450*fbf31dd5STom Zanussi /* Save DMA error status. */
451*fbf31dd5STom Zanussi aes_dev->dma_err_mask = aes_dma_isr &
452*fbf31dd5STom Zanussi (AES_DMA_CPD_ERR_INT |
453*fbf31dd5STom Zanussi AES_DMA_OUTBUF_RD_ERR_INT |
454*fbf31dd5STom Zanussi AES_DMA_OUTBUF_WR_ERR_INT |
455*fbf31dd5STom Zanussi AES_DMA_INBUF_RD_ERR_INT |
456*fbf31dd5STom Zanussi AES_DMA_INBUF_WR_ERR_INT |
457*fbf31dd5STom Zanussi AES_DMA_BAD_COMP_INT |
458*fbf31dd5STom Zanussi AES_DMA_SAI_INT);
459*fbf31dd5STom Zanussi
460*fbf31dd5STom Zanussi /* Signal IRQ completion. */
461*fbf31dd5STom Zanussi complete(&aes_dev->irq_completion);
462*fbf31dd5STom Zanussi
463*fbf31dd5STom Zanussi return IRQ_HANDLED;
464*fbf31dd5STom Zanussi }
465*fbf31dd5STom Zanussi
466*fbf31dd5STom Zanussi /**
467*fbf31dd5STom Zanussi * ocs_aes_set_key() - Write key into OCS AES hardware.
468*fbf31dd5STom Zanussi * @aes_dev: The OCS AES device to write the key to.
469*fbf31dd5STom Zanussi * @key_size: The size of the key (in bytes).
470*fbf31dd5STom Zanussi * @key: The key to write.
471*fbf31dd5STom Zanussi * @cipher: The cipher the key is for.
472*fbf31dd5STom Zanussi *
473*fbf31dd5STom Zanussi * For AES @key_size must be either 16 or 32. For SM4 @key_size must be 16.
474*fbf31dd5STom Zanussi *
475*fbf31dd5STom Zanussi * Return: 0 on success, negative error code otherwise.
476*fbf31dd5STom Zanussi */
ocs_aes_set_key(struct ocs_aes_dev * aes_dev,u32 key_size,const u8 * key,enum ocs_cipher cipher)477*fbf31dd5STom Zanussi int ocs_aes_set_key(struct ocs_aes_dev *aes_dev, u32 key_size, const u8 *key,
478*fbf31dd5STom Zanussi enum ocs_cipher cipher)
479*fbf31dd5STom Zanussi {
480*fbf31dd5STom Zanussi const u32 *key_u32;
481*fbf31dd5STom Zanussi u32 val;
482*fbf31dd5STom Zanussi int i;
483*fbf31dd5STom Zanussi
484*fbf31dd5STom Zanussi /* OCS AES supports 128-bit and 256-bit keys only. */
485*fbf31dd5STom Zanussi if (cipher == OCS_AES && !(key_size == 32 || key_size == 16)) {
486*fbf31dd5STom Zanussi dev_err(aes_dev->dev,
487*fbf31dd5STom Zanussi "%d-bit keys not supported by AES cipher\n",
488*fbf31dd5STom Zanussi key_size * 8);
489*fbf31dd5STom Zanussi return -EINVAL;
490*fbf31dd5STom Zanussi }
491*fbf31dd5STom Zanussi /* OCS SM4 supports 128-bit keys only. */
492*fbf31dd5STom Zanussi if (cipher == OCS_SM4 && key_size != 16) {
493*fbf31dd5STom Zanussi dev_err(aes_dev->dev,
494*fbf31dd5STom Zanussi "%d-bit keys not supported for SM4 cipher\n",
495*fbf31dd5STom Zanussi key_size * 8);
496*fbf31dd5STom Zanussi return -EINVAL;
497*fbf31dd5STom Zanussi }
498*fbf31dd5STom Zanussi
499*fbf31dd5STom Zanussi if (!key)
500*fbf31dd5STom Zanussi return -EINVAL;
501*fbf31dd5STom Zanussi
502*fbf31dd5STom Zanussi key_u32 = (const u32 *)key;
503*fbf31dd5STom Zanussi
504*fbf31dd5STom Zanussi /* Write key to AES_KEY[0-7] registers */
505*fbf31dd5STom Zanussi for (i = 0; i < (key_size / sizeof(u32)); i++) {
506*fbf31dd5STom Zanussi iowrite32(key_u32[i],
507*fbf31dd5STom Zanussi aes_dev->base_reg + AES_KEY_0_OFFSET +
508*fbf31dd5STom Zanussi (i * sizeof(u32)));
509*fbf31dd5STom Zanussi }
510*fbf31dd5STom Zanussi /*
511*fbf31dd5STom Zanussi * Write key size
512*fbf31dd5STom Zanussi * bits [31:1] - reserved
513*fbf31dd5STom Zanussi * bit [0] - AES_KEY_SIZE
514*fbf31dd5STom Zanussi * 0 - 128 bit key
515*fbf31dd5STom Zanussi * 1 - 256 bit key
516*fbf31dd5STom Zanussi */
517*fbf31dd5STom Zanussi val = (key_size == 16) ? AES_128_BIT_KEY : AES_256_BIT_KEY;
518*fbf31dd5STom Zanussi iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET);
519*fbf31dd5STom Zanussi
520*fbf31dd5STom Zanussi return 0;
521*fbf31dd5STom Zanussi }
522*fbf31dd5STom Zanussi
523*fbf31dd5STom Zanussi /* Write AES_COMMAND */
set_ocs_aes_command(struct ocs_aes_dev * aes_dev,enum ocs_cipher cipher,enum ocs_mode mode,enum ocs_instruction instruction)524*fbf31dd5STom Zanussi static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev,
525*fbf31dd5STom Zanussi enum ocs_cipher cipher,
526*fbf31dd5STom Zanussi enum ocs_mode mode,
527*fbf31dd5STom Zanussi enum ocs_instruction instruction)
528*fbf31dd5STom Zanussi {
529*fbf31dd5STom Zanussi u32 val;
530*fbf31dd5STom Zanussi
531*fbf31dd5STom Zanussi /* AES_COMMAND
532*fbf31dd5STom Zanussi * default 0x000000CC
533*fbf31dd5STom Zanussi * bit [14] - CIPHER_SELECT
534*fbf31dd5STom Zanussi * 0 - AES
535*fbf31dd5STom Zanussi * 1 - SM4
536*fbf31dd5STom Zanussi * bits [11:8] - OCS_AES_MODE
537*fbf31dd5STom Zanussi * 0000 - ECB
538*fbf31dd5STom Zanussi * 0001 - CBC
539*fbf31dd5STom Zanussi * 0010 - CTR
540*fbf31dd5STom Zanussi * 0110 - CCM
541*fbf31dd5STom Zanussi * 0111 - GCM
542*fbf31dd5STom Zanussi * 1001 - CTS
543*fbf31dd5STom Zanussi * bits [7:6] - AES_INSTRUCTION
544*fbf31dd5STom Zanussi * 00 - ENCRYPT
545*fbf31dd5STom Zanussi * 01 - DECRYPT
546*fbf31dd5STom Zanussi * 10 - EXPAND
547*fbf31dd5STom Zanussi * 11 - BYPASS
548*fbf31dd5STom Zanussi * bits [3:2] - CTR_M_BITS
549*fbf31dd5STom Zanussi * 00 - No increment
550*fbf31dd5STom Zanussi * 01 - Least significant 32 bits are incremented
551*fbf31dd5STom Zanussi * 10 - Least significant 64 bits are incremented
552*fbf31dd5STom Zanussi * 11 - Full 128 bits are incremented
553*fbf31dd5STom Zanussi */
554*fbf31dd5STom Zanussi val = (cipher << 14) | (mode << 8) | (instruction << 6) |
555*fbf31dd5STom Zanussi (AES_CTR_M_128_INC << 2);
556*fbf31dd5STom Zanussi iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET);
557*fbf31dd5STom Zanussi }
558*fbf31dd5STom Zanussi
ocs_aes_init(struct ocs_aes_dev * aes_dev,enum ocs_mode mode,enum ocs_cipher cipher,enum ocs_instruction instruction)559*fbf31dd5STom Zanussi static void ocs_aes_init(struct ocs_aes_dev *aes_dev,
560*fbf31dd5STom Zanussi enum ocs_mode mode,
561*fbf31dd5STom Zanussi enum ocs_cipher cipher,
562*fbf31dd5STom Zanussi enum ocs_instruction instruction)
563*fbf31dd5STom Zanussi {
564*fbf31dd5STom Zanussi /* Ensure interrupts are disabled and pending interrupts cleared. */
565*fbf31dd5STom Zanussi aes_irq_disable(aes_dev);
566*fbf31dd5STom Zanussi
567*fbf31dd5STom Zanussi /* Set endianness recommended by data-sheet. */
568*fbf31dd5STom Zanussi aes_a_set_endianness(aes_dev);
569*fbf31dd5STom Zanussi
570*fbf31dd5STom Zanussi /* Set AES_COMMAND register. */
571*fbf31dd5STom Zanussi set_ocs_aes_command(aes_dev, cipher, mode, instruction);
572*fbf31dd5STom Zanussi }
573*fbf31dd5STom Zanussi
574*fbf31dd5STom Zanussi /*
575*fbf31dd5STom Zanussi * Write the byte length of the last AES/SM4 block of Payload data (without
576*fbf31dd5STom Zanussi * zero padding and without the length of the MAC) in register AES_PLEN.
577*fbf31dd5STom Zanussi */
ocs_aes_write_last_data_blk_len(struct ocs_aes_dev * aes_dev,u32 size)578*fbf31dd5STom Zanussi static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev,
579*fbf31dd5STom Zanussi u32 size)
580*fbf31dd5STom Zanussi {
581*fbf31dd5STom Zanussi u32 val;
582*fbf31dd5STom Zanussi
583*fbf31dd5STom Zanussi if (size == 0) {
584*fbf31dd5STom Zanussi val = 0;
585*fbf31dd5STom Zanussi goto exit;
586*fbf31dd5STom Zanussi }
587*fbf31dd5STom Zanussi
588*fbf31dd5STom Zanussi val = size % AES_BLOCK_SIZE;
589*fbf31dd5STom Zanussi if (val == 0)
590*fbf31dd5STom Zanussi val = AES_BLOCK_SIZE;
591*fbf31dd5STom Zanussi
592*fbf31dd5STom Zanussi exit:
593*fbf31dd5STom Zanussi iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET);
594*fbf31dd5STom Zanussi }
595*fbf31dd5STom Zanussi
596*fbf31dd5STom Zanussi /*
597*fbf31dd5STom Zanussi * Validate inputs according to mode.
598*fbf31dd5STom Zanussi * If OK return 0; else return -EINVAL.
599*fbf31dd5STom Zanussi */
ocs_aes_validate_inputs(dma_addr_t src_dma_list,u32 src_size,const u8 * iv,u32 iv_size,dma_addr_t aad_dma_list,u32 aad_size,const u8 * tag,u32 tag_size,enum ocs_cipher cipher,enum ocs_mode mode,enum ocs_instruction instruction,dma_addr_t dst_dma_list)600*fbf31dd5STom Zanussi static int ocs_aes_validate_inputs(dma_addr_t src_dma_list, u32 src_size,
601*fbf31dd5STom Zanussi const u8 *iv, u32 iv_size,
602*fbf31dd5STom Zanussi dma_addr_t aad_dma_list, u32 aad_size,
603*fbf31dd5STom Zanussi const u8 *tag, u32 tag_size,
604*fbf31dd5STom Zanussi enum ocs_cipher cipher, enum ocs_mode mode,
605*fbf31dd5STom Zanussi enum ocs_instruction instruction,
606*fbf31dd5STom Zanussi dma_addr_t dst_dma_list)
607*fbf31dd5STom Zanussi {
608*fbf31dd5STom Zanussi /* Ensure cipher, mode and instruction are valid. */
609*fbf31dd5STom Zanussi if (!(cipher == OCS_AES || cipher == OCS_SM4))
610*fbf31dd5STom Zanussi return -EINVAL;
611*fbf31dd5STom Zanussi
612*fbf31dd5STom Zanussi if (mode != OCS_MODE_ECB && mode != OCS_MODE_CBC &&
613*fbf31dd5STom Zanussi mode != OCS_MODE_CTR && mode != OCS_MODE_CCM &&
614*fbf31dd5STom Zanussi mode != OCS_MODE_GCM && mode != OCS_MODE_CTS)
615*fbf31dd5STom Zanussi return -EINVAL;
616*fbf31dd5STom Zanussi
617*fbf31dd5STom Zanussi if (instruction != OCS_ENCRYPT && instruction != OCS_DECRYPT &&
618*fbf31dd5STom Zanussi instruction != OCS_EXPAND && instruction != OCS_BYPASS)
619*fbf31dd5STom Zanussi return -EINVAL;
620*fbf31dd5STom Zanussi
621*fbf31dd5STom Zanussi /*
622*fbf31dd5STom Zanussi * When instruction is OCS_BYPASS, OCS simply copies data from source
623*fbf31dd5STom Zanussi * to destination using DMA.
624*fbf31dd5STom Zanussi *
625*fbf31dd5STom Zanussi * AES mode is irrelevant, but both source and destination DMA
626*fbf31dd5STom Zanussi * linked-list must be defined.
627*fbf31dd5STom Zanussi */
628*fbf31dd5STom Zanussi if (instruction == OCS_BYPASS) {
629*fbf31dd5STom Zanussi if (src_dma_list == DMA_MAPPING_ERROR ||
630*fbf31dd5STom Zanussi dst_dma_list == DMA_MAPPING_ERROR)
631*fbf31dd5STom Zanussi return -EINVAL;
632*fbf31dd5STom Zanussi
633*fbf31dd5STom Zanussi return 0;
634*fbf31dd5STom Zanussi }
635*fbf31dd5STom Zanussi
636*fbf31dd5STom Zanussi /*
637*fbf31dd5STom Zanussi * For performance reasons switch based on mode to limit unnecessary
638*fbf31dd5STom Zanussi * conditionals for each mode
639*fbf31dd5STom Zanussi */
640*fbf31dd5STom Zanussi switch (mode) {
641*fbf31dd5STom Zanussi case OCS_MODE_ECB:
642*fbf31dd5STom Zanussi /* Ensure input length is multiple of block size */
643*fbf31dd5STom Zanussi if (src_size % AES_BLOCK_SIZE != 0)
644*fbf31dd5STom Zanussi return -EINVAL;
645*fbf31dd5STom Zanussi
646*fbf31dd5STom Zanussi /* Ensure source and destination linked lists are created */
647*fbf31dd5STom Zanussi if (src_dma_list == DMA_MAPPING_ERROR ||
648*fbf31dd5STom Zanussi dst_dma_list == DMA_MAPPING_ERROR)
649*fbf31dd5STom Zanussi return -EINVAL;
650*fbf31dd5STom Zanussi
651*fbf31dd5STom Zanussi return 0;
652*fbf31dd5STom Zanussi
653*fbf31dd5STom Zanussi case OCS_MODE_CBC:
654*fbf31dd5STom Zanussi /* Ensure input length is multiple of block size */
655*fbf31dd5STom Zanussi if (src_size % AES_BLOCK_SIZE != 0)
656*fbf31dd5STom Zanussi return -EINVAL;
657*fbf31dd5STom Zanussi
658*fbf31dd5STom Zanussi /* Ensure source and destination linked lists are created */
659*fbf31dd5STom Zanussi if (src_dma_list == DMA_MAPPING_ERROR ||
660*fbf31dd5STom Zanussi dst_dma_list == DMA_MAPPING_ERROR)
661*fbf31dd5STom Zanussi return -EINVAL;
662*fbf31dd5STom Zanussi
663*fbf31dd5STom Zanussi /* Ensure IV is present and block size in length */
664*fbf31dd5STom Zanussi if (!iv || iv_size != AES_BLOCK_SIZE)
665*fbf31dd5STom Zanussi return -EINVAL;
666*fbf31dd5STom Zanussi
667*fbf31dd5STom Zanussi return 0;
668*fbf31dd5STom Zanussi
669*fbf31dd5STom Zanussi case OCS_MODE_CTR:
670*fbf31dd5STom Zanussi /* Ensure input length of 1 byte or greater */
671*fbf31dd5STom Zanussi if (src_size == 0)
672*fbf31dd5STom Zanussi return -EINVAL;
673*fbf31dd5STom Zanussi
674*fbf31dd5STom Zanussi /* Ensure source and destination linked lists are created */
675*fbf31dd5STom Zanussi if (src_dma_list == DMA_MAPPING_ERROR ||
676*fbf31dd5STom Zanussi dst_dma_list == DMA_MAPPING_ERROR)
677*fbf31dd5STom Zanussi return -EINVAL;
678*fbf31dd5STom Zanussi
679*fbf31dd5STom Zanussi /* Ensure IV is present and block size in length */
680*fbf31dd5STom Zanussi if (!iv || iv_size != AES_BLOCK_SIZE)
681*fbf31dd5STom Zanussi return -EINVAL;
682*fbf31dd5STom Zanussi
683*fbf31dd5STom Zanussi return 0;
684*fbf31dd5STom Zanussi
685*fbf31dd5STom Zanussi case OCS_MODE_CTS:
686*fbf31dd5STom Zanussi /* Ensure input length >= block size */
687*fbf31dd5STom Zanussi if (src_size < AES_BLOCK_SIZE)
688*fbf31dd5STom Zanussi return -EINVAL;
689*fbf31dd5STom Zanussi
690*fbf31dd5STom Zanussi /* Ensure source and destination linked lists are created */
691*fbf31dd5STom Zanussi if (src_dma_list == DMA_MAPPING_ERROR ||
692*fbf31dd5STom Zanussi dst_dma_list == DMA_MAPPING_ERROR)
693*fbf31dd5STom Zanussi return -EINVAL;
694*fbf31dd5STom Zanussi
695*fbf31dd5STom Zanussi /* Ensure IV is present and block size in length */
696*fbf31dd5STom Zanussi if (!iv || iv_size != AES_BLOCK_SIZE)
697*fbf31dd5STom Zanussi return -EINVAL;
698*fbf31dd5STom Zanussi
699*fbf31dd5STom Zanussi return 0;
700*fbf31dd5STom Zanussi
701*fbf31dd5STom Zanussi case OCS_MODE_GCM:
702*fbf31dd5STom Zanussi /* Ensure IV is present and GCM_AES_IV_SIZE in length */
703*fbf31dd5STom Zanussi if (!iv || iv_size != GCM_AES_IV_SIZE)
704*fbf31dd5STom Zanussi return -EINVAL;
705*fbf31dd5STom Zanussi
706*fbf31dd5STom Zanussi /*
707*fbf31dd5STom Zanussi * If input data present ensure source and destination linked
708*fbf31dd5STom Zanussi * lists are created
709*fbf31dd5STom Zanussi */
710*fbf31dd5STom Zanussi if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
711*fbf31dd5STom Zanussi dst_dma_list == DMA_MAPPING_ERROR))
712*fbf31dd5STom Zanussi return -EINVAL;
713*fbf31dd5STom Zanussi
714*fbf31dd5STom Zanussi /* If aad present ensure aad linked list is created */
715*fbf31dd5STom Zanussi if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
716*fbf31dd5STom Zanussi return -EINVAL;
717*fbf31dd5STom Zanussi
718*fbf31dd5STom Zanussi /* Ensure tag destination is set */
719*fbf31dd5STom Zanussi if (!tag)
720*fbf31dd5STom Zanussi return -EINVAL;
721*fbf31dd5STom Zanussi
722*fbf31dd5STom Zanussi /* Just ensure that tag_size doesn't cause overflows. */
723*fbf31dd5STom Zanussi if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
724*fbf31dd5STom Zanussi return -EINVAL;
725*fbf31dd5STom Zanussi
726*fbf31dd5STom Zanussi return 0;
727*fbf31dd5STom Zanussi
728*fbf31dd5STom Zanussi case OCS_MODE_CCM:
729*fbf31dd5STom Zanussi /* Ensure IV is present and block size in length */
730*fbf31dd5STom Zanussi if (!iv || iv_size != AES_BLOCK_SIZE)
731*fbf31dd5STom Zanussi return -EINVAL;
732*fbf31dd5STom Zanussi
733*fbf31dd5STom Zanussi /* 2 <= L <= 8, so 1 <= L' <= 7 */
734*fbf31dd5STom Zanussi if (iv[L_PRIME_IDX] < L_PRIME_MIN ||
735*fbf31dd5STom Zanussi iv[L_PRIME_IDX] > L_PRIME_MAX)
736*fbf31dd5STom Zanussi return -EINVAL;
737*fbf31dd5STom Zanussi
738*fbf31dd5STom Zanussi /* If aad present ensure aad linked list is created */
739*fbf31dd5STom Zanussi if (aad_size && aad_dma_list == DMA_MAPPING_ERROR)
740*fbf31dd5STom Zanussi return -EINVAL;
741*fbf31dd5STom Zanussi
742*fbf31dd5STom Zanussi /* Just ensure that tag_size doesn't cause overflows. */
743*fbf31dd5STom Zanussi if (tag_size > (AES_MAX_TAG_SIZE_U32 * sizeof(u32)))
744*fbf31dd5STom Zanussi return -EINVAL;
745*fbf31dd5STom Zanussi
746*fbf31dd5STom Zanussi if (instruction == OCS_DECRYPT) {
747*fbf31dd5STom Zanussi /*
748*fbf31dd5STom Zanussi * If input data present ensure source and destination
749*fbf31dd5STom Zanussi * linked lists are created
750*fbf31dd5STom Zanussi */
751*fbf31dd5STom Zanussi if (src_size && (src_dma_list == DMA_MAPPING_ERROR ||
752*fbf31dd5STom Zanussi dst_dma_list == DMA_MAPPING_ERROR))
753*fbf31dd5STom Zanussi return -EINVAL;
754*fbf31dd5STom Zanussi
755*fbf31dd5STom Zanussi /* Ensure input tag is present */
756*fbf31dd5STom Zanussi if (!tag)
757*fbf31dd5STom Zanussi return -EINVAL;
758*fbf31dd5STom Zanussi
759*fbf31dd5STom Zanussi return 0;
760*fbf31dd5STom Zanussi }
761*fbf31dd5STom Zanussi
762*fbf31dd5STom Zanussi /* Instruction == OCS_ENCRYPT */
763*fbf31dd5STom Zanussi
764*fbf31dd5STom Zanussi /*
765*fbf31dd5STom Zanussi * Destination linked list always required (for tag even if no
766*fbf31dd5STom Zanussi * input data)
767*fbf31dd5STom Zanussi */
768*fbf31dd5STom Zanussi if (dst_dma_list == DMA_MAPPING_ERROR)
769*fbf31dd5STom Zanussi return -EINVAL;
770*fbf31dd5STom Zanussi
771*fbf31dd5STom Zanussi /* If input data present ensure src linked list is created */
772*fbf31dd5STom Zanussi if (src_size && src_dma_list == DMA_MAPPING_ERROR)
773*fbf31dd5STom Zanussi return -EINVAL;
774*fbf31dd5STom Zanussi
775*fbf31dd5STom Zanussi return 0;
776*fbf31dd5STom Zanussi
777*fbf31dd5STom Zanussi default:
778*fbf31dd5STom Zanussi return -EINVAL;
779*fbf31dd5STom Zanussi }
780*fbf31dd5STom Zanussi }
781*fbf31dd5STom Zanussi
782*fbf31dd5STom Zanussi /**
783*fbf31dd5STom Zanussi * ocs_aes_op() - Perform AES/SM4 operation.
784*fbf31dd5STom Zanussi * @aes_dev: The OCS AES device to use.
785*fbf31dd5STom Zanussi * @mode: The mode to use (ECB, CBC, CTR, or CTS).
786*fbf31dd5STom Zanussi * @cipher: The cipher to use (AES or SM4).
787*fbf31dd5STom Zanussi * @instruction: The instruction to perform (encrypt or decrypt).
788*fbf31dd5STom Zanussi * @dst_dma_list: The OCS DMA list mapping output memory.
789*fbf31dd5STom Zanussi * @src_dma_list: The OCS DMA list mapping input payload data.
790*fbf31dd5STom Zanussi * @src_size: The amount of data mapped by @src_dma_list.
791*fbf31dd5STom Zanussi * @iv: The IV vector.
792*fbf31dd5STom Zanussi * @iv_size: The size (in bytes) of @iv.
793*fbf31dd5STom Zanussi *
794*fbf31dd5STom Zanussi * Return: 0 on success, negative error code otherwise.
795*fbf31dd5STom Zanussi */
ocs_aes_op(struct ocs_aes_dev * aes_dev,enum ocs_mode mode,enum ocs_cipher cipher,enum ocs_instruction instruction,dma_addr_t dst_dma_list,dma_addr_t src_dma_list,u32 src_size,u8 * iv,u32 iv_size)796*fbf31dd5STom Zanussi int ocs_aes_op(struct ocs_aes_dev *aes_dev,
797*fbf31dd5STom Zanussi enum ocs_mode mode,
798*fbf31dd5STom Zanussi enum ocs_cipher cipher,
799*fbf31dd5STom Zanussi enum ocs_instruction instruction,
800*fbf31dd5STom Zanussi dma_addr_t dst_dma_list,
801*fbf31dd5STom Zanussi dma_addr_t src_dma_list,
802*fbf31dd5STom Zanussi u32 src_size,
803*fbf31dd5STom Zanussi u8 *iv,
804*fbf31dd5STom Zanussi u32 iv_size)
805*fbf31dd5STom Zanussi {
806*fbf31dd5STom Zanussi u32 *iv32;
807*fbf31dd5STom Zanussi int rc;
808*fbf31dd5STom Zanussi
809*fbf31dd5STom Zanussi rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv, iv_size, 0, 0,
810*fbf31dd5STom Zanussi NULL, 0, cipher, mode, instruction,
811*fbf31dd5STom Zanussi dst_dma_list);
812*fbf31dd5STom Zanussi if (rc)
813*fbf31dd5STom Zanussi return rc;
814*fbf31dd5STom Zanussi /*
815*fbf31dd5STom Zanussi * ocs_aes_validate_inputs() is a generic check, now ensure mode is not
816*fbf31dd5STom Zanussi * GCM or CCM.
817*fbf31dd5STom Zanussi */
818*fbf31dd5STom Zanussi if (mode == OCS_MODE_GCM || mode == OCS_MODE_CCM)
819*fbf31dd5STom Zanussi return -EINVAL;
820*fbf31dd5STom Zanussi
821*fbf31dd5STom Zanussi /* Cast IV to u32 array. */
822*fbf31dd5STom Zanussi iv32 = (u32 *)iv;
823*fbf31dd5STom Zanussi
824*fbf31dd5STom Zanussi ocs_aes_init(aes_dev, mode, cipher, instruction);
825*fbf31dd5STom Zanussi
826*fbf31dd5STom Zanussi if (mode == OCS_MODE_CTS) {
827*fbf31dd5STom Zanussi /* Write the byte length of the last data block to engine. */
828*fbf31dd5STom Zanussi ocs_aes_write_last_data_blk_len(aes_dev, src_size);
829*fbf31dd5STom Zanussi }
830*fbf31dd5STom Zanussi
831*fbf31dd5STom Zanussi /* ECB is the only mode that doesn't use IV. */
832*fbf31dd5STom Zanussi if (mode != OCS_MODE_ECB) {
833*fbf31dd5STom Zanussi iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET);
834*fbf31dd5STom Zanussi iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET);
835*fbf31dd5STom Zanussi iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET);
836*fbf31dd5STom Zanussi iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET);
837*fbf31dd5STom Zanussi }
838*fbf31dd5STom Zanussi
839*fbf31dd5STom Zanussi /* Set AES_ACTIVE.TRIGGER to start the operation. */
840*fbf31dd5STom Zanussi aes_a_op_trigger(aes_dev);
841*fbf31dd5STom Zanussi
842*fbf31dd5STom Zanussi /* Configure and activate input / output DMA. */
843*fbf31dd5STom Zanussi dma_to_ocs_aes_ll(aes_dev, src_dma_list);
844*fbf31dd5STom Zanussi dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
845*fbf31dd5STom Zanussi aes_a_dma_active_src_dst_ll_en(aes_dev);
846*fbf31dd5STom Zanussi
847*fbf31dd5STom Zanussi if (mode == OCS_MODE_CTS) {
848*fbf31dd5STom Zanussi /*
849*fbf31dd5STom Zanussi * For CTS mode, instruct engine to activate ciphertext
850*fbf31dd5STom Zanussi * stealing if last block of data is incomplete.
851*fbf31dd5STom Zanussi */
852*fbf31dd5STom Zanussi aes_a_set_last_gcx(aes_dev);
853*fbf31dd5STom Zanussi } else {
854*fbf31dd5STom Zanussi /* For all other modes, just write the 'termination' bit. */
855*fbf31dd5STom Zanussi aes_a_op_termination(aes_dev);
856*fbf31dd5STom Zanussi }
857*fbf31dd5STom Zanussi
858*fbf31dd5STom Zanussi /* Wait for engine to complete processing. */
859*fbf31dd5STom Zanussi rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
860*fbf31dd5STom Zanussi if (rc)
861*fbf31dd5STom Zanussi return rc;
862*fbf31dd5STom Zanussi
863*fbf31dd5STom Zanussi if (mode == OCS_MODE_CTR) {
864*fbf31dd5STom Zanussi /* Read back IV for streaming mode */
865*fbf31dd5STom Zanussi iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET);
866*fbf31dd5STom Zanussi iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET);
867*fbf31dd5STom Zanussi iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET);
868*fbf31dd5STom Zanussi iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET);
869*fbf31dd5STom Zanussi }
870*fbf31dd5STom Zanussi
871*fbf31dd5STom Zanussi return 0;
872*fbf31dd5STom Zanussi }
873*fbf31dd5STom Zanussi
874*fbf31dd5STom Zanussi /* Compute and write J0 to engine registers. */
ocs_aes_gcm_write_j0(const struct ocs_aes_dev * aes_dev,const u8 * iv)875*fbf31dd5STom Zanussi static void ocs_aes_gcm_write_j0(const struct ocs_aes_dev *aes_dev,
876*fbf31dd5STom Zanussi const u8 *iv)
877*fbf31dd5STom Zanussi {
878*fbf31dd5STom Zanussi const u32 *j0 = (u32 *)iv;
879*fbf31dd5STom Zanussi
880*fbf31dd5STom Zanussi /*
881*fbf31dd5STom Zanussi * IV must be 12 bytes; Other sizes not supported as Linux crypto API
882*fbf31dd5STom Zanussi * does only expects/allows 12 byte IV for GCM
883*fbf31dd5STom Zanussi */
884*fbf31dd5STom Zanussi iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET);
885*fbf31dd5STom Zanussi iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET);
886*fbf31dd5STom Zanussi iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET);
887*fbf31dd5STom Zanussi iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET);
888*fbf31dd5STom Zanussi }
889*fbf31dd5STom Zanussi
890*fbf31dd5STom Zanussi /* Read GCM tag from engine registers. */
ocs_aes_gcm_read_tag(struct ocs_aes_dev * aes_dev,u8 * tag,u32 tag_size)891*fbf31dd5STom Zanussi static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev,
892*fbf31dd5STom Zanussi u8 *tag, u32 tag_size)
893*fbf31dd5STom Zanussi {
894*fbf31dd5STom Zanussi u32 tag_u32[AES_MAX_TAG_SIZE_U32];
895*fbf31dd5STom Zanussi
896*fbf31dd5STom Zanussi /*
897*fbf31dd5STom Zanussi * The Authentication Tag T is stored in Little Endian order in the
898*fbf31dd5STom Zanussi * registers with the most significant bytes stored from AES_T_MAC[3]
899*fbf31dd5STom Zanussi * downward.
900*fbf31dd5STom Zanussi */
901*fbf31dd5STom Zanussi tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET));
902*fbf31dd5STom Zanussi tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET));
903*fbf31dd5STom Zanussi tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET));
904*fbf31dd5STom Zanussi tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET));
905*fbf31dd5STom Zanussi
906*fbf31dd5STom Zanussi memcpy(tag, tag_u32, tag_size);
907*fbf31dd5STom Zanussi }
908*fbf31dd5STom Zanussi
909*fbf31dd5STom Zanussi /**
910*fbf31dd5STom Zanussi * ocs_aes_gcm_op() - Perform GCM operation.
911*fbf31dd5STom Zanussi * @aes_dev: The OCS AES device to use.
912*fbf31dd5STom Zanussi * @cipher: The Cipher to use (AES or SM4).
913*fbf31dd5STom Zanussi * @instruction: The instruction to perform (encrypt or decrypt).
914*fbf31dd5STom Zanussi * @dst_dma_list: The OCS DMA list mapping output memory.
915*fbf31dd5STom Zanussi * @src_dma_list: The OCS DMA list mapping input payload data.
916*fbf31dd5STom Zanussi * @src_size: The amount of data mapped by @src_dma_list.
917*fbf31dd5STom Zanussi * @iv: The input IV vector.
918*fbf31dd5STom Zanussi * @aad_dma_list: The OCS DMA list mapping input AAD data.
919*fbf31dd5STom Zanussi * @aad_size: The amount of data mapped by @aad_dma_list.
920*fbf31dd5STom Zanussi * @out_tag: Where to store computed tag.
921*fbf31dd5STom Zanussi * @tag_size: The size (in bytes) of @out_tag.
922*fbf31dd5STom Zanussi *
923*fbf31dd5STom Zanussi * Return: 0 on success, negative error code otherwise.
924*fbf31dd5STom Zanussi */
ocs_aes_gcm_op(struct ocs_aes_dev * aes_dev,enum ocs_cipher cipher,enum ocs_instruction instruction,dma_addr_t dst_dma_list,dma_addr_t src_dma_list,u32 src_size,const u8 * iv,dma_addr_t aad_dma_list,u32 aad_size,u8 * out_tag,u32 tag_size)925*fbf31dd5STom Zanussi int ocs_aes_gcm_op(struct ocs_aes_dev *aes_dev,
926*fbf31dd5STom Zanussi enum ocs_cipher cipher,
927*fbf31dd5STom Zanussi enum ocs_instruction instruction,
928*fbf31dd5STom Zanussi dma_addr_t dst_dma_list,
929*fbf31dd5STom Zanussi dma_addr_t src_dma_list,
930*fbf31dd5STom Zanussi u32 src_size,
931*fbf31dd5STom Zanussi const u8 *iv,
932*fbf31dd5STom Zanussi dma_addr_t aad_dma_list,
933*fbf31dd5STom Zanussi u32 aad_size,
934*fbf31dd5STom Zanussi u8 *out_tag,
935*fbf31dd5STom Zanussi u32 tag_size)
936*fbf31dd5STom Zanussi {
937*fbf31dd5STom Zanussi u64 bit_len;
938*fbf31dd5STom Zanussi u32 val;
939*fbf31dd5STom Zanussi int rc;
940*fbf31dd5STom Zanussi
941*fbf31dd5STom Zanussi rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
942*fbf31dd5STom Zanussi GCM_AES_IV_SIZE, aad_dma_list,
943*fbf31dd5STom Zanussi aad_size, out_tag, tag_size, cipher,
944*fbf31dd5STom Zanussi OCS_MODE_GCM, instruction,
945*fbf31dd5STom Zanussi dst_dma_list);
946*fbf31dd5STom Zanussi if (rc)
947*fbf31dd5STom Zanussi return rc;
948*fbf31dd5STom Zanussi
949*fbf31dd5STom Zanussi ocs_aes_init(aes_dev, OCS_MODE_GCM, cipher, instruction);
950*fbf31dd5STom Zanussi
951*fbf31dd5STom Zanussi /* Compute and write J0 to OCS HW. */
952*fbf31dd5STom Zanussi ocs_aes_gcm_write_j0(aes_dev, iv);
953*fbf31dd5STom Zanussi
954*fbf31dd5STom Zanussi /* Write out_tag byte length */
955*fbf31dd5STom Zanussi iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
956*fbf31dd5STom Zanussi
957*fbf31dd5STom Zanussi /* Write the byte length of the last plaintext / ciphertext block. */
958*fbf31dd5STom Zanussi ocs_aes_write_last_data_blk_len(aes_dev, src_size);
959*fbf31dd5STom Zanussi
960*fbf31dd5STom Zanussi /* Write ciphertext bit length */
961*fbf31dd5STom Zanussi bit_len = (u64)src_size * 8;
962*fbf31dd5STom Zanussi val = bit_len & 0xFFFFFFFF;
963*fbf31dd5STom Zanussi iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET);
964*fbf31dd5STom Zanussi val = bit_len >> 32;
965*fbf31dd5STom Zanussi iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET);
966*fbf31dd5STom Zanussi
967*fbf31dd5STom Zanussi /* Write aad bit length */
968*fbf31dd5STom Zanussi bit_len = (u64)aad_size * 8;
969*fbf31dd5STom Zanussi val = bit_len & 0xFFFFFFFF;
970*fbf31dd5STom Zanussi iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET);
971*fbf31dd5STom Zanussi val = bit_len >> 32;
972*fbf31dd5STom Zanussi iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET);
973*fbf31dd5STom Zanussi
974*fbf31dd5STom Zanussi /* Set AES_ACTIVE.TRIGGER to start the operation. */
975*fbf31dd5STom Zanussi aes_a_op_trigger(aes_dev);
976*fbf31dd5STom Zanussi
977*fbf31dd5STom Zanussi /* Process AAD. */
978*fbf31dd5STom Zanussi if (aad_size) {
979*fbf31dd5STom Zanussi /* If aad present, configure DMA to feed it to the engine. */
980*fbf31dd5STom Zanussi dma_to_ocs_aes_ll(aes_dev, aad_dma_list);
981*fbf31dd5STom Zanussi aes_a_dma_active_src_ll_en(aes_dev);
982*fbf31dd5STom Zanussi
983*fbf31dd5STom Zanussi /* Instructs engine to pad last block of aad, if needed. */
984*fbf31dd5STom Zanussi aes_a_set_last_gcx_and_adata(aes_dev);
985*fbf31dd5STom Zanussi
986*fbf31dd5STom Zanussi /* Wait for DMA transfer to complete. */
987*fbf31dd5STom Zanussi rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
988*fbf31dd5STom Zanussi if (rc)
989*fbf31dd5STom Zanussi return rc;
990*fbf31dd5STom Zanussi } else {
991*fbf31dd5STom Zanussi aes_a_set_last_gcx_and_adata(aes_dev);
992*fbf31dd5STom Zanussi }
993*fbf31dd5STom Zanussi
994*fbf31dd5STom Zanussi /* Wait until adata (if present) has been processed. */
995*fbf31dd5STom Zanussi aes_a_wait_last_gcx(aes_dev);
996*fbf31dd5STom Zanussi aes_a_dma_wait_input_buffer_occupancy(aes_dev);
997*fbf31dd5STom Zanussi
998*fbf31dd5STom Zanussi /* Now process payload. */
999*fbf31dd5STom Zanussi if (src_size) {
1000*fbf31dd5STom Zanussi /* Configure and activate DMA for both input and output data. */
1001*fbf31dd5STom Zanussi dma_to_ocs_aes_ll(aes_dev, src_dma_list);
1002*fbf31dd5STom Zanussi dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1003*fbf31dd5STom Zanussi aes_a_dma_active_src_dst_ll_en(aes_dev);
1004*fbf31dd5STom Zanussi } else {
1005*fbf31dd5STom Zanussi aes_a_dma_set_xfer_size_zero(aes_dev);
1006*fbf31dd5STom Zanussi aes_a_dma_active(aes_dev);
1007*fbf31dd5STom Zanussi }
1008*fbf31dd5STom Zanussi
1009*fbf31dd5STom Zanussi /* Instruct AES/SMA4 engine payload processing is over. */
1010*fbf31dd5STom Zanussi aes_a_set_last_gcx(aes_dev);
1011*fbf31dd5STom Zanussi
1012*fbf31dd5STom Zanussi /* Wait for OCS AES engine to complete processing. */
1013*fbf31dd5STom Zanussi rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
1014*fbf31dd5STom Zanussi if (rc)
1015*fbf31dd5STom Zanussi return rc;
1016*fbf31dd5STom Zanussi
1017*fbf31dd5STom Zanussi ocs_aes_gcm_read_tag(aes_dev, out_tag, tag_size);
1018*fbf31dd5STom Zanussi
1019*fbf31dd5STom Zanussi return 0;
1020*fbf31dd5STom Zanussi }
1021*fbf31dd5STom Zanussi
1022*fbf31dd5STom Zanussi /* Write encrypted tag to AES/SM4 engine. */
ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev * aes_dev,const u8 * in_tag,u32 tag_size)1023*fbf31dd5STom Zanussi static void ocs_aes_ccm_write_encrypted_tag(struct ocs_aes_dev *aes_dev,
1024*fbf31dd5STom Zanussi const u8 *in_tag, u32 tag_size)
1025*fbf31dd5STom Zanussi {
1026*fbf31dd5STom Zanussi int i;
1027*fbf31dd5STom Zanussi
1028*fbf31dd5STom Zanussi /* Ensure DMA input buffer is empty */
1029*fbf31dd5STom Zanussi aes_a_dma_wait_input_buffer_occupancy(aes_dev);
1030*fbf31dd5STom Zanussi
1031*fbf31dd5STom Zanussi /*
1032*fbf31dd5STom Zanussi * During CCM decrypt, the OCS block needs to finish processing the
1033*fbf31dd5STom Zanussi * ciphertext before the tag is written. So delay needed after DMA has
1034*fbf31dd5STom Zanussi * completed writing the ciphertext
1035*fbf31dd5STom Zanussi */
1036*fbf31dd5STom Zanussi aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
1037*fbf31dd5STom Zanussi aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
1038*fbf31dd5STom Zanussi CCM_DECRYPT_DELAY_TAG_CLK_COUNT);
1039*fbf31dd5STom Zanussi
1040*fbf31dd5STom Zanussi /* Write encrypted tag to AES/SM4 engine. */
1041*fbf31dd5STom Zanussi for (i = 0; i < tag_size; i++) {
1042*fbf31dd5STom Zanussi iowrite8(in_tag[i], aes_dev->base_reg +
1043*fbf31dd5STom Zanussi AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
1044*fbf31dd5STom Zanussi }
1045*fbf31dd5STom Zanussi }
1046*fbf31dd5STom Zanussi
1047*fbf31dd5STom Zanussi /*
1048*fbf31dd5STom Zanussi * Write B0 CCM block to OCS AES HW.
1049*fbf31dd5STom Zanussi *
1050*fbf31dd5STom Zanussi * Note: B0 format is documented in NIST Special Publication 800-38C
1051*fbf31dd5STom Zanussi * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1052*fbf31dd5STom Zanussi * (see Section A.2.1)
1053*fbf31dd5STom Zanussi */
ocs_aes_ccm_write_b0(const struct ocs_aes_dev * aes_dev,const u8 * iv,u32 adata_size,u32 tag_size,u32 cryptlen)1054*fbf31dd5STom Zanussi static int ocs_aes_ccm_write_b0(const struct ocs_aes_dev *aes_dev,
1055*fbf31dd5STom Zanussi const u8 *iv, u32 adata_size, u32 tag_size,
1056*fbf31dd5STom Zanussi u32 cryptlen)
1057*fbf31dd5STom Zanussi {
1058*fbf31dd5STom Zanussi u8 b0[16]; /* CCM B0 block is 16 bytes long. */
1059*fbf31dd5STom Zanussi int i, q;
1060*fbf31dd5STom Zanussi
1061*fbf31dd5STom Zanussi /* Initialize B0 to 0. */
1062*fbf31dd5STom Zanussi memset(b0, 0, sizeof(b0));
1063*fbf31dd5STom Zanussi
1064*fbf31dd5STom Zanussi /*
1065*fbf31dd5STom Zanussi * B0[0] is the 'Flags Octet' and has the following structure:
1066*fbf31dd5STom Zanussi * bit 7: Reserved
1067*fbf31dd5STom Zanussi * bit 6: Adata flag
1068*fbf31dd5STom Zanussi * bit 5-3: t value encoded as (t-2)/2
1069*fbf31dd5STom Zanussi * bit 2-0: q value encoded as q - 1
1070*fbf31dd5STom Zanussi */
1071*fbf31dd5STom Zanussi /* If there is AAD data, set the Adata flag. */
1072*fbf31dd5STom Zanussi if (adata_size)
1073*fbf31dd5STom Zanussi b0[0] |= BIT(6);
1074*fbf31dd5STom Zanussi /*
1075*fbf31dd5STom Zanussi * t denotes the octet length of T.
1076*fbf31dd5STom Zanussi * t can only be an element of { 4, 6, 8, 10, 12, 14, 16} and is
1077*fbf31dd5STom Zanussi * encoded as (t - 2) / 2
1078*fbf31dd5STom Zanussi */
1079*fbf31dd5STom Zanussi b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3;
1080*fbf31dd5STom Zanussi /*
1081*fbf31dd5STom Zanussi * q is the octet length of Q.
1082*fbf31dd5STom Zanussi * q can only be an element of {2, 3, 4, 5, 6, 7, 8} and is encoded as
1083*fbf31dd5STom Zanussi * q - 1 == iv[0] & 0x7;
1084*fbf31dd5STom Zanussi */
1085*fbf31dd5STom Zanussi b0[0] |= iv[0] & 0x7;
1086*fbf31dd5STom Zanussi /*
1087*fbf31dd5STom Zanussi * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q]
1088*fbf31dd5STom Zanussi * and must be copied to b0[1]..b0[15-q].
1089*fbf31dd5STom Zanussi * q == (iv[0] & 0x7) + 1
1090*fbf31dd5STom Zanussi */
1091*fbf31dd5STom Zanussi q = (iv[0] & 0x7) + 1;
1092*fbf31dd5STom Zanussi for (i = 1; i <= 15 - q; i++)
1093*fbf31dd5STom Zanussi b0[i] = iv[i];
1094*fbf31dd5STom Zanussi /*
1095*fbf31dd5STom Zanussi * The rest of B0 must contain Q, i.e., the message length.
1096*fbf31dd5STom Zanussi * Q is encoded in q octets, in big-endian order, so to write it, we
1097*fbf31dd5STom Zanussi * start from the end of B0 and we move backward.
1098*fbf31dd5STom Zanussi */
1099*fbf31dd5STom Zanussi i = sizeof(b0) - 1;
1100*fbf31dd5STom Zanussi while (q) {
1101*fbf31dd5STom Zanussi b0[i] = cryptlen & 0xff;
1102*fbf31dd5STom Zanussi cryptlen >>= 8;
1103*fbf31dd5STom Zanussi i--;
1104*fbf31dd5STom Zanussi q--;
1105*fbf31dd5STom Zanussi }
1106*fbf31dd5STom Zanussi /*
1107*fbf31dd5STom Zanussi * If cryptlen is not zero at this point, it means that its original
1108*fbf31dd5STom Zanussi * value was too big.
1109*fbf31dd5STom Zanussi */
1110*fbf31dd5STom Zanussi if (cryptlen)
1111*fbf31dd5STom Zanussi return -EOVERFLOW;
1112*fbf31dd5STom Zanussi /* Now write B0 to OCS AES input buffer. */
1113*fbf31dd5STom Zanussi for (i = 0; i < sizeof(b0); i++)
1114*fbf31dd5STom Zanussi iowrite8(b0[i], aes_dev->base_reg +
1115*fbf31dd5STom Zanussi AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
1116*fbf31dd5STom Zanussi return 0;
1117*fbf31dd5STom Zanussi }
1118*fbf31dd5STom Zanussi
1119*fbf31dd5STom Zanussi /*
1120*fbf31dd5STom Zanussi * Write adata length to OCS AES HW.
1121*fbf31dd5STom Zanussi *
1122*fbf31dd5STom Zanussi * Note: adata len encoding is documented in NIST Special Publication 800-38C
1123*fbf31dd5STom Zanussi * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1124*fbf31dd5STom Zanussi * (see Section A.2.2)
1125*fbf31dd5STom Zanussi */
ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev * aes_dev,u64 adata_len)1126*fbf31dd5STom Zanussi static void ocs_aes_ccm_write_adata_len(const struct ocs_aes_dev *aes_dev,
1127*fbf31dd5STom Zanussi u64 adata_len)
1128*fbf31dd5STom Zanussi {
1129*fbf31dd5STom Zanussi u8 enc_a[10]; /* Maximum encoded size: 10 octets. */
1130*fbf31dd5STom Zanussi int i, len;
1131*fbf31dd5STom Zanussi
1132*fbf31dd5STom Zanussi /*
1133*fbf31dd5STom Zanussi * adata_len ('a') is encoded as follows:
1134*fbf31dd5STom Zanussi * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets
1135*fbf31dd5STom Zanussi * (big endian).
1136*fbf31dd5STom Zanussi * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32,
1137*fbf31dd5STom Zanussi * i.e., six octets (big endian).
1138*fbf31dd5STom Zanussi * If 2^32 ≤ a < 2^64 ==> 'a' encoded as 0xff || 0xff || [a]64,
1139*fbf31dd5STom Zanussi * i.e., ten octets (big endian).
1140*fbf31dd5STom Zanussi */
1141*fbf31dd5STom Zanussi if (adata_len < 65280) {
1142*fbf31dd5STom Zanussi len = 2;
1143*fbf31dd5STom Zanussi *(__be16 *)enc_a = cpu_to_be16(adata_len);
1144*fbf31dd5STom Zanussi } else if (adata_len <= 0xFFFFFFFF) {
1145*fbf31dd5STom Zanussi len = 6;
1146*fbf31dd5STom Zanussi *(__be16 *)enc_a = cpu_to_be16(0xfffe);
1147*fbf31dd5STom Zanussi *(__be32 *)&enc_a[2] = cpu_to_be32(adata_len);
1148*fbf31dd5STom Zanussi } else { /* adata_len >= 2^32 */
1149*fbf31dd5STom Zanussi len = 10;
1150*fbf31dd5STom Zanussi *(__be16 *)enc_a = cpu_to_be16(0xffff);
1151*fbf31dd5STom Zanussi *(__be64 *)&enc_a[2] = cpu_to_be64(adata_len);
1152*fbf31dd5STom Zanussi }
1153*fbf31dd5STom Zanussi for (i = 0; i < len; i++)
1154*fbf31dd5STom Zanussi iowrite8(enc_a[i],
1155*fbf31dd5STom Zanussi aes_dev->base_reg +
1156*fbf31dd5STom Zanussi AES_A_DMA_INBUFFER_WRITE_FIFO_OFFSET);
1157*fbf31dd5STom Zanussi }
1158*fbf31dd5STom Zanussi
ocs_aes_ccm_do_adata(struct ocs_aes_dev * aes_dev,dma_addr_t adata_dma_list,u32 adata_size)1159*fbf31dd5STom Zanussi static int ocs_aes_ccm_do_adata(struct ocs_aes_dev *aes_dev,
1160*fbf31dd5STom Zanussi dma_addr_t adata_dma_list, u32 adata_size)
1161*fbf31dd5STom Zanussi {
1162*fbf31dd5STom Zanussi int rc;
1163*fbf31dd5STom Zanussi
1164*fbf31dd5STom Zanussi if (!adata_size) {
1165*fbf31dd5STom Zanussi /* Since no aad the LAST_GCX bit can be set now */
1166*fbf31dd5STom Zanussi aes_a_set_last_gcx_and_adata(aes_dev);
1167*fbf31dd5STom Zanussi goto exit;
1168*fbf31dd5STom Zanussi }
1169*fbf31dd5STom Zanussi
1170*fbf31dd5STom Zanussi /* Adata case. */
1171*fbf31dd5STom Zanussi
1172*fbf31dd5STom Zanussi /*
1173*fbf31dd5STom Zanussi * Form the encoding of the Associated data length and write it
1174*fbf31dd5STom Zanussi * to the AES/SM4 input buffer.
1175*fbf31dd5STom Zanussi */
1176*fbf31dd5STom Zanussi ocs_aes_ccm_write_adata_len(aes_dev, adata_size);
1177*fbf31dd5STom Zanussi
1178*fbf31dd5STom Zanussi /* Configure the AES/SM4 DMA to fetch the Associated Data */
1179*fbf31dd5STom Zanussi dma_to_ocs_aes_ll(aes_dev, adata_dma_list);
1180*fbf31dd5STom Zanussi
1181*fbf31dd5STom Zanussi /* Activate DMA to fetch Associated data. */
1182*fbf31dd5STom Zanussi aes_a_dma_active_src_ll_en(aes_dev);
1183*fbf31dd5STom Zanussi
1184*fbf31dd5STom Zanussi /* Set LAST_GCX and LAST_ADATA in AES ACTIVE register. */
1185*fbf31dd5STom Zanussi aes_a_set_last_gcx_and_adata(aes_dev);
1186*fbf31dd5STom Zanussi
1187*fbf31dd5STom Zanussi /* Wait for DMA transfer to complete. */
1188*fbf31dd5STom Zanussi rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
1189*fbf31dd5STom Zanussi if (rc)
1190*fbf31dd5STom Zanussi return rc;
1191*fbf31dd5STom Zanussi
1192*fbf31dd5STom Zanussi exit:
1193*fbf31dd5STom Zanussi /* Wait until adata (if present) has been processed. */
1194*fbf31dd5STom Zanussi aes_a_wait_last_gcx(aes_dev);
1195*fbf31dd5STom Zanussi aes_a_dma_wait_input_buffer_occupancy(aes_dev);
1196*fbf31dd5STom Zanussi
1197*fbf31dd5STom Zanussi return 0;
1198*fbf31dd5STom Zanussi }
1199*fbf31dd5STom Zanussi
ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev * aes_dev,dma_addr_t dst_dma_list,dma_addr_t src_dma_list,u32 src_size)1200*fbf31dd5STom Zanussi static int ocs_aes_ccm_encrypt_do_payload(struct ocs_aes_dev *aes_dev,
1201*fbf31dd5STom Zanussi dma_addr_t dst_dma_list,
1202*fbf31dd5STom Zanussi dma_addr_t src_dma_list,
1203*fbf31dd5STom Zanussi u32 src_size)
1204*fbf31dd5STom Zanussi {
1205*fbf31dd5STom Zanussi if (src_size) {
1206*fbf31dd5STom Zanussi /*
1207*fbf31dd5STom Zanussi * Configure and activate DMA for both input and output
1208*fbf31dd5STom Zanussi * data.
1209*fbf31dd5STom Zanussi */
1210*fbf31dd5STom Zanussi dma_to_ocs_aes_ll(aes_dev, src_dma_list);
1211*fbf31dd5STom Zanussi dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1212*fbf31dd5STom Zanussi aes_a_dma_active_src_dst_ll_en(aes_dev);
1213*fbf31dd5STom Zanussi } else {
1214*fbf31dd5STom Zanussi /* Configure and activate DMA for output data only. */
1215*fbf31dd5STom Zanussi dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1216*fbf31dd5STom Zanussi aes_a_dma_active_dst_ll_en(aes_dev);
1217*fbf31dd5STom Zanussi }
1218*fbf31dd5STom Zanussi
1219*fbf31dd5STom Zanussi /*
1220*fbf31dd5STom Zanussi * Set the LAST GCX bit in AES_ACTIVE Register to instruct
1221*fbf31dd5STom Zanussi * AES/SM4 engine to pad the last block of data.
1222*fbf31dd5STom Zanussi */
1223*fbf31dd5STom Zanussi aes_a_set_last_gcx(aes_dev);
1224*fbf31dd5STom Zanussi
1225*fbf31dd5STom Zanussi /* We are done, wait for IRQ and return. */
1226*fbf31dd5STom Zanussi return ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
1227*fbf31dd5STom Zanussi }
1228*fbf31dd5STom Zanussi
ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev * aes_dev,dma_addr_t dst_dma_list,dma_addr_t src_dma_list,u32 src_size)1229*fbf31dd5STom Zanussi static int ocs_aes_ccm_decrypt_do_payload(struct ocs_aes_dev *aes_dev,
1230*fbf31dd5STom Zanussi dma_addr_t dst_dma_list,
1231*fbf31dd5STom Zanussi dma_addr_t src_dma_list,
1232*fbf31dd5STom Zanussi u32 src_size)
1233*fbf31dd5STom Zanussi {
1234*fbf31dd5STom Zanussi if (!src_size) {
1235*fbf31dd5STom Zanussi /* Let engine process 0-length input. */
1236*fbf31dd5STom Zanussi aes_a_dma_set_xfer_size_zero(aes_dev);
1237*fbf31dd5STom Zanussi aes_a_dma_active(aes_dev);
1238*fbf31dd5STom Zanussi aes_a_set_last_gcx(aes_dev);
1239*fbf31dd5STom Zanussi
1240*fbf31dd5STom Zanussi return 0;
1241*fbf31dd5STom Zanussi }
1242*fbf31dd5STom Zanussi
1243*fbf31dd5STom Zanussi /*
1244*fbf31dd5STom Zanussi * Configure and activate DMA for both input and output
1245*fbf31dd5STom Zanussi * data.
1246*fbf31dd5STom Zanussi */
1247*fbf31dd5STom Zanussi dma_to_ocs_aes_ll(aes_dev, src_dma_list);
1248*fbf31dd5STom Zanussi dma_from_ocs_aes_ll(aes_dev, dst_dma_list);
1249*fbf31dd5STom Zanussi aes_a_dma_active_src_dst_ll_en(aes_dev);
1250*fbf31dd5STom Zanussi /*
1251*fbf31dd5STom Zanussi * Set the LAST GCX bit in AES_ACTIVE Register; this allows the
1252*fbf31dd5STom Zanussi * AES/SM4 engine to differentiate between encrypted data and
1253*fbf31dd5STom Zanussi * encrypted MAC.
1254*fbf31dd5STom Zanussi */
1255*fbf31dd5STom Zanussi aes_a_set_last_gcx(aes_dev);
1256*fbf31dd5STom Zanussi /*
1257*fbf31dd5STom Zanussi * Enable DMA DONE interrupt; once DMA transfer is over,
1258*fbf31dd5STom Zanussi * interrupt handler will process the MAC/tag.
1259*fbf31dd5STom Zanussi */
1260*fbf31dd5STom Zanussi return ocs_aes_irq_enable_and_wait(aes_dev, AES_DMA_SRC_DONE_INT);
1261*fbf31dd5STom Zanussi }
1262*fbf31dd5STom Zanussi
1263*fbf31dd5STom Zanussi /*
1264*fbf31dd5STom Zanussi * Compare Tag to Yr.
1265*fbf31dd5STom Zanussi *
1266*fbf31dd5STom Zanussi * Only used at the end of CCM decrypt. If tag == yr, message authentication
1267*fbf31dd5STom Zanussi * has succeeded.
1268*fbf31dd5STom Zanussi */
ccm_compare_tag_to_yr(struct ocs_aes_dev * aes_dev,u8 tag_size_bytes)1269*fbf31dd5STom Zanussi static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev,
1270*fbf31dd5STom Zanussi u8 tag_size_bytes)
1271*fbf31dd5STom Zanussi {
1272*fbf31dd5STom Zanussi u32 tag[AES_MAX_TAG_SIZE_U32];
1273*fbf31dd5STom Zanussi u32 yr[AES_MAX_TAG_SIZE_U32];
1274*fbf31dd5STom Zanussi u8 i;
1275*fbf31dd5STom Zanussi
1276*fbf31dd5STom Zanussi /* Read Tag and Yr from AES registers. */
1277*fbf31dd5STom Zanussi for (i = 0; i < AES_MAX_TAG_SIZE_U32; i++) {
1278*fbf31dd5STom Zanussi tag[i] = ioread32(aes_dev->base_reg +
1279*fbf31dd5STom Zanussi AES_T_MAC_0_OFFSET + (i * sizeof(u32)));
1280*fbf31dd5STom Zanussi yr[i] = ioread32(aes_dev->base_reg +
1281*fbf31dd5STom Zanussi AES_MULTIPURPOSE2_0_OFFSET +
1282*fbf31dd5STom Zanussi (i * sizeof(u32)));
1283*fbf31dd5STom Zanussi }
1284*fbf31dd5STom Zanussi
1285*fbf31dd5STom Zanussi return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0;
1286*fbf31dd5STom Zanussi }
1287*fbf31dd5STom Zanussi
1288*fbf31dd5STom Zanussi /**
1289*fbf31dd5STom Zanussi * ocs_aes_ccm_op() - Perform CCM operation.
1290*fbf31dd5STom Zanussi * @aes_dev: The OCS AES device to use.
1291*fbf31dd5STom Zanussi * @cipher: The Cipher to use (AES or SM4).
1292*fbf31dd5STom Zanussi * @instruction: The instruction to perform (encrypt or decrypt).
1293*fbf31dd5STom Zanussi * @dst_dma_list: The OCS DMA list mapping output memory.
1294*fbf31dd5STom Zanussi * @src_dma_list: The OCS DMA list mapping input payload data.
1295*fbf31dd5STom Zanussi * @src_size: The amount of data mapped by @src_dma_list.
1296*fbf31dd5STom Zanussi * @iv: The input IV vector.
1297*fbf31dd5STom Zanussi * @adata_dma_list: The OCS DMA list mapping input A-data.
1298*fbf31dd5STom Zanussi * @adata_size: The amount of data mapped by @adata_dma_list.
1299*fbf31dd5STom Zanussi * @in_tag: Input tag.
1300*fbf31dd5STom Zanussi * @tag_size: The size (in bytes) of @in_tag.
1301*fbf31dd5STom Zanussi *
1302*fbf31dd5STom Zanussi * Note: for encrypt the tag is appended to the ciphertext (in the memory
1303*fbf31dd5STom Zanussi * mapped by @dst_dma_list).
1304*fbf31dd5STom Zanussi *
1305*fbf31dd5STom Zanussi * Return: 0 on success, negative error code otherwise.
1306*fbf31dd5STom Zanussi */
ocs_aes_ccm_op(struct ocs_aes_dev * aes_dev,enum ocs_cipher cipher,enum ocs_instruction instruction,dma_addr_t dst_dma_list,dma_addr_t src_dma_list,u32 src_size,u8 * iv,dma_addr_t adata_dma_list,u32 adata_size,u8 * in_tag,u32 tag_size)1307*fbf31dd5STom Zanussi int ocs_aes_ccm_op(struct ocs_aes_dev *aes_dev,
1308*fbf31dd5STom Zanussi enum ocs_cipher cipher,
1309*fbf31dd5STom Zanussi enum ocs_instruction instruction,
1310*fbf31dd5STom Zanussi dma_addr_t dst_dma_list,
1311*fbf31dd5STom Zanussi dma_addr_t src_dma_list,
1312*fbf31dd5STom Zanussi u32 src_size,
1313*fbf31dd5STom Zanussi u8 *iv,
1314*fbf31dd5STom Zanussi dma_addr_t adata_dma_list,
1315*fbf31dd5STom Zanussi u32 adata_size,
1316*fbf31dd5STom Zanussi u8 *in_tag,
1317*fbf31dd5STom Zanussi u32 tag_size)
1318*fbf31dd5STom Zanussi {
1319*fbf31dd5STom Zanussi u32 *iv_32;
1320*fbf31dd5STom Zanussi u8 lprime;
1321*fbf31dd5STom Zanussi int rc;
1322*fbf31dd5STom Zanussi
1323*fbf31dd5STom Zanussi rc = ocs_aes_validate_inputs(src_dma_list, src_size, iv,
1324*fbf31dd5STom Zanussi AES_BLOCK_SIZE, adata_dma_list, adata_size,
1325*fbf31dd5STom Zanussi in_tag, tag_size, cipher, OCS_MODE_CCM,
1326*fbf31dd5STom Zanussi instruction, dst_dma_list);
1327*fbf31dd5STom Zanussi if (rc)
1328*fbf31dd5STom Zanussi return rc;
1329*fbf31dd5STom Zanussi
1330*fbf31dd5STom Zanussi ocs_aes_init(aes_dev, OCS_MODE_CCM, cipher, instruction);
1331*fbf31dd5STom Zanussi
1332*fbf31dd5STom Zanussi /*
1333*fbf31dd5STom Zanussi * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt
1334*fbf31dd5STom Zanussi * auth tag so ensure this is the case
1335*fbf31dd5STom Zanussi */
1336*fbf31dd5STom Zanussi lprime = iv[L_PRIME_IDX];
1337*fbf31dd5STom Zanussi memset(&iv[COUNTER_START(lprime)], 0, COUNTER_LEN(lprime));
1338*fbf31dd5STom Zanussi
1339*fbf31dd5STom Zanussi /*
1340*fbf31dd5STom Zanussi * Nonce is already converted to ctr0 before being passed into this
1341*fbf31dd5STom Zanussi * function as iv.
1342*fbf31dd5STom Zanussi */
1343*fbf31dd5STom Zanussi iv_32 = (u32 *)iv;
1344*fbf31dd5STom Zanussi iowrite32(__swab32(iv_32[0]),
1345*fbf31dd5STom Zanussi aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET);
1346*fbf31dd5STom Zanussi iowrite32(__swab32(iv_32[1]),
1347*fbf31dd5STom Zanussi aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET);
1348*fbf31dd5STom Zanussi iowrite32(__swab32(iv_32[2]),
1349*fbf31dd5STom Zanussi aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET);
1350*fbf31dd5STom Zanussi iowrite32(__swab32(iv_32[3]),
1351*fbf31dd5STom Zanussi aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET);
1352*fbf31dd5STom Zanussi
1353*fbf31dd5STom Zanussi /* Write MAC/tag length in register AES_TLEN */
1354*fbf31dd5STom Zanussi iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET);
1355*fbf31dd5STom Zanussi /*
1356*fbf31dd5STom Zanussi * Write the byte length of the last AES/SM4 block of Payload data
1357*fbf31dd5STom Zanussi * (without zero padding and without the length of the MAC) in register
1358*fbf31dd5STom Zanussi * AES_PLEN.
1359*fbf31dd5STom Zanussi */
1360*fbf31dd5STom Zanussi ocs_aes_write_last_data_blk_len(aes_dev, src_size);
1361*fbf31dd5STom Zanussi
1362*fbf31dd5STom Zanussi /* Set AES_ACTIVE.TRIGGER to start the operation. */
1363*fbf31dd5STom Zanussi aes_a_op_trigger(aes_dev);
1364*fbf31dd5STom Zanussi
1365*fbf31dd5STom Zanussi aes_a_dma_reset_and_activate_perf_cntr(aes_dev);
1366*fbf31dd5STom Zanussi
1367*fbf31dd5STom Zanussi /* Form block B0 and write it to the AES/SM4 input buffer. */
1368*fbf31dd5STom Zanussi rc = ocs_aes_ccm_write_b0(aes_dev, iv, adata_size, tag_size, src_size);
1369*fbf31dd5STom Zanussi if (rc)
1370*fbf31dd5STom Zanussi return rc;
1371*fbf31dd5STom Zanussi /*
1372*fbf31dd5STom Zanussi * Ensure there has been at least CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT
1373*fbf31dd5STom Zanussi * clock cycles since TRIGGER bit was set
1374*fbf31dd5STom Zanussi */
1375*fbf31dd5STom Zanussi aes_a_dma_wait_and_deactivate_perf_cntr(aes_dev,
1376*fbf31dd5STom Zanussi CCM_DECRYPT_DELAY_LAST_GCX_CLK_COUNT);
1377*fbf31dd5STom Zanussi
1378*fbf31dd5STom Zanussi /* Process Adata. */
1379*fbf31dd5STom Zanussi ocs_aes_ccm_do_adata(aes_dev, adata_dma_list, adata_size);
1380*fbf31dd5STom Zanussi
1381*fbf31dd5STom Zanussi /* For Encrypt case we just process the payload and return. */
1382*fbf31dd5STom Zanussi if (instruction == OCS_ENCRYPT) {
1383*fbf31dd5STom Zanussi return ocs_aes_ccm_encrypt_do_payload(aes_dev, dst_dma_list,
1384*fbf31dd5STom Zanussi src_dma_list, src_size);
1385*fbf31dd5STom Zanussi }
1386*fbf31dd5STom Zanussi /* For Decypt we need to process the payload and then the tag. */
1387*fbf31dd5STom Zanussi rc = ocs_aes_ccm_decrypt_do_payload(aes_dev, dst_dma_list,
1388*fbf31dd5STom Zanussi src_dma_list, src_size);
1389*fbf31dd5STom Zanussi if (rc)
1390*fbf31dd5STom Zanussi return rc;
1391*fbf31dd5STom Zanussi
1392*fbf31dd5STom Zanussi /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */
1393*fbf31dd5STom Zanussi ocs_aes_ccm_write_encrypted_tag(aes_dev, in_tag, tag_size);
1394*fbf31dd5STom Zanussi rc = ocs_aes_irq_enable_and_wait(aes_dev, AES_COMPLETE_INT);
1395*fbf31dd5STom Zanussi if (rc)
1396*fbf31dd5STom Zanussi return rc;
1397*fbf31dd5STom Zanussi
1398*fbf31dd5STom Zanussi return ccm_compare_tag_to_yr(aes_dev, tag_size);
1399*fbf31dd5STom Zanussi }
1400*fbf31dd5STom Zanussi
1401*fbf31dd5STom Zanussi /**
1402*fbf31dd5STom Zanussi * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list.
1403*fbf31dd5STom Zanussi * @aes_dev: The OCS AES device the list will be created for.
1404*fbf31dd5STom Zanussi * @sg: The SG list OCS DMA linked list will be created from. When
1405*fbf31dd5STom Zanussi * passed to this function, @sg must have been already mapped
1406*fbf31dd5STom Zanussi * with dma_map_sg().
1407*fbf31dd5STom Zanussi * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the
1408*fbf31dd5STom Zanussi * value returned by dma_map_sg() when @sg was mapped.
1409*fbf31dd5STom Zanussi * @dll_desc: The OCS DMA dma_list to use to store information about the
1410*fbf31dd5STom Zanussi * created linked list.
1411*fbf31dd5STom Zanussi * @data_size: The size of the data (from the SG list) to be mapped into the
1412*fbf31dd5STom Zanussi * OCS DMA linked list.
1413*fbf31dd5STom Zanussi * @data_offset: The offset (within the SG list) of the data to be mapped.
1414*fbf31dd5STom Zanussi *
1415*fbf31dd5STom Zanussi * Return: 0 on success, negative error code otherwise.
1416*fbf31dd5STom Zanussi */
ocs_create_linked_list_from_sg(const struct ocs_aes_dev * aes_dev,struct scatterlist * sg,int sg_dma_count,struct ocs_dll_desc * dll_desc,size_t data_size,size_t data_offset)1417*fbf31dd5STom Zanussi int ocs_create_linked_list_from_sg(const struct ocs_aes_dev *aes_dev,
1418*fbf31dd5STom Zanussi struct scatterlist *sg,
1419*fbf31dd5STom Zanussi int sg_dma_count,
1420*fbf31dd5STom Zanussi struct ocs_dll_desc *dll_desc,
1421*fbf31dd5STom Zanussi size_t data_size, size_t data_offset)
1422*fbf31dd5STom Zanussi {
1423*fbf31dd5STom Zanussi struct ocs_dma_linked_list *ll = NULL;
1424*fbf31dd5STom Zanussi struct scatterlist *sg_tmp;
1425*fbf31dd5STom Zanussi unsigned int tmp;
1426*fbf31dd5STom Zanussi int dma_nents;
1427*fbf31dd5STom Zanussi int i;
1428*fbf31dd5STom Zanussi
1429*fbf31dd5STom Zanussi if (!dll_desc || !sg || !aes_dev)
1430*fbf31dd5STom Zanussi return -EINVAL;
1431*fbf31dd5STom Zanussi
1432*fbf31dd5STom Zanussi /* Default values for when no ddl_desc is created. */
1433*fbf31dd5STom Zanussi dll_desc->vaddr = NULL;
1434*fbf31dd5STom Zanussi dll_desc->dma_addr = DMA_MAPPING_ERROR;
1435*fbf31dd5STom Zanussi dll_desc->size = 0;
1436*fbf31dd5STom Zanussi
1437*fbf31dd5STom Zanussi if (data_size == 0)
1438*fbf31dd5STom Zanussi return 0;
1439*fbf31dd5STom Zanussi
1440*fbf31dd5STom Zanussi /* Loop over sg_list until we reach entry at specified offset. */
1441*fbf31dd5STom Zanussi while (data_offset >= sg_dma_len(sg)) {
1442*fbf31dd5STom Zanussi data_offset -= sg_dma_len(sg);
1443*fbf31dd5STom Zanussi sg_dma_count--;
1444*fbf31dd5STom Zanussi sg = sg_next(sg);
1445*fbf31dd5STom Zanussi /* If we reach the end of the list, offset was invalid. */
1446*fbf31dd5STom Zanussi if (!sg || sg_dma_count == 0)
1447*fbf31dd5STom Zanussi return -EINVAL;
1448*fbf31dd5STom Zanussi }
1449*fbf31dd5STom Zanussi
1450*fbf31dd5STom Zanussi /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */
1451*fbf31dd5STom Zanussi dma_nents = 0;
1452*fbf31dd5STom Zanussi tmp = 0;
1453*fbf31dd5STom Zanussi sg_tmp = sg;
1454*fbf31dd5STom Zanussi while (tmp < data_offset + data_size) {
1455*fbf31dd5STom Zanussi /* If we reach the end of the list, data_size was invalid. */
1456*fbf31dd5STom Zanussi if (!sg_tmp)
1457*fbf31dd5STom Zanussi return -EINVAL;
1458*fbf31dd5STom Zanussi tmp += sg_dma_len(sg_tmp);
1459*fbf31dd5STom Zanussi dma_nents++;
1460*fbf31dd5STom Zanussi sg_tmp = sg_next(sg_tmp);
1461*fbf31dd5STom Zanussi }
1462*fbf31dd5STom Zanussi if (dma_nents > sg_dma_count)
1463*fbf31dd5STom Zanussi return -EINVAL;
1464*fbf31dd5STom Zanussi
1465*fbf31dd5STom Zanussi /* Allocate the DMA list, one entry for each SG entry. */
1466*fbf31dd5STom Zanussi dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents;
1467*fbf31dd5STom Zanussi dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size,
1468*fbf31dd5STom Zanussi &dll_desc->dma_addr, GFP_KERNEL);
1469*fbf31dd5STom Zanussi if (!dll_desc->vaddr)
1470*fbf31dd5STom Zanussi return -ENOMEM;
1471*fbf31dd5STom Zanussi
1472*fbf31dd5STom Zanussi /* Populate DMA linked list entries. */
1473*fbf31dd5STom Zanussi ll = dll_desc->vaddr;
1474*fbf31dd5STom Zanussi for (i = 0; i < dma_nents; i++, sg = sg_next(sg)) {
1475*fbf31dd5STom Zanussi ll[i].src_addr = sg_dma_address(sg) + data_offset;
1476*fbf31dd5STom Zanussi ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ?
1477*fbf31dd5STom Zanussi (sg_dma_len(sg) - data_offset) : data_size;
1478*fbf31dd5STom Zanussi data_offset = 0;
1479*fbf31dd5STom Zanussi data_size -= ll[i].src_len;
1480*fbf31dd5STom Zanussi /* Current element points to the DMA address of the next one. */
1481*fbf31dd5STom Zanussi ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1));
1482*fbf31dd5STom Zanussi ll[i].ll_flags = 0;
1483*fbf31dd5STom Zanussi }
1484*fbf31dd5STom Zanussi /* Terminate last element. */
1485*fbf31dd5STom Zanussi ll[i - 1].next = 0;
1486*fbf31dd5STom Zanussi ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE;
1487*fbf31dd5STom Zanussi
1488*fbf31dd5STom Zanussi return 0;
1489*fbf31dd5STom Zanussi }
1490