xref: /openbmc/linux/drivers/crypto/bcm/cipher.h (revision 654627ad)
1 
2 /* SPDX-License-Identifier: GPL-2.0-only */
3 /*
4  * Copyright 2016 Broadcom
5  */
6 
7 #ifndef _CIPHER_H
8 #define _CIPHER_H
9 
10 #include <linux/atomic.h>
11 #include <linux/mailbox/brcm-message.h>
12 #include <linux/mailbox_client.h>
13 #include <crypto/aes.h>
14 #include <crypto/internal/hash.h>
15 #include <crypto/internal/skcipher.h>
16 #include <crypto/aead.h>
17 #include <crypto/arc4.h>
18 #include <crypto/gcm.h>
19 #include <crypto/sha1.h>
20 #include <crypto/sha2.h>
21 #include <crypto/sha3.h>
22 
23 #include "spu.h"
24 #include "spum.h"
25 #include "spu2.h"
26 
27 /* Driver supports up to MAX_SPUS SPU blocks */
28 #define MAX_SPUS 16
29 
30 #define ARC4_STATE_SIZE     4
31 
32 #define CCM_AES_IV_SIZE    16
33 #define CCM_ESP_IV_SIZE     8
34 #define RFC4543_ICV_SIZE   16
35 
36 #define MAX_KEY_SIZE	ARC4_MAX_KEY_SIZE
37 #define MAX_IV_SIZE	AES_BLOCK_SIZE
38 #define MAX_DIGEST_SIZE	SHA3_512_DIGEST_SIZE
39 #define MAX_ASSOC_SIZE	512
40 
41 /* size of salt value for AES-GCM-ESP and AES-CCM-ESP */
42 #define GCM_ESP_SALT_SIZE   4
43 #define CCM_ESP_SALT_SIZE   3
44 #define MAX_SALT_SIZE       GCM_ESP_SALT_SIZE
45 #define GCM_ESP_SALT_OFFSET 0
46 #define CCM_ESP_SALT_OFFSET 1
47 
48 #define GCM_ESP_DIGESTSIZE 16
49 
50 #define MAX_HASH_BLOCK_SIZE SHA512_BLOCK_SIZE
51 
52 /*
53  * Maximum number of bytes from a non-final hash request that can be deferred
54  * until more data is available. With new crypto API framework, this
55  * can be no more than one block of data.
56  */
57 #define HASH_CARRY_MAX  MAX_HASH_BLOCK_SIZE
58 
59 /* Force at least 4-byte alignment of all SPU message fields */
60 #define SPU_MSG_ALIGN  4
61 
62 /* Number of times to resend mailbox message if mb queue is full */
63 #define SPU_MB_RETRY_MAX  1000
64 
65 /* op_counts[] indexes */
66 enum op_type {
67 	SPU_OP_CIPHER,
68 	SPU_OP_HASH,
69 	SPU_OP_HMAC,
70 	SPU_OP_AEAD,
71 	SPU_OP_NUM
72 };
73 
74 enum spu_spu_type {
75 	SPU_TYPE_SPUM,
76 	SPU_TYPE_SPU2,
77 };
78 
79 /*
80  * SPUM_NS2 and SPUM_NSP are the SPU-M block on Northstar 2 and Northstar Plus,
81  * respectively.
82  */
83 enum spu_spu_subtype {
84 	SPU_SUBTYPE_SPUM_NS2,
85 	SPU_SUBTYPE_SPUM_NSP,
86 	SPU_SUBTYPE_SPU2_V1,
87 	SPU_SUBTYPE_SPU2_V2
88 };
89 
90 struct spu_type_subtype {
91 	enum spu_spu_type type;
92 	enum spu_spu_subtype subtype;
93 };
94 
95 struct cipher_op {
96 	enum spu_cipher_alg alg;
97 	enum spu_cipher_mode mode;
98 };
99 
100 struct auth_op {
101 	enum hash_alg alg;
102 	enum hash_mode mode;
103 };
104 
105 struct iproc_alg_s {
106 	u32 type;
107 	union {
108 		struct skcipher_alg skcipher;
109 		struct ahash_alg hash;
110 		struct aead_alg aead;
111 	} alg;
112 	struct cipher_op cipher_info;
113 	struct auth_op auth_info;
114 	bool auth_first;
115 	bool registered;
116 };
117 
118 /*
119  * Buffers for a SPU request/reply message pair. All part of one structure to
120  * allow a single alloc per request.
121  */
122 struct spu_msg_buf {
123 	/* Request message fragments */
124 
125 	/*
126 	 * SPU request message header. For SPU-M, holds MH, EMH, SCTX, BDESC,
127 	 * and BD header. For SPU2, holds FMD, OMD.
128 	 */
129 	u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
130 
131 	/* IV or counter. Size to include salt. Also used for XTS tweek. */
132 	u8 iv_ctr[ALIGN(2 * AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
133 
134 	/* Hash digest. request and response. */
135 	u8 digest[ALIGN(MAX_DIGEST_SIZE, SPU_MSG_ALIGN)];
136 
137 	/* SPU request message padding */
138 	u8 spu_req_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
139 
140 	/* SPU-M request message STATUS field */
141 	u8 tx_stat[ALIGN(SPU_TX_STATUS_LEN, SPU_MSG_ALIGN)];
142 
143 	/* Response message fragments */
144 
145 	/* SPU response message header */
146 	u8 spu_resp_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
147 
148 	/* SPU response message STATUS field padding */
149 	u8 rx_stat_pad[ALIGN(SPU_STAT_PAD_MAX, SPU_MSG_ALIGN)];
150 
151 	/* SPU response message STATUS field */
152 	u8 rx_stat[ALIGN(SPU_RX_STATUS_LEN, SPU_MSG_ALIGN)];
153 
154 	union {
155 		/* Buffers only used for skcipher */
156 		struct {
157 			/*
158 			 * Field used for either SUPDT when RC4 is used
159 			 * -OR- tweak value when XTS/AES is used
160 			 */
161 			u8 supdt_tweak[ALIGN(SPU_SUPDT_LEN, SPU_MSG_ALIGN)];
162 		} c;
163 
164 		/* Buffers only used for aead */
165 		struct {
166 			/* SPU response pad for GCM data */
167 			u8 gcmpad[ALIGN(AES_BLOCK_SIZE, SPU_MSG_ALIGN)];
168 
169 			/* SPU request msg padding for GCM AAD */
170 			u8 req_aad_pad[ALIGN(SPU_PAD_LEN_MAX, SPU_MSG_ALIGN)];
171 
172 			/* SPU response data to be discarded */
173 			u8 resp_aad[ALIGN(MAX_ASSOC_SIZE + MAX_IV_SIZE,
174 					  SPU_MSG_ALIGN)];
175 		} a;
176 	};
177 };
178 
179 struct iproc_ctx_s {
180 	u8 enckey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
181 	unsigned int enckeylen;
182 
183 	u8 authkey[MAX_KEY_SIZE + ARC4_STATE_SIZE];
184 	unsigned int authkeylen;
185 
186 	u8 salt[MAX_SALT_SIZE];
187 	unsigned int salt_len;
188 	unsigned int salt_offset;
189 	u8 iv[MAX_IV_SIZE];
190 
191 	unsigned int digestsize;
192 
193 	struct iproc_alg_s *alg;
194 	bool is_esp;
195 
196 	struct cipher_op cipher;
197 	enum spu_cipher_type cipher_type;
198 
199 	struct auth_op auth;
200 	bool auth_first;
201 
202 	/*
203 	 * The maximum length in bytes of the payload in a SPU message for this
204 	 * context. For SPU-M, the payload is the combination of AAD and data.
205 	 * For SPU2, the payload is just data. A value of SPU_MAX_PAYLOAD_INF
206 	 * indicates that there is no limit to the length of the SPU message
207 	 * payload.
208 	 */
209 	unsigned int max_payload;
210 
211 	struct crypto_aead *fallback_cipher;
212 
213 	/* auth_type is determined during processing of request */
214 
215 	u8 ipad[MAX_HASH_BLOCK_SIZE];
216 	u8 opad[MAX_HASH_BLOCK_SIZE];
217 
218 	/*
219 	 * Buffer to hold SPU message header template. Template is created at
220 	 * setkey time for skcipher requests, since most of the fields in the
221 	 * header are known at that time. At request time, just fill in a few
222 	 * missing pieces related to length of data in the request and IVs, etc.
223 	 */
224 	u8 bcm_spu_req_hdr[ALIGN(SPU2_HEADER_ALLOC_LEN, SPU_MSG_ALIGN)];
225 
226 	/* Length of SPU request header */
227 	u16 spu_req_hdr_len;
228 
229 	/* Expected length of SPU response header */
230 	u16 spu_resp_hdr_len;
231 
232 	/*
233 	 * shash descriptor - needed to perform incremental hashing in
234 	 * software, when hw doesn't support it.
235 	 */
236 	struct shash_desc *shash;
237 
238 	bool is_rfc4543;	/* RFC 4543 style of GMAC */
239 };
240 
241 /* state from iproc_reqctx_s necessary for hash state export/import */
242 struct spu_hash_export_s {
243 	unsigned int total_todo;
244 	unsigned int total_sent;
245 	u8 hash_carry[HASH_CARRY_MAX];
246 	unsigned int hash_carry_len;
247 	u8 incr_hash[MAX_DIGEST_SIZE];
248 	bool is_sw_hmac;
249 };
250 
251 struct iproc_reqctx_s {
252 	/* general context */
253 	struct crypto_async_request *parent;
254 
255 	/* only valid after enqueue() */
256 	struct iproc_ctx_s *ctx;
257 
258 	u8 chan_idx;   /* Mailbox channel to be used to submit this request */
259 
260 	/* total todo, rx'd, and sent for this request */
261 	unsigned int total_todo;
262 	unsigned int total_received;	/* only valid for skcipher */
263 	unsigned int total_sent;
264 
265 	/*
266 	 * num bytes sent to hw from the src sg in this request. This can differ
267 	 * from total_sent for incremental hashing. total_sent includes previous
268 	 * init() and update() data. src_sent does not.
269 	 */
270 	unsigned int src_sent;
271 
272 	/*
273 	 * For AEAD requests, start of associated data. This will typically
274 	 * point to the beginning of the src scatterlist from the request,
275 	 * since assoc data is at the beginning of the src scatterlist rather
276 	 * than in its own sg.
277 	 */
278 	struct scatterlist *assoc;
279 
280 	/*
281 	 * scatterlist entry and offset to start of data for next chunk. Crypto
282 	 * API src scatterlist for AEAD starts with AAD, if present. For first
283 	 * chunk, src_sg is sg entry at beginning of input data (after AAD).
284 	 * src_skip begins at the offset in that sg entry where data begins.
285 	 */
286 	struct scatterlist *src_sg;
287 	int src_nents;		/* Number of src entries with data */
288 	u32 src_skip;		/* bytes of current sg entry already used */
289 
290 	/*
291 	 * Same for destination. For AEAD, if there is AAD, output data must
292 	 * be written at offset following AAD.
293 	 */
294 	struct scatterlist *dst_sg;
295 	int dst_nents;		/* Number of dst entries with data */
296 	u32 dst_skip;		/* bytes of current sg entry already written */
297 
298 	/* Mailbox message used to send this request to PDC driver */
299 	struct brcm_message mb_mssg;
300 
301 	bool bd_suppress;	/* suppress BD field in SPU response? */
302 
303 	/* cipher context */
304 	bool is_encrypt;
305 
306 	/*
307 	 * CBC mode: IV.  CTR mode: counter.  Else empty. Used as a DMA
308 	 * buffer for AEAD requests. So allocate as DMAable memory. If IV
309 	 * concatenated with salt, includes the salt.
310 	 */
311 	u8 *iv_ctr;
312 	/* Length of IV or counter, in bytes */
313 	unsigned int iv_ctr_len;
314 
315 	/*
316 	 * Hash requests can be of any size, whether initial, update, or final.
317 	 * A non-final request must be submitted to the SPU as an integral
318 	 * number of blocks. This may leave data at the end of the request
319 	 * that is not a full block. Since the request is non-final, it cannot
320 	 * be padded. So, we write the remainder to this hash_carry buffer and
321 	 * hold it until the next request arrives. The carry data is then
322 	 * submitted at the beginning of the data in the next SPU msg.
323 	 * hash_carry_len is the number of bytes currently in hash_carry. These
324 	 * fields are only used for ahash requests.
325 	 */
326 	u8 hash_carry[HASH_CARRY_MAX];
327 	unsigned int hash_carry_len;
328 	unsigned int is_final;	/* is this the final for the hash op? */
329 
330 	/*
331 	 * Digest from incremental hash is saved here to include in next hash
332 	 * operation. Cannot be stored in req->result for truncated hashes,
333 	 * since result may be sized for final digest. Cannot be saved in
334 	 * msg_buf because that gets deleted between incremental hash ops
335 	 * and is not saved as part of export().
336 	 */
337 	u8 incr_hash[MAX_DIGEST_SIZE];
338 
339 	/* hmac context */
340 	bool is_sw_hmac;
341 
342 	gfp_t gfp;
343 
344 	/* Buffers used to build SPU request and response messages */
345 	struct spu_msg_buf msg_buf;
346 
347 	struct aead_request req;
348 };
349 
350 /*
351  * Structure encapsulates a set of function pointers specific to the type of
352  * SPU hardware running. These functions handling creation and parsing of
353  * SPU request messages and SPU response messages. Includes hardware-specific
354  * values read from device tree.
355  */
356 struct spu_hw {
357 	void (*spu_dump_msg_hdr)(u8 *buf, unsigned int buf_len);
358 	u32 (*spu_ctx_max_payload)(enum spu_cipher_alg cipher_alg,
359 				   enum spu_cipher_mode cipher_mode,
360 				   unsigned int blocksize);
361 	u32 (*spu_payload_length)(u8 *spu_hdr);
362 	u16 (*spu_response_hdr_len)(u16 auth_key_len, u16 enc_key_len,
363 				    bool is_hash);
364 	u16 (*spu_hash_pad_len)(enum hash_alg hash_alg,
365 				enum hash_mode hash_mode, u32 chunksize,
366 				u16 hash_block_size);
367 	u32 (*spu_gcm_ccm_pad_len)(enum spu_cipher_mode cipher_mode,
368 				   unsigned int data_size);
369 	u32 (*spu_assoc_resp_len)(enum spu_cipher_mode cipher_mode,
370 				  unsigned int assoc_len,
371 				  unsigned int iv_len, bool is_encrypt);
372 	u8 (*spu_aead_ivlen)(enum spu_cipher_mode cipher_mode,
373 			     u16 iv_len);
374 	enum hash_type (*spu_hash_type)(u32 src_sent);
375 	u32 (*spu_digest_size)(u32 digest_size, enum hash_alg alg,
376 			       enum hash_type);
377 	u32 (*spu_create_request)(u8 *spu_hdr,
378 				  struct spu_request_opts *req_opts,
379 				  struct spu_cipher_parms *cipher_parms,
380 				  struct spu_hash_parms *hash_parms,
381 				  struct spu_aead_parms *aead_parms,
382 				  unsigned int data_size);
383 	u16 (*spu_cipher_req_init)(u8 *spu_hdr,
384 				   struct spu_cipher_parms *cipher_parms);
385 	void (*spu_cipher_req_finish)(u8 *spu_hdr,
386 				      u16 spu_req_hdr_len,
387 				      unsigned int is_inbound,
388 				      struct spu_cipher_parms *cipher_parms,
389 				      unsigned int data_size);
390 	void (*spu_request_pad)(u8 *pad_start, u32 gcm_padding,
391 				u32 hash_pad_len, enum hash_alg auth_alg,
392 				enum hash_mode auth_mode,
393 				unsigned int total_sent, u32 status_padding);
394 	u8 (*spu_xts_tweak_in_payload)(void);
395 	u8 (*spu_tx_status_len)(void);
396 	u8 (*spu_rx_status_len)(void);
397 	int (*spu_status_process)(u8 *statp);
398 	void (*spu_ccm_update_iv)(unsigned int digestsize,
399 				  struct spu_cipher_parms *cipher_parms,
400 				  unsigned int assoclen, unsigned int chunksize,
401 				  bool is_encrypt, bool is_esp);
402 	u32 (*spu_wordalign_padlen)(u32 data_size);
403 
404 	/* The base virtual address of the SPU hw registers */
405 	void __iomem *reg_vbase[MAX_SPUS];
406 
407 	/* Version of the SPU hardware */
408 	enum spu_spu_type spu_type;
409 
410 	/* Sub-version of the SPU hardware */
411 	enum spu_spu_subtype spu_subtype;
412 
413 	/* The number of SPUs on this platform */
414 	u32 num_spu;
415 
416 	/* The number of SPU channels on this platform */
417 	u32 num_chan;
418 };
419 
420 struct bcm_device_private {
421 	struct platform_device *pdev;
422 
423 	struct spu_hw spu;
424 
425 	atomic_t session_count;	/* number of streams active */
426 	atomic_t stream_count;	/* monotonic counter for streamID's */
427 
428 	/* Length of BCM header. Set to 0 when hw does not expect BCM HEADER. */
429 	u8 bcm_hdr_len;
430 
431 	/* The index of the channel to use for the next crypto request */
432 	atomic_t next_chan;
433 
434 	struct dentry *debugfs_dir;
435 	struct dentry *debugfs_stats;
436 
437 	/* Number of request bytes processed and result bytes returned */
438 	atomic64_t bytes_in;
439 	atomic64_t bytes_out;
440 
441 	/* Number of operations of each type */
442 	atomic_t op_counts[SPU_OP_NUM];
443 
444 	atomic_t cipher_cnt[CIPHER_ALG_LAST][CIPHER_MODE_LAST];
445 	atomic_t hash_cnt[HASH_ALG_LAST];
446 	atomic_t hmac_cnt[HASH_ALG_LAST];
447 	atomic_t aead_cnt[AEAD_TYPE_LAST];
448 
449 	/* Number of calls to setkey() for each operation type */
450 	atomic_t setkey_cnt[SPU_OP_NUM];
451 
452 	/* Number of times request was resubmitted because mb was full */
453 	atomic_t mb_no_spc;
454 
455 	/* Number of mailbox send failures */
456 	atomic_t mb_send_fail;
457 
458 	/* Number of ICV check failures for AEAD messages */
459 	atomic_t bad_icv;
460 
461 	struct mbox_client mcl;
462 
463 	/* Array of mailbox channel pointers, one for each channel */
464 	struct mbox_chan **mbox;
465 };
466 
467 extern struct bcm_device_private iproc_priv;
468 
469 #endif
470