1 // SPDX-License-Identifier: (BSD-3-Clause OR GPL-2.0-only)
2 /* Copyright(c) 2014 - 2020 Intel Corporation */
3 #include <linux/module.h>
4 #include <linux/slab.h>
5 #include <linux/crypto.h>
6 #include <crypto/internal/aead.h>
7 #include <crypto/internal/cipher.h>
8 #include <crypto/internal/skcipher.h>
9 #include <crypto/aes.h>
10 #include <crypto/sha1.h>
11 #include <crypto/sha2.h>
12 #include <crypto/hash.h>
13 #include <crypto/hmac.h>
14 #include <crypto/algapi.h>
15 #include <crypto/authenc.h>
16 #include <crypto/scatterwalk.h>
17 #include <crypto/xts.h>
18 #include <linux/dma-mapping.h>
19 #include "adf_accel_devices.h"
20 #include "qat_algs_send.h"
21 #include "adf_common_drv.h"
22 #include "qat_crypto.h"
23 #include "icp_qat_hw.h"
24 #include "icp_qat_fw.h"
25 #include "icp_qat_fw_la.h"
26 #include "qat_bl.h"
27 
28 #define QAT_AES_HW_CONFIG_ENC(alg, mode) \
29 	ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
30 				       ICP_QAT_HW_CIPHER_NO_CONVERT, \
31 				       ICP_QAT_HW_CIPHER_ENCRYPT)
32 
33 #define QAT_AES_HW_CONFIG_DEC(alg, mode) \
34 	ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
35 				       ICP_QAT_HW_CIPHER_KEY_CONVERT, \
36 				       ICP_QAT_HW_CIPHER_DECRYPT)
37 
38 #define QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode) \
39 	ICP_QAT_HW_CIPHER_CONFIG_BUILD(mode, alg, \
40 				       ICP_QAT_HW_CIPHER_NO_CONVERT, \
41 				       ICP_QAT_HW_CIPHER_DECRYPT)
42 
43 #define HW_CAP_AES_V2(accel_dev) \
44 	(GET_HW_DATA(accel_dev)->accel_capabilities_mask & \
45 	 ICP_ACCEL_CAPABILITIES_AES_V2)
46 
47 static DEFINE_MUTEX(algs_lock);
48 static unsigned int active_devs;
49 
50 /* Common content descriptor */
51 struct qat_alg_cd {
52 	union {
53 		struct qat_enc { /* Encrypt content desc */
54 			struct icp_qat_hw_cipher_algo_blk cipher;
55 			struct icp_qat_hw_auth_algo_blk hash;
56 		} qat_enc_cd;
57 		struct qat_dec { /* Decrypt content desc */
58 			struct icp_qat_hw_auth_algo_blk hash;
59 			struct icp_qat_hw_cipher_algo_blk cipher;
60 		} qat_dec_cd;
61 	};
62 } __aligned(64);
63 
64 struct qat_alg_aead_ctx {
65 	struct qat_alg_cd *enc_cd;
66 	struct qat_alg_cd *dec_cd;
67 	dma_addr_t enc_cd_paddr;
68 	dma_addr_t dec_cd_paddr;
69 	struct icp_qat_fw_la_bulk_req enc_fw_req;
70 	struct icp_qat_fw_la_bulk_req dec_fw_req;
71 	struct crypto_shash *hash_tfm;
72 	enum icp_qat_hw_auth_algo qat_hash_alg;
73 	struct qat_crypto_instance *inst;
74 	union {
75 		struct sha1_state sha1;
76 		struct sha256_state sha256;
77 		struct sha512_state sha512;
78 	};
79 	char ipad[SHA512_BLOCK_SIZE]; /* sufficient for SHA-1/SHA-256 as well */
80 	char opad[SHA512_BLOCK_SIZE];
81 };
82 
83 struct qat_alg_skcipher_ctx {
84 	struct icp_qat_hw_cipher_algo_blk *enc_cd;
85 	struct icp_qat_hw_cipher_algo_blk *dec_cd;
86 	dma_addr_t enc_cd_paddr;
87 	dma_addr_t dec_cd_paddr;
88 	struct icp_qat_fw_la_bulk_req enc_fw_req;
89 	struct icp_qat_fw_la_bulk_req dec_fw_req;
90 	struct qat_crypto_instance *inst;
91 	struct crypto_skcipher *ftfm;
92 	struct crypto_cipher *tweak;
93 	bool fallback;
94 	int mode;
95 };
96 
qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)97 static int qat_get_inter_state_size(enum icp_qat_hw_auth_algo qat_hash_alg)
98 {
99 	switch (qat_hash_alg) {
100 	case ICP_QAT_HW_AUTH_ALGO_SHA1:
101 		return ICP_QAT_HW_SHA1_STATE1_SZ;
102 	case ICP_QAT_HW_AUTH_ALGO_SHA256:
103 		return ICP_QAT_HW_SHA256_STATE1_SZ;
104 	case ICP_QAT_HW_AUTH_ALGO_SHA512:
105 		return ICP_QAT_HW_SHA512_STATE1_SZ;
106 	default:
107 		return -EFAULT;
108 	}
109 }
110 
qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk * hash,struct qat_alg_aead_ctx * ctx,const u8 * auth_key,unsigned int auth_keylen)111 static int qat_alg_do_precomputes(struct icp_qat_hw_auth_algo_blk *hash,
112 				  struct qat_alg_aead_ctx *ctx,
113 				  const u8 *auth_key,
114 				  unsigned int auth_keylen)
115 {
116 	SHASH_DESC_ON_STACK(shash, ctx->hash_tfm);
117 	int block_size = crypto_shash_blocksize(ctx->hash_tfm);
118 	int digest_size = crypto_shash_digestsize(ctx->hash_tfm);
119 	__be32 *hash_state_out;
120 	__be64 *hash512_state_out;
121 	int i, offset;
122 
123 	memset(ctx->ipad, 0, block_size);
124 	memset(ctx->opad, 0, block_size);
125 	shash->tfm = ctx->hash_tfm;
126 
127 	if (auth_keylen > block_size) {
128 		int ret = crypto_shash_digest(shash, auth_key,
129 					      auth_keylen, ctx->ipad);
130 		if (ret)
131 			return ret;
132 
133 		memcpy(ctx->opad, ctx->ipad, digest_size);
134 	} else {
135 		memcpy(ctx->ipad, auth_key, auth_keylen);
136 		memcpy(ctx->opad, auth_key, auth_keylen);
137 	}
138 
139 	for (i = 0; i < block_size; i++) {
140 		char *ipad_ptr = ctx->ipad + i;
141 		char *opad_ptr = ctx->opad + i;
142 		*ipad_ptr ^= HMAC_IPAD_VALUE;
143 		*opad_ptr ^= HMAC_OPAD_VALUE;
144 	}
145 
146 	if (crypto_shash_init(shash))
147 		return -EFAULT;
148 
149 	if (crypto_shash_update(shash, ctx->ipad, block_size))
150 		return -EFAULT;
151 
152 	hash_state_out = (__be32 *)hash->sha.state1;
153 	hash512_state_out = (__be64 *)hash_state_out;
154 
155 	switch (ctx->qat_hash_alg) {
156 	case ICP_QAT_HW_AUTH_ALGO_SHA1:
157 		if (crypto_shash_export(shash, &ctx->sha1))
158 			return -EFAULT;
159 		for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
160 			*hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
161 		break;
162 	case ICP_QAT_HW_AUTH_ALGO_SHA256:
163 		if (crypto_shash_export(shash, &ctx->sha256))
164 			return -EFAULT;
165 		for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
166 			*hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
167 		break;
168 	case ICP_QAT_HW_AUTH_ALGO_SHA512:
169 		if (crypto_shash_export(shash, &ctx->sha512))
170 			return -EFAULT;
171 		for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
172 			*hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
173 		break;
174 	default:
175 		return -EFAULT;
176 	}
177 
178 	if (crypto_shash_init(shash))
179 		return -EFAULT;
180 
181 	if (crypto_shash_update(shash, ctx->opad, block_size))
182 		return -EFAULT;
183 
184 	offset = round_up(qat_get_inter_state_size(ctx->qat_hash_alg), 8);
185 	if (offset < 0)
186 		return -EFAULT;
187 
188 	hash_state_out = (__be32 *)(hash->sha.state1 + offset);
189 	hash512_state_out = (__be64 *)hash_state_out;
190 
191 	switch (ctx->qat_hash_alg) {
192 	case ICP_QAT_HW_AUTH_ALGO_SHA1:
193 		if (crypto_shash_export(shash, &ctx->sha1))
194 			return -EFAULT;
195 		for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
196 			*hash_state_out = cpu_to_be32(ctx->sha1.state[i]);
197 		break;
198 	case ICP_QAT_HW_AUTH_ALGO_SHA256:
199 		if (crypto_shash_export(shash, &ctx->sha256))
200 			return -EFAULT;
201 		for (i = 0; i < digest_size >> 2; i++, hash_state_out++)
202 			*hash_state_out = cpu_to_be32(ctx->sha256.state[i]);
203 		break;
204 	case ICP_QAT_HW_AUTH_ALGO_SHA512:
205 		if (crypto_shash_export(shash, &ctx->sha512))
206 			return -EFAULT;
207 		for (i = 0; i < digest_size >> 3; i++, hash512_state_out++)
208 			*hash512_state_out = cpu_to_be64(ctx->sha512.state[i]);
209 		break;
210 	default:
211 		return -EFAULT;
212 	}
213 	memzero_explicit(ctx->ipad, block_size);
214 	memzero_explicit(ctx->opad, block_size);
215 	return 0;
216 }
217 
qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr * header)218 static void qat_alg_init_common_hdr(struct icp_qat_fw_comn_req_hdr *header)
219 {
220 	header->hdr_flags =
221 		ICP_QAT_FW_COMN_HDR_FLAGS_BUILD(ICP_QAT_FW_COMN_REQ_FLAG_SET);
222 	header->service_type = ICP_QAT_FW_COMN_REQ_CPM_FW_LA;
223 	header->comn_req_flags =
224 		ICP_QAT_FW_COMN_FLAGS_BUILD(QAT_COMN_CD_FLD_TYPE_64BIT_ADR,
225 					    QAT_COMN_PTR_TYPE_SGL);
226 	ICP_QAT_FW_LA_PARTIAL_SET(header->serv_specif_flags,
227 				  ICP_QAT_FW_LA_PARTIAL_NONE);
228 	ICP_QAT_FW_LA_CIPH_IV_FLD_FLAG_SET(header->serv_specif_flags,
229 					   ICP_QAT_FW_CIPH_IV_16BYTE_DATA);
230 	ICP_QAT_FW_LA_PROTO_SET(header->serv_specif_flags,
231 				ICP_QAT_FW_LA_NO_PROTO);
232 	ICP_QAT_FW_LA_UPDATE_STATE_SET(header->serv_specif_flags,
233 				       ICP_QAT_FW_LA_NO_UPDATE_STATE);
234 }
235 
qat_alg_aead_init_enc_session(struct crypto_aead * aead_tfm,int alg,struct crypto_authenc_keys * keys,int mode)236 static int qat_alg_aead_init_enc_session(struct crypto_aead *aead_tfm,
237 					 int alg,
238 					 struct crypto_authenc_keys *keys,
239 					 int mode)
240 {
241 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
242 	unsigned int digestsize = crypto_aead_authsize(aead_tfm);
243 	struct qat_enc *enc_ctx = &ctx->enc_cd->qat_enc_cd;
244 	struct icp_qat_hw_cipher_algo_blk *cipher = &enc_ctx->cipher;
245 	struct icp_qat_hw_auth_algo_blk *hash =
246 		(struct icp_qat_hw_auth_algo_blk *)((char *)enc_ctx +
247 		sizeof(struct icp_qat_hw_auth_setup) + keys->enckeylen);
248 	struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->enc_fw_req;
249 	struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
250 	struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
251 	void *ptr = &req_tmpl->cd_ctrl;
252 	struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
253 	struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
254 
255 	/* CD setup */
256 	cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
257 	memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
258 	hash->sha.inner_setup.auth_config.config =
259 		ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
260 					     ctx->qat_hash_alg, digestsize);
261 	hash->sha.inner_setup.auth_counter.counter =
262 		cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
263 
264 	if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
265 		return -EFAULT;
266 
267 	/* Request setup */
268 	qat_alg_init_common_hdr(header);
269 	header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER_HASH;
270 	ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
271 					   ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
272 	ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
273 				   ICP_QAT_FW_LA_RET_AUTH_RES);
274 	ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
275 				   ICP_QAT_FW_LA_NO_CMP_AUTH_RES);
276 	cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
277 	cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
278 
279 	/* Cipher CD config setup */
280 	cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
281 	cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
282 	cipher_cd_ctrl->cipher_cfg_offset = 0;
283 	ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
284 	ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
285 	/* Auth CD config setup */
286 	hash_cd_ctrl->hash_cfg_offset = ((char *)hash - (char *)cipher) >> 3;
287 	hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
288 	hash_cd_ctrl->inner_res_sz = digestsize;
289 	hash_cd_ctrl->final_sz = digestsize;
290 
291 	switch (ctx->qat_hash_alg) {
292 	case ICP_QAT_HW_AUTH_ALGO_SHA1:
293 		hash_cd_ctrl->inner_state1_sz =
294 			round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
295 		hash_cd_ctrl->inner_state2_sz =
296 			round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
297 		break;
298 	case ICP_QAT_HW_AUTH_ALGO_SHA256:
299 		hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
300 		hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
301 		break;
302 	case ICP_QAT_HW_AUTH_ALGO_SHA512:
303 		hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
304 		hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
305 		break;
306 	default:
307 		break;
308 	}
309 	hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
310 			((sizeof(struct icp_qat_hw_auth_setup) +
311 			 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
312 	ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
313 	ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
314 	return 0;
315 }
316 
qat_alg_aead_init_dec_session(struct crypto_aead * aead_tfm,int alg,struct crypto_authenc_keys * keys,int mode)317 static int qat_alg_aead_init_dec_session(struct crypto_aead *aead_tfm,
318 					 int alg,
319 					 struct crypto_authenc_keys *keys,
320 					 int mode)
321 {
322 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(aead_tfm);
323 	unsigned int digestsize = crypto_aead_authsize(aead_tfm);
324 	struct qat_dec *dec_ctx = &ctx->dec_cd->qat_dec_cd;
325 	struct icp_qat_hw_auth_algo_blk *hash = &dec_ctx->hash;
326 	struct icp_qat_hw_cipher_algo_blk *cipher =
327 		(struct icp_qat_hw_cipher_algo_blk *)((char *)dec_ctx +
328 		sizeof(struct icp_qat_hw_auth_setup) +
329 		roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2);
330 	struct icp_qat_fw_la_bulk_req *req_tmpl = &ctx->dec_fw_req;
331 	struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req_tmpl->cd_pars;
332 	struct icp_qat_fw_comn_req_hdr *header = &req_tmpl->comn_hdr;
333 	void *ptr = &req_tmpl->cd_ctrl;
334 	struct icp_qat_fw_cipher_cd_ctrl_hdr *cipher_cd_ctrl = ptr;
335 	struct icp_qat_fw_auth_cd_ctrl_hdr *hash_cd_ctrl = ptr;
336 	struct icp_qat_fw_la_auth_req_params *auth_param =
337 		(struct icp_qat_fw_la_auth_req_params *)
338 		((char *)&req_tmpl->serv_specif_rqpars +
339 		sizeof(struct icp_qat_fw_la_cipher_req_params));
340 
341 	/* CD setup */
342 	cipher->aes.cipher_config.val = QAT_AES_HW_CONFIG_DEC(alg, mode);
343 	memcpy(cipher->aes.key, keys->enckey, keys->enckeylen);
344 	hash->sha.inner_setup.auth_config.config =
345 		ICP_QAT_HW_AUTH_CONFIG_BUILD(ICP_QAT_HW_AUTH_MODE1,
346 					     ctx->qat_hash_alg,
347 					     digestsize);
348 	hash->sha.inner_setup.auth_counter.counter =
349 		cpu_to_be32(crypto_shash_blocksize(ctx->hash_tfm));
350 
351 	if (qat_alg_do_precomputes(hash, ctx, keys->authkey, keys->authkeylen))
352 		return -EFAULT;
353 
354 	/* Request setup */
355 	qat_alg_init_common_hdr(header);
356 	header->service_cmd_id = ICP_QAT_FW_LA_CMD_HASH_CIPHER;
357 	ICP_QAT_FW_LA_DIGEST_IN_BUFFER_SET(header->serv_specif_flags,
358 					   ICP_QAT_FW_LA_DIGEST_IN_BUFFER);
359 	ICP_QAT_FW_LA_RET_AUTH_SET(header->serv_specif_flags,
360 				   ICP_QAT_FW_LA_NO_RET_AUTH_RES);
361 	ICP_QAT_FW_LA_CMP_AUTH_SET(header->serv_specif_flags,
362 				   ICP_QAT_FW_LA_CMP_AUTH_RES);
363 	cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
364 	cd_pars->u.s.content_desc_params_sz = sizeof(struct qat_alg_cd) >> 3;
365 
366 	/* Cipher CD config setup */
367 	cipher_cd_ctrl->cipher_key_sz = keys->enckeylen >> 3;
368 	cipher_cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
369 	cipher_cd_ctrl->cipher_cfg_offset =
370 		(sizeof(struct icp_qat_hw_auth_setup) +
371 		 roundup(crypto_shash_digestsize(ctx->hash_tfm), 8) * 2) >> 3;
372 	ICP_QAT_FW_COMN_CURR_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
373 	ICP_QAT_FW_COMN_NEXT_ID_SET(cipher_cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
374 
375 	/* Auth CD config setup */
376 	hash_cd_ctrl->hash_cfg_offset = 0;
377 	hash_cd_ctrl->hash_flags = ICP_QAT_FW_AUTH_HDR_FLAG_NO_NESTED;
378 	hash_cd_ctrl->inner_res_sz = digestsize;
379 	hash_cd_ctrl->final_sz = digestsize;
380 
381 	switch (ctx->qat_hash_alg) {
382 	case ICP_QAT_HW_AUTH_ALGO_SHA1:
383 		hash_cd_ctrl->inner_state1_sz =
384 			round_up(ICP_QAT_HW_SHA1_STATE1_SZ, 8);
385 		hash_cd_ctrl->inner_state2_sz =
386 			round_up(ICP_QAT_HW_SHA1_STATE2_SZ, 8);
387 		break;
388 	case ICP_QAT_HW_AUTH_ALGO_SHA256:
389 		hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA256_STATE1_SZ;
390 		hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA256_STATE2_SZ;
391 		break;
392 	case ICP_QAT_HW_AUTH_ALGO_SHA512:
393 		hash_cd_ctrl->inner_state1_sz = ICP_QAT_HW_SHA512_STATE1_SZ;
394 		hash_cd_ctrl->inner_state2_sz = ICP_QAT_HW_SHA512_STATE2_SZ;
395 		break;
396 	default:
397 		break;
398 	}
399 
400 	hash_cd_ctrl->inner_state2_offset = hash_cd_ctrl->hash_cfg_offset +
401 			((sizeof(struct icp_qat_hw_auth_setup) +
402 			 round_up(hash_cd_ctrl->inner_state1_sz, 8)) >> 3);
403 	auth_param->auth_res_sz = digestsize;
404 	ICP_QAT_FW_COMN_CURR_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_AUTH);
405 	ICP_QAT_FW_COMN_NEXT_ID_SET(hash_cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
406 	return 0;
407 }
408 
qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx * ctx,struct icp_qat_fw_la_bulk_req * req,struct icp_qat_hw_cipher_algo_blk * cd,const u8 * key,unsigned int keylen)409 static void qat_alg_skcipher_init_com(struct qat_alg_skcipher_ctx *ctx,
410 				      struct icp_qat_fw_la_bulk_req *req,
411 				      struct icp_qat_hw_cipher_algo_blk *cd,
412 				      const u8 *key, unsigned int keylen)
413 {
414 	struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
415 	struct icp_qat_fw_comn_req_hdr *header = &req->comn_hdr;
416 	struct icp_qat_fw_cipher_cd_ctrl_hdr *cd_ctrl = (void *)&req->cd_ctrl;
417 	bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
418 	int mode = ctx->mode;
419 
420 	qat_alg_init_common_hdr(header);
421 	header->service_cmd_id = ICP_QAT_FW_LA_CMD_CIPHER;
422 	cd_pars->u.s.content_desc_params_sz =
423 				sizeof(struct icp_qat_hw_cipher_algo_blk) >> 3;
424 
425 	if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
426 		ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
427 					     ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
428 
429 		/* Store both XTS keys in CD, only the first key is sent
430 		 * to the HW, the second key is used for tweak calculation
431 		 */
432 		memcpy(cd->ucs_aes.key, key, keylen);
433 		keylen = keylen / 2;
434 	} else if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_CTR_MODE) {
435 		ICP_QAT_FW_LA_SLICE_TYPE_SET(header->serv_specif_flags,
436 					     ICP_QAT_FW_LA_USE_UCS_SLICE_TYPE);
437 		memcpy(cd->ucs_aes.key, key, keylen);
438 		keylen = round_up(keylen, 16);
439 	} else {
440 		memcpy(cd->aes.key, key, keylen);
441 	}
442 
443 	/* Cipher CD config setup */
444 	cd_ctrl->cipher_key_sz = keylen >> 3;
445 	cd_ctrl->cipher_state_sz = AES_BLOCK_SIZE >> 3;
446 	cd_ctrl->cipher_cfg_offset = 0;
447 	ICP_QAT_FW_COMN_CURR_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_CIPHER);
448 	ICP_QAT_FW_COMN_NEXT_ID_SET(cd_ctrl, ICP_QAT_FW_SLICE_DRAM_WR);
449 }
450 
qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx * ctx,int alg,const u8 * key,unsigned int keylen,int mode)451 static void qat_alg_skcipher_init_enc(struct qat_alg_skcipher_ctx *ctx,
452 				      int alg, const u8 *key,
453 				      unsigned int keylen, int mode)
454 {
455 	struct icp_qat_hw_cipher_algo_blk *enc_cd = ctx->enc_cd;
456 	struct icp_qat_fw_la_bulk_req *req = &ctx->enc_fw_req;
457 	struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
458 
459 	qat_alg_skcipher_init_com(ctx, req, enc_cd, key, keylen);
460 	cd_pars->u.s.content_desc_addr = ctx->enc_cd_paddr;
461 	enc_cd->aes.cipher_config.val = QAT_AES_HW_CONFIG_ENC(alg, mode);
462 }
463 
qat_alg_xts_reverse_key(const u8 * key_forward,unsigned int keylen,u8 * key_reverse)464 static void qat_alg_xts_reverse_key(const u8 *key_forward, unsigned int keylen,
465 				    u8 *key_reverse)
466 {
467 	struct crypto_aes_ctx aes_expanded;
468 	int nrounds;
469 	u8 *key;
470 
471 	aes_expandkey(&aes_expanded, key_forward, keylen);
472 	if (keylen == AES_KEYSIZE_128) {
473 		nrounds = 10;
474 		key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
475 		memcpy(key_reverse, key, AES_BLOCK_SIZE);
476 	} else {
477 		/* AES_KEYSIZE_256 */
478 		nrounds = 14;
479 		key = (u8 *)aes_expanded.key_enc + (AES_BLOCK_SIZE * nrounds);
480 		memcpy(key_reverse, key, AES_BLOCK_SIZE);
481 		memcpy(key_reverse + AES_BLOCK_SIZE, key - AES_BLOCK_SIZE,
482 		       AES_BLOCK_SIZE);
483 	}
484 }
485 
qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx * ctx,int alg,const u8 * key,unsigned int keylen,int mode)486 static void qat_alg_skcipher_init_dec(struct qat_alg_skcipher_ctx *ctx,
487 				      int alg, const u8 *key,
488 				      unsigned int keylen, int mode)
489 {
490 	struct icp_qat_hw_cipher_algo_blk *dec_cd = ctx->dec_cd;
491 	struct icp_qat_fw_la_bulk_req *req = &ctx->dec_fw_req;
492 	struct icp_qat_fw_comn_req_hdr_cd_pars *cd_pars = &req->cd_pars;
493 	bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
494 
495 	qat_alg_skcipher_init_com(ctx, req, dec_cd, key, keylen);
496 	cd_pars->u.s.content_desc_addr = ctx->dec_cd_paddr;
497 
498 	if (aes_v2_capable && mode == ICP_QAT_HW_CIPHER_XTS_MODE) {
499 		/* Key reversing not supported, set no convert */
500 		dec_cd->aes.cipher_config.val =
501 				QAT_AES_HW_CONFIG_DEC_NO_CONV(alg, mode);
502 
503 		/* In-place key reversal */
504 		qat_alg_xts_reverse_key(dec_cd->ucs_aes.key, keylen / 2,
505 					dec_cd->ucs_aes.key);
506 	} else if (mode != ICP_QAT_HW_CIPHER_CTR_MODE) {
507 		dec_cd->aes.cipher_config.val =
508 					QAT_AES_HW_CONFIG_DEC(alg, mode);
509 	} else {
510 		dec_cd->aes.cipher_config.val =
511 					QAT_AES_HW_CONFIG_ENC(alg, mode);
512 	}
513 }
514 
qat_alg_validate_key(int key_len,int * alg,int mode)515 static int qat_alg_validate_key(int key_len, int *alg, int mode)
516 {
517 	if (mode != ICP_QAT_HW_CIPHER_XTS_MODE) {
518 		switch (key_len) {
519 		case AES_KEYSIZE_128:
520 			*alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
521 			break;
522 		case AES_KEYSIZE_192:
523 			*alg = ICP_QAT_HW_CIPHER_ALGO_AES192;
524 			break;
525 		case AES_KEYSIZE_256:
526 			*alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
527 			break;
528 		default:
529 			return -EINVAL;
530 		}
531 	} else {
532 		switch (key_len) {
533 		case AES_KEYSIZE_128 << 1:
534 			*alg = ICP_QAT_HW_CIPHER_ALGO_AES128;
535 			break;
536 		case AES_KEYSIZE_256 << 1:
537 			*alg = ICP_QAT_HW_CIPHER_ALGO_AES256;
538 			break;
539 		default:
540 			return -EINVAL;
541 		}
542 	}
543 	return 0;
544 }
545 
qat_alg_aead_init_sessions(struct crypto_aead * tfm,const u8 * key,unsigned int keylen,int mode)546 static int qat_alg_aead_init_sessions(struct crypto_aead *tfm, const u8 *key,
547 				      unsigned int keylen,  int mode)
548 {
549 	struct crypto_authenc_keys keys;
550 	int alg;
551 
552 	if (crypto_authenc_extractkeys(&keys, key, keylen))
553 		goto bad_key;
554 
555 	if (qat_alg_validate_key(keys.enckeylen, &alg, mode))
556 		goto bad_key;
557 
558 	if (qat_alg_aead_init_enc_session(tfm, alg, &keys, mode))
559 		goto error;
560 
561 	if (qat_alg_aead_init_dec_session(tfm, alg, &keys, mode))
562 		goto error;
563 
564 	memzero_explicit(&keys, sizeof(keys));
565 	return 0;
566 bad_key:
567 	memzero_explicit(&keys, sizeof(keys));
568 	return -EINVAL;
569 error:
570 	memzero_explicit(&keys, sizeof(keys));
571 	return -EFAULT;
572 }
573 
qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx * ctx,const u8 * key,unsigned int keylen,int mode)574 static int qat_alg_skcipher_init_sessions(struct qat_alg_skcipher_ctx *ctx,
575 					  const u8 *key,
576 					  unsigned int keylen,
577 					  int mode)
578 {
579 	int alg;
580 
581 	if (qat_alg_validate_key(keylen, &alg, mode))
582 		return -EINVAL;
583 
584 	qat_alg_skcipher_init_enc(ctx, alg, key, keylen, mode);
585 	qat_alg_skcipher_init_dec(ctx, alg, key, keylen, mode);
586 	return 0;
587 }
588 
qat_alg_aead_rekey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)589 static int qat_alg_aead_rekey(struct crypto_aead *tfm, const u8 *key,
590 			      unsigned int keylen)
591 {
592 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
593 
594 	memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
595 	memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
596 	memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
597 	memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
598 
599 	return qat_alg_aead_init_sessions(tfm, key, keylen,
600 					  ICP_QAT_HW_CIPHER_CBC_MODE);
601 }
602 
qat_alg_aead_newkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)603 static int qat_alg_aead_newkey(struct crypto_aead *tfm, const u8 *key,
604 			       unsigned int keylen)
605 {
606 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
607 	struct qat_crypto_instance *inst = NULL;
608 	int node = numa_node_id();
609 	struct device *dev;
610 	int ret;
611 
612 	inst = qat_crypto_get_instance_node(node);
613 	if (!inst)
614 		return -EINVAL;
615 	dev = &GET_DEV(inst->accel_dev);
616 	ctx->inst = inst;
617 	ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
618 					 &ctx->enc_cd_paddr,
619 					 GFP_ATOMIC);
620 	if (!ctx->enc_cd) {
621 		ret = -ENOMEM;
622 		goto out_free_inst;
623 	}
624 	ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
625 					 &ctx->dec_cd_paddr,
626 					 GFP_ATOMIC);
627 	if (!ctx->dec_cd) {
628 		ret = -ENOMEM;
629 		goto out_free_enc;
630 	}
631 
632 	ret = qat_alg_aead_init_sessions(tfm, key, keylen,
633 					 ICP_QAT_HW_CIPHER_CBC_MODE);
634 	if (ret)
635 		goto out_free_all;
636 
637 	return 0;
638 
639 out_free_all:
640 	memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
641 	dma_free_coherent(dev, sizeof(struct qat_alg_cd),
642 			  ctx->dec_cd, ctx->dec_cd_paddr);
643 	ctx->dec_cd = NULL;
644 out_free_enc:
645 	memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
646 	dma_free_coherent(dev, sizeof(struct qat_alg_cd),
647 			  ctx->enc_cd, ctx->enc_cd_paddr);
648 	ctx->enc_cd = NULL;
649 out_free_inst:
650 	ctx->inst = NULL;
651 	qat_crypto_put_instance(inst);
652 	return ret;
653 }
654 
qat_alg_aead_setkey(struct crypto_aead * tfm,const u8 * key,unsigned int keylen)655 static int qat_alg_aead_setkey(struct crypto_aead *tfm, const u8 *key,
656 			       unsigned int keylen)
657 {
658 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
659 
660 	if (ctx->enc_cd)
661 		return qat_alg_aead_rekey(tfm, key, keylen);
662 	else
663 		return qat_alg_aead_newkey(tfm, key, keylen);
664 }
665 
qat_aead_alg_callback(struct icp_qat_fw_la_resp * qat_resp,struct qat_crypto_request * qat_req)666 static void qat_aead_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
667 				  struct qat_crypto_request *qat_req)
668 {
669 	struct qat_alg_aead_ctx *ctx = qat_req->aead_ctx;
670 	struct qat_crypto_instance *inst = ctx->inst;
671 	struct aead_request *areq = qat_req->aead_req;
672 	u8 stat_filed = qat_resp->comn_resp.comn_status;
673 	int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
674 
675 	qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);
676 	if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
677 		res = -EBADMSG;
678 	aead_request_complete(areq, res);
679 }
680 
qat_alg_update_iv_ctr_mode(struct qat_crypto_request * qat_req)681 static void qat_alg_update_iv_ctr_mode(struct qat_crypto_request *qat_req)
682 {
683 	struct skcipher_request *sreq = qat_req->skcipher_req;
684 	u64 iv_lo_prev;
685 	u64 iv_lo;
686 	u64 iv_hi;
687 
688 	memcpy(qat_req->iv, sreq->iv, AES_BLOCK_SIZE);
689 
690 	iv_lo = be64_to_cpu(qat_req->iv_lo);
691 	iv_hi = be64_to_cpu(qat_req->iv_hi);
692 
693 	iv_lo_prev = iv_lo;
694 	iv_lo += DIV_ROUND_UP(sreq->cryptlen, AES_BLOCK_SIZE);
695 	if (iv_lo < iv_lo_prev)
696 		iv_hi++;
697 
698 	qat_req->iv_lo = cpu_to_be64(iv_lo);
699 	qat_req->iv_hi = cpu_to_be64(iv_hi);
700 }
701 
qat_alg_update_iv_cbc_mode(struct qat_crypto_request * qat_req)702 static void qat_alg_update_iv_cbc_mode(struct qat_crypto_request *qat_req)
703 {
704 	struct skcipher_request *sreq = qat_req->skcipher_req;
705 	int offset = sreq->cryptlen - AES_BLOCK_SIZE;
706 	struct scatterlist *sgl;
707 
708 	if (qat_req->encryption)
709 		sgl = sreq->dst;
710 	else
711 		sgl = sreq->src;
712 
713 	scatterwalk_map_and_copy(qat_req->iv, sgl, offset, AES_BLOCK_SIZE, 0);
714 }
715 
qat_alg_update_iv(struct qat_crypto_request * qat_req)716 static void qat_alg_update_iv(struct qat_crypto_request *qat_req)
717 {
718 	struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
719 	struct device *dev = &GET_DEV(ctx->inst->accel_dev);
720 
721 	switch (ctx->mode) {
722 	case ICP_QAT_HW_CIPHER_CTR_MODE:
723 		qat_alg_update_iv_ctr_mode(qat_req);
724 		break;
725 	case ICP_QAT_HW_CIPHER_CBC_MODE:
726 		qat_alg_update_iv_cbc_mode(qat_req);
727 		break;
728 	case ICP_QAT_HW_CIPHER_XTS_MODE:
729 		break;
730 	default:
731 		dev_warn(dev, "Unsupported IV update for cipher mode %d\n",
732 			 ctx->mode);
733 	}
734 }
735 
qat_skcipher_alg_callback(struct icp_qat_fw_la_resp * qat_resp,struct qat_crypto_request * qat_req)736 static void qat_skcipher_alg_callback(struct icp_qat_fw_la_resp *qat_resp,
737 				      struct qat_crypto_request *qat_req)
738 {
739 	struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
740 	struct qat_crypto_instance *inst = ctx->inst;
741 	struct skcipher_request *sreq = qat_req->skcipher_req;
742 	u8 stat_filed = qat_resp->comn_resp.comn_status;
743 	int res = 0, qat_res = ICP_QAT_FW_COMN_RESP_CRYPTO_STAT_GET(stat_filed);
744 
745 	qat_bl_free_bufl(inst->accel_dev, &qat_req->buf);
746 	if (unlikely(qat_res != ICP_QAT_FW_COMN_STATUS_FLAG_OK))
747 		res = -EINVAL;
748 
749 	if (qat_req->encryption)
750 		qat_alg_update_iv(qat_req);
751 
752 	memcpy(sreq->iv, qat_req->iv, AES_BLOCK_SIZE);
753 
754 	skcipher_request_complete(sreq, res);
755 }
756 
qat_alg_callback(void * resp)757 void qat_alg_callback(void *resp)
758 {
759 	struct icp_qat_fw_la_resp *qat_resp = resp;
760 	struct qat_crypto_request *qat_req =
761 				(void *)(__force long)qat_resp->opaque_data;
762 	struct qat_instance_backlog *backlog = qat_req->alg_req.backlog;
763 
764 	qat_req->cb(qat_resp, qat_req);
765 
766 	qat_alg_send_backlog(backlog);
767 }
768 
qat_alg_send_sym_message(struct qat_crypto_request * qat_req,struct qat_crypto_instance * inst,struct crypto_async_request * base)769 static int qat_alg_send_sym_message(struct qat_crypto_request *qat_req,
770 				    struct qat_crypto_instance *inst,
771 				    struct crypto_async_request *base)
772 {
773 	struct qat_alg_req *alg_req = &qat_req->alg_req;
774 
775 	alg_req->fw_req = (u32 *)&qat_req->req;
776 	alg_req->tx_ring = inst->sym_tx;
777 	alg_req->base = base;
778 	alg_req->backlog = &inst->backlog;
779 
780 	return qat_alg_send_message(alg_req);
781 }
782 
qat_alg_aead_dec(struct aead_request * areq)783 static int qat_alg_aead_dec(struct aead_request *areq)
784 {
785 	struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
786 	struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
787 	struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
788 	struct qat_crypto_request *qat_req = aead_request_ctx(areq);
789 	struct icp_qat_fw_la_cipher_req_params *cipher_param;
790 	struct icp_qat_fw_la_auth_req_params *auth_param;
791 	struct icp_qat_fw_la_bulk_req *msg;
792 	int digst_size = crypto_aead_authsize(aead_tfm);
793 	gfp_t f = qat_algs_alloc_flags(&areq->base);
794 	int ret;
795 	u32 cipher_len;
796 
797 	cipher_len = areq->cryptlen - digst_size;
798 	if (cipher_len % AES_BLOCK_SIZE != 0)
799 		return -EINVAL;
800 
801 	ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
802 				 &qat_req->buf, NULL, f);
803 	if (unlikely(ret))
804 		return ret;
805 
806 	msg = &qat_req->req;
807 	*msg = ctx->dec_fw_req;
808 	qat_req->aead_ctx = ctx;
809 	qat_req->aead_req = areq;
810 	qat_req->cb = qat_aead_alg_callback;
811 	qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
812 	qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
813 	qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
814 	cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
815 	cipher_param->cipher_length = cipher_len;
816 	cipher_param->cipher_offset = areq->assoclen;
817 	memcpy(cipher_param->u.cipher_IV_array, areq->iv, AES_BLOCK_SIZE);
818 	auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
819 	auth_param->auth_off = 0;
820 	auth_param->auth_len = areq->assoclen + cipher_param->cipher_length;
821 
822 	ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
823 	if (ret == -ENOSPC)
824 		qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
825 
826 	return ret;
827 }
828 
qat_alg_aead_enc(struct aead_request * areq)829 static int qat_alg_aead_enc(struct aead_request *areq)
830 {
831 	struct crypto_aead *aead_tfm = crypto_aead_reqtfm(areq);
832 	struct crypto_tfm *tfm = crypto_aead_tfm(aead_tfm);
833 	struct qat_alg_aead_ctx *ctx = crypto_tfm_ctx(tfm);
834 	struct qat_crypto_request *qat_req = aead_request_ctx(areq);
835 	struct icp_qat_fw_la_cipher_req_params *cipher_param;
836 	struct icp_qat_fw_la_auth_req_params *auth_param;
837 	gfp_t f = qat_algs_alloc_flags(&areq->base);
838 	struct icp_qat_fw_la_bulk_req *msg;
839 	u8 *iv = areq->iv;
840 	int ret;
841 
842 	if (areq->cryptlen % AES_BLOCK_SIZE != 0)
843 		return -EINVAL;
844 
845 	ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, areq->src, areq->dst,
846 				 &qat_req->buf, NULL, f);
847 	if (unlikely(ret))
848 		return ret;
849 
850 	msg = &qat_req->req;
851 	*msg = ctx->enc_fw_req;
852 	qat_req->aead_ctx = ctx;
853 	qat_req->aead_req = areq;
854 	qat_req->cb = qat_aead_alg_callback;
855 	qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
856 	qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
857 	qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
858 	cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
859 	auth_param = (void *)((u8 *)cipher_param + sizeof(*cipher_param));
860 
861 	memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
862 	cipher_param->cipher_length = areq->cryptlen;
863 	cipher_param->cipher_offset = areq->assoclen;
864 
865 	auth_param->auth_off = 0;
866 	auth_param->auth_len = areq->assoclen + areq->cryptlen;
867 
868 	ret = qat_alg_send_sym_message(qat_req, ctx->inst, &areq->base);
869 	if (ret == -ENOSPC)
870 		qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
871 
872 	return ret;
873 }
874 
qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx * ctx,const u8 * key,unsigned int keylen,int mode)875 static int qat_alg_skcipher_rekey(struct qat_alg_skcipher_ctx *ctx,
876 				  const u8 *key, unsigned int keylen,
877 				  int mode)
878 {
879 	memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
880 	memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
881 	memset(&ctx->enc_fw_req, 0, sizeof(ctx->enc_fw_req));
882 	memset(&ctx->dec_fw_req, 0, sizeof(ctx->dec_fw_req));
883 
884 	return qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
885 }
886 
qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx * ctx,const u8 * key,unsigned int keylen,int mode)887 static int qat_alg_skcipher_newkey(struct qat_alg_skcipher_ctx *ctx,
888 				   const u8 *key, unsigned int keylen,
889 				   int mode)
890 {
891 	struct qat_crypto_instance *inst = NULL;
892 	struct device *dev;
893 	int node = numa_node_id();
894 	int ret;
895 
896 	inst = qat_crypto_get_instance_node(node);
897 	if (!inst)
898 		return -EINVAL;
899 	dev = &GET_DEV(inst->accel_dev);
900 	ctx->inst = inst;
901 	ctx->enc_cd = dma_alloc_coherent(dev, sizeof(*ctx->enc_cd),
902 					 &ctx->enc_cd_paddr,
903 					 GFP_ATOMIC);
904 	if (!ctx->enc_cd) {
905 		ret = -ENOMEM;
906 		goto out_free_instance;
907 	}
908 	ctx->dec_cd = dma_alloc_coherent(dev, sizeof(*ctx->dec_cd),
909 					 &ctx->dec_cd_paddr,
910 					 GFP_ATOMIC);
911 	if (!ctx->dec_cd) {
912 		ret = -ENOMEM;
913 		goto out_free_enc;
914 	}
915 
916 	ret = qat_alg_skcipher_init_sessions(ctx, key, keylen, mode);
917 	if (ret)
918 		goto out_free_all;
919 
920 	return 0;
921 
922 out_free_all:
923 	memset(ctx->dec_cd, 0, sizeof(*ctx->dec_cd));
924 	dma_free_coherent(dev, sizeof(*ctx->dec_cd),
925 			  ctx->dec_cd, ctx->dec_cd_paddr);
926 	ctx->dec_cd = NULL;
927 out_free_enc:
928 	memset(ctx->enc_cd, 0, sizeof(*ctx->enc_cd));
929 	dma_free_coherent(dev, sizeof(*ctx->enc_cd),
930 			  ctx->enc_cd, ctx->enc_cd_paddr);
931 	ctx->enc_cd = NULL;
932 out_free_instance:
933 	ctx->inst = NULL;
934 	qat_crypto_put_instance(inst);
935 	return ret;
936 }
937 
qat_alg_skcipher_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen,int mode)938 static int qat_alg_skcipher_setkey(struct crypto_skcipher *tfm,
939 				   const u8 *key, unsigned int keylen,
940 				   int mode)
941 {
942 	struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
943 
944 	ctx->mode = mode;
945 
946 	if (ctx->enc_cd)
947 		return qat_alg_skcipher_rekey(ctx, key, keylen, mode);
948 	else
949 		return qat_alg_skcipher_newkey(ctx, key, keylen, mode);
950 }
951 
qat_alg_skcipher_cbc_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)952 static int qat_alg_skcipher_cbc_setkey(struct crypto_skcipher *tfm,
953 				       const u8 *key, unsigned int keylen)
954 {
955 	return qat_alg_skcipher_setkey(tfm, key, keylen,
956 				       ICP_QAT_HW_CIPHER_CBC_MODE);
957 }
958 
qat_alg_skcipher_ctr_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)959 static int qat_alg_skcipher_ctr_setkey(struct crypto_skcipher *tfm,
960 				       const u8 *key, unsigned int keylen)
961 {
962 	return qat_alg_skcipher_setkey(tfm, key, keylen,
963 				       ICP_QAT_HW_CIPHER_CTR_MODE);
964 }
965 
qat_alg_skcipher_xts_setkey(struct crypto_skcipher * tfm,const u8 * key,unsigned int keylen)966 static int qat_alg_skcipher_xts_setkey(struct crypto_skcipher *tfm,
967 				       const u8 *key, unsigned int keylen)
968 {
969 	struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
970 	int ret;
971 
972 	ret = xts_verify_key(tfm, key, keylen);
973 	if (ret)
974 		return ret;
975 
976 	if (keylen >> 1 == AES_KEYSIZE_192) {
977 		ret = crypto_skcipher_setkey(ctx->ftfm, key, keylen);
978 		if (ret)
979 			return ret;
980 
981 		ctx->fallback = true;
982 
983 		return 0;
984 	}
985 
986 	ctx->fallback = false;
987 
988 	ret = qat_alg_skcipher_setkey(tfm, key, keylen,
989 				      ICP_QAT_HW_CIPHER_XTS_MODE);
990 	if (ret)
991 		return ret;
992 
993 	if (HW_CAP_AES_V2(ctx->inst->accel_dev))
994 		ret = crypto_cipher_setkey(ctx->tweak, key + (keylen / 2),
995 					   keylen / 2);
996 
997 	return ret;
998 }
999 
qat_alg_set_req_iv(struct qat_crypto_request * qat_req)1000 static void qat_alg_set_req_iv(struct qat_crypto_request *qat_req)
1001 {
1002 	struct icp_qat_fw_la_cipher_req_params *cipher_param;
1003 	struct qat_alg_skcipher_ctx *ctx = qat_req->skcipher_ctx;
1004 	bool aes_v2_capable = HW_CAP_AES_V2(ctx->inst->accel_dev);
1005 	u8 *iv = qat_req->skcipher_req->iv;
1006 
1007 	cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1008 
1009 	if (aes_v2_capable && ctx->mode == ICP_QAT_HW_CIPHER_XTS_MODE)
1010 		crypto_cipher_encrypt_one(ctx->tweak,
1011 					  (u8 *)cipher_param->u.cipher_IV_array,
1012 					  iv);
1013 	else
1014 		memcpy(cipher_param->u.cipher_IV_array, iv, AES_BLOCK_SIZE);
1015 }
1016 
qat_alg_skcipher_encrypt(struct skcipher_request * req)1017 static int qat_alg_skcipher_encrypt(struct skcipher_request *req)
1018 {
1019 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1020 	struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1021 	struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1022 	struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1023 	struct icp_qat_fw_la_cipher_req_params *cipher_param;
1024 	gfp_t f = qat_algs_alloc_flags(&req->base);
1025 	struct icp_qat_fw_la_bulk_req *msg;
1026 	int ret;
1027 
1028 	if (req->cryptlen == 0)
1029 		return 0;
1030 
1031 	ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst,
1032 				 &qat_req->buf, NULL, f);
1033 	if (unlikely(ret))
1034 		return ret;
1035 
1036 	msg = &qat_req->req;
1037 	*msg = ctx->enc_fw_req;
1038 	qat_req->skcipher_ctx = ctx;
1039 	qat_req->skcipher_req = req;
1040 	qat_req->cb = qat_skcipher_alg_callback;
1041 	qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1042 	qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1043 	qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1044 	qat_req->encryption = true;
1045 	cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1046 	cipher_param->cipher_length = req->cryptlen;
1047 	cipher_param->cipher_offset = 0;
1048 
1049 	qat_alg_set_req_iv(qat_req);
1050 
1051 	ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
1052 	if (ret == -ENOSPC)
1053 		qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
1054 
1055 	return ret;
1056 }
1057 
qat_alg_skcipher_blk_encrypt(struct skcipher_request * req)1058 static int qat_alg_skcipher_blk_encrypt(struct skcipher_request *req)
1059 {
1060 	if (req->cryptlen % AES_BLOCK_SIZE != 0)
1061 		return -EINVAL;
1062 
1063 	return qat_alg_skcipher_encrypt(req);
1064 }
1065 
qat_alg_skcipher_xts_encrypt(struct skcipher_request * req)1066 static int qat_alg_skcipher_xts_encrypt(struct skcipher_request *req)
1067 {
1068 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1069 	struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);
1070 	struct skcipher_request *nreq = skcipher_request_ctx(req);
1071 
1072 	if (req->cryptlen < XTS_BLOCK_SIZE)
1073 		return -EINVAL;
1074 
1075 	if (ctx->fallback) {
1076 		memcpy(nreq, req, sizeof(*req));
1077 		skcipher_request_set_tfm(nreq, ctx->ftfm);
1078 		return crypto_skcipher_encrypt(nreq);
1079 	}
1080 
1081 	return qat_alg_skcipher_encrypt(req);
1082 }
1083 
qat_alg_skcipher_decrypt(struct skcipher_request * req)1084 static int qat_alg_skcipher_decrypt(struct skcipher_request *req)
1085 {
1086 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1087 	struct crypto_tfm *tfm = crypto_skcipher_tfm(stfm);
1088 	struct qat_alg_skcipher_ctx *ctx = crypto_tfm_ctx(tfm);
1089 	struct qat_crypto_request *qat_req = skcipher_request_ctx(req);
1090 	struct icp_qat_fw_la_cipher_req_params *cipher_param;
1091 	gfp_t f = qat_algs_alloc_flags(&req->base);
1092 	struct icp_qat_fw_la_bulk_req *msg;
1093 	int ret;
1094 
1095 	if (req->cryptlen == 0)
1096 		return 0;
1097 
1098 	ret = qat_bl_sgl_to_bufl(ctx->inst->accel_dev, req->src, req->dst,
1099 				 &qat_req->buf, NULL, f);
1100 	if (unlikely(ret))
1101 		return ret;
1102 
1103 	msg = &qat_req->req;
1104 	*msg = ctx->dec_fw_req;
1105 	qat_req->skcipher_ctx = ctx;
1106 	qat_req->skcipher_req = req;
1107 	qat_req->cb = qat_skcipher_alg_callback;
1108 	qat_req->req.comn_mid.opaque_data = (u64)(__force long)qat_req;
1109 	qat_req->req.comn_mid.src_data_addr = qat_req->buf.blp;
1110 	qat_req->req.comn_mid.dest_data_addr = qat_req->buf.bloutp;
1111 	qat_req->encryption = false;
1112 	cipher_param = (void *)&qat_req->req.serv_specif_rqpars;
1113 	cipher_param->cipher_length = req->cryptlen;
1114 	cipher_param->cipher_offset = 0;
1115 
1116 	qat_alg_set_req_iv(qat_req);
1117 	qat_alg_update_iv(qat_req);
1118 
1119 	ret = qat_alg_send_sym_message(qat_req, ctx->inst, &req->base);
1120 	if (ret == -ENOSPC)
1121 		qat_bl_free_bufl(ctx->inst->accel_dev, &qat_req->buf);
1122 
1123 	return ret;
1124 }
1125 
qat_alg_skcipher_blk_decrypt(struct skcipher_request * req)1126 static int qat_alg_skcipher_blk_decrypt(struct skcipher_request *req)
1127 {
1128 	if (req->cryptlen % AES_BLOCK_SIZE != 0)
1129 		return -EINVAL;
1130 
1131 	return qat_alg_skcipher_decrypt(req);
1132 }
1133 
qat_alg_skcipher_xts_decrypt(struct skcipher_request * req)1134 static int qat_alg_skcipher_xts_decrypt(struct skcipher_request *req)
1135 {
1136 	struct crypto_skcipher *stfm = crypto_skcipher_reqtfm(req);
1137 	struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(stfm);
1138 	struct skcipher_request *nreq = skcipher_request_ctx(req);
1139 
1140 	if (req->cryptlen < XTS_BLOCK_SIZE)
1141 		return -EINVAL;
1142 
1143 	if (ctx->fallback) {
1144 		memcpy(nreq, req, sizeof(*req));
1145 		skcipher_request_set_tfm(nreq, ctx->ftfm);
1146 		return crypto_skcipher_decrypt(nreq);
1147 	}
1148 
1149 	return qat_alg_skcipher_decrypt(req);
1150 }
1151 
qat_alg_aead_init(struct crypto_aead * tfm,enum icp_qat_hw_auth_algo hash,const char * hash_name)1152 static int qat_alg_aead_init(struct crypto_aead *tfm,
1153 			     enum icp_qat_hw_auth_algo hash,
1154 			     const char *hash_name)
1155 {
1156 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1157 
1158 	ctx->hash_tfm = crypto_alloc_shash(hash_name, 0, 0);
1159 	if (IS_ERR(ctx->hash_tfm))
1160 		return PTR_ERR(ctx->hash_tfm);
1161 	ctx->qat_hash_alg = hash;
1162 	crypto_aead_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1163 	return 0;
1164 }
1165 
qat_alg_aead_sha1_init(struct crypto_aead * tfm)1166 static int qat_alg_aead_sha1_init(struct crypto_aead *tfm)
1167 {
1168 	return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA1, "sha1");
1169 }
1170 
qat_alg_aead_sha256_init(struct crypto_aead * tfm)1171 static int qat_alg_aead_sha256_init(struct crypto_aead *tfm)
1172 {
1173 	return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA256, "sha256");
1174 }
1175 
qat_alg_aead_sha512_init(struct crypto_aead * tfm)1176 static int qat_alg_aead_sha512_init(struct crypto_aead *tfm)
1177 {
1178 	return qat_alg_aead_init(tfm, ICP_QAT_HW_AUTH_ALGO_SHA512, "sha512");
1179 }
1180 
qat_alg_aead_exit(struct crypto_aead * tfm)1181 static void qat_alg_aead_exit(struct crypto_aead *tfm)
1182 {
1183 	struct qat_alg_aead_ctx *ctx = crypto_aead_ctx(tfm);
1184 	struct qat_crypto_instance *inst = ctx->inst;
1185 	struct device *dev;
1186 
1187 	crypto_free_shash(ctx->hash_tfm);
1188 
1189 	if (!inst)
1190 		return;
1191 
1192 	dev = &GET_DEV(inst->accel_dev);
1193 	if (ctx->enc_cd) {
1194 		memset(ctx->enc_cd, 0, sizeof(struct qat_alg_cd));
1195 		dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1196 				  ctx->enc_cd, ctx->enc_cd_paddr);
1197 	}
1198 	if (ctx->dec_cd) {
1199 		memset(ctx->dec_cd, 0, sizeof(struct qat_alg_cd));
1200 		dma_free_coherent(dev, sizeof(struct qat_alg_cd),
1201 				  ctx->dec_cd, ctx->dec_cd_paddr);
1202 	}
1203 	qat_crypto_put_instance(inst);
1204 }
1205 
qat_alg_skcipher_init_tfm(struct crypto_skcipher * tfm)1206 static int qat_alg_skcipher_init_tfm(struct crypto_skcipher *tfm)
1207 {
1208 	crypto_skcipher_set_reqsize(tfm, sizeof(struct qat_crypto_request));
1209 	return 0;
1210 }
1211 
qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher * tfm)1212 static int qat_alg_skcipher_init_xts_tfm(struct crypto_skcipher *tfm)
1213 {
1214 	struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1215 	int reqsize;
1216 
1217 	ctx->ftfm = crypto_alloc_skcipher("xts(aes)", 0,
1218 					  CRYPTO_ALG_NEED_FALLBACK);
1219 	if (IS_ERR(ctx->ftfm))
1220 		return PTR_ERR(ctx->ftfm);
1221 
1222 	ctx->tweak = crypto_alloc_cipher("aes", 0, 0);
1223 	if (IS_ERR(ctx->tweak)) {
1224 		crypto_free_skcipher(ctx->ftfm);
1225 		return PTR_ERR(ctx->tweak);
1226 	}
1227 
1228 	reqsize = max(sizeof(struct qat_crypto_request),
1229 		      sizeof(struct skcipher_request) +
1230 		      crypto_skcipher_reqsize(ctx->ftfm));
1231 	crypto_skcipher_set_reqsize(tfm, reqsize);
1232 
1233 	return 0;
1234 }
1235 
qat_alg_skcipher_exit_tfm(struct crypto_skcipher * tfm)1236 static void qat_alg_skcipher_exit_tfm(struct crypto_skcipher *tfm)
1237 {
1238 	struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1239 	struct qat_crypto_instance *inst = ctx->inst;
1240 	struct device *dev;
1241 
1242 	if (!inst)
1243 		return;
1244 
1245 	dev = &GET_DEV(inst->accel_dev);
1246 	if (ctx->enc_cd) {
1247 		memset(ctx->enc_cd, 0,
1248 		       sizeof(struct icp_qat_hw_cipher_algo_blk));
1249 		dma_free_coherent(dev,
1250 				  sizeof(struct icp_qat_hw_cipher_algo_blk),
1251 				  ctx->enc_cd, ctx->enc_cd_paddr);
1252 	}
1253 	if (ctx->dec_cd) {
1254 		memset(ctx->dec_cd, 0,
1255 		       sizeof(struct icp_qat_hw_cipher_algo_blk));
1256 		dma_free_coherent(dev,
1257 				  sizeof(struct icp_qat_hw_cipher_algo_blk),
1258 				  ctx->dec_cd, ctx->dec_cd_paddr);
1259 	}
1260 	qat_crypto_put_instance(inst);
1261 }
1262 
qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher * tfm)1263 static void qat_alg_skcipher_exit_xts_tfm(struct crypto_skcipher *tfm)
1264 {
1265 	struct qat_alg_skcipher_ctx *ctx = crypto_skcipher_ctx(tfm);
1266 
1267 	if (ctx->ftfm)
1268 		crypto_free_skcipher(ctx->ftfm);
1269 
1270 	if (ctx->tweak)
1271 		crypto_free_cipher(ctx->tweak);
1272 
1273 	qat_alg_skcipher_exit_tfm(tfm);
1274 }
1275 
1276 static struct aead_alg qat_aeads[] = { {
1277 	.base = {
1278 		.cra_name = "authenc(hmac(sha1),cbc(aes))",
1279 		.cra_driver_name = "qat_aes_cbc_hmac_sha1",
1280 		.cra_priority = 4001,
1281 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1282 		.cra_blocksize = AES_BLOCK_SIZE,
1283 		.cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1284 		.cra_module = THIS_MODULE,
1285 	},
1286 	.init = qat_alg_aead_sha1_init,
1287 	.exit = qat_alg_aead_exit,
1288 	.setkey = qat_alg_aead_setkey,
1289 	.decrypt = qat_alg_aead_dec,
1290 	.encrypt = qat_alg_aead_enc,
1291 	.ivsize = AES_BLOCK_SIZE,
1292 	.maxauthsize = SHA1_DIGEST_SIZE,
1293 }, {
1294 	.base = {
1295 		.cra_name = "authenc(hmac(sha256),cbc(aes))",
1296 		.cra_driver_name = "qat_aes_cbc_hmac_sha256",
1297 		.cra_priority = 4001,
1298 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1299 		.cra_blocksize = AES_BLOCK_SIZE,
1300 		.cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1301 		.cra_module = THIS_MODULE,
1302 	},
1303 	.init = qat_alg_aead_sha256_init,
1304 	.exit = qat_alg_aead_exit,
1305 	.setkey = qat_alg_aead_setkey,
1306 	.decrypt = qat_alg_aead_dec,
1307 	.encrypt = qat_alg_aead_enc,
1308 	.ivsize = AES_BLOCK_SIZE,
1309 	.maxauthsize = SHA256_DIGEST_SIZE,
1310 }, {
1311 	.base = {
1312 		.cra_name = "authenc(hmac(sha512),cbc(aes))",
1313 		.cra_driver_name = "qat_aes_cbc_hmac_sha512",
1314 		.cra_priority = 4001,
1315 		.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1316 		.cra_blocksize = AES_BLOCK_SIZE,
1317 		.cra_ctxsize = sizeof(struct qat_alg_aead_ctx),
1318 		.cra_module = THIS_MODULE,
1319 	},
1320 	.init = qat_alg_aead_sha512_init,
1321 	.exit = qat_alg_aead_exit,
1322 	.setkey = qat_alg_aead_setkey,
1323 	.decrypt = qat_alg_aead_dec,
1324 	.encrypt = qat_alg_aead_enc,
1325 	.ivsize = AES_BLOCK_SIZE,
1326 	.maxauthsize = SHA512_DIGEST_SIZE,
1327 } };
1328 
1329 static struct skcipher_alg qat_skciphers[] = { {
1330 	.base.cra_name = "cbc(aes)",
1331 	.base.cra_driver_name = "qat_aes_cbc",
1332 	.base.cra_priority = 4001,
1333 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1334 	.base.cra_blocksize = AES_BLOCK_SIZE,
1335 	.base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1336 	.base.cra_alignmask = 0,
1337 	.base.cra_module = THIS_MODULE,
1338 
1339 	.init = qat_alg_skcipher_init_tfm,
1340 	.exit = qat_alg_skcipher_exit_tfm,
1341 	.setkey = qat_alg_skcipher_cbc_setkey,
1342 	.decrypt = qat_alg_skcipher_blk_decrypt,
1343 	.encrypt = qat_alg_skcipher_blk_encrypt,
1344 	.min_keysize = AES_MIN_KEY_SIZE,
1345 	.max_keysize = AES_MAX_KEY_SIZE,
1346 	.ivsize = AES_BLOCK_SIZE,
1347 }, {
1348 	.base.cra_name = "ctr(aes)",
1349 	.base.cra_driver_name = "qat_aes_ctr",
1350 	.base.cra_priority = 4001,
1351 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,
1352 	.base.cra_blocksize = 1,
1353 	.base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1354 	.base.cra_alignmask = 0,
1355 	.base.cra_module = THIS_MODULE,
1356 
1357 	.init = qat_alg_skcipher_init_tfm,
1358 	.exit = qat_alg_skcipher_exit_tfm,
1359 	.setkey = qat_alg_skcipher_ctr_setkey,
1360 	.decrypt = qat_alg_skcipher_decrypt,
1361 	.encrypt = qat_alg_skcipher_encrypt,
1362 	.min_keysize = AES_MIN_KEY_SIZE,
1363 	.max_keysize = AES_MAX_KEY_SIZE,
1364 	.ivsize = AES_BLOCK_SIZE,
1365 }, {
1366 	.base.cra_name = "xts(aes)",
1367 	.base.cra_driver_name = "qat_aes_xts",
1368 	.base.cra_priority = 4001,
1369 	.base.cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_NEED_FALLBACK |
1370 			  CRYPTO_ALG_ALLOCATES_MEMORY,
1371 	.base.cra_blocksize = AES_BLOCK_SIZE,
1372 	.base.cra_ctxsize = sizeof(struct qat_alg_skcipher_ctx),
1373 	.base.cra_alignmask = 0,
1374 	.base.cra_module = THIS_MODULE,
1375 
1376 	.init = qat_alg_skcipher_init_xts_tfm,
1377 	.exit = qat_alg_skcipher_exit_xts_tfm,
1378 	.setkey = qat_alg_skcipher_xts_setkey,
1379 	.decrypt = qat_alg_skcipher_xts_decrypt,
1380 	.encrypt = qat_alg_skcipher_xts_encrypt,
1381 	.min_keysize = 2 * AES_MIN_KEY_SIZE,
1382 	.max_keysize = 2 * AES_MAX_KEY_SIZE,
1383 	.ivsize = AES_BLOCK_SIZE,
1384 } };
1385 
qat_algs_register(void)1386 int qat_algs_register(void)
1387 {
1388 	int ret = 0;
1389 
1390 	mutex_lock(&algs_lock);
1391 	if (++active_devs != 1)
1392 		goto unlock;
1393 
1394 	ret = crypto_register_skciphers(qat_skciphers,
1395 					ARRAY_SIZE(qat_skciphers));
1396 	if (ret)
1397 		goto unlock;
1398 
1399 	ret = crypto_register_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1400 	if (ret)
1401 		goto unreg_algs;
1402 
1403 unlock:
1404 	mutex_unlock(&algs_lock);
1405 	return ret;
1406 
1407 unreg_algs:
1408 	crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1409 	goto unlock;
1410 }
1411 
qat_algs_unregister(void)1412 void qat_algs_unregister(void)
1413 {
1414 	mutex_lock(&algs_lock);
1415 	if (--active_devs != 0)
1416 		goto unlock;
1417 
1418 	crypto_unregister_aeads(qat_aeads, ARRAY_SIZE(qat_aeads));
1419 	crypto_unregister_skciphers(qat_skciphers, ARRAY_SIZE(qat_skciphers));
1420 
1421 unlock:
1422 	mutex_unlock(&algs_lock);
1423 }
1424