1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (C) 2012-2019 ARM Limited (or its affiliates). */ 3 4 #include <crypto/internal/aead.h> 5 #include <crypto/authenc.h> 6 #include <crypto/scatterwalk.h> 7 #include <linux/dmapool.h> 8 #include <linux/dma-mapping.h> 9 10 #include "cc_buffer_mgr.h" 11 #include "cc_lli_defs.h" 12 #include "cc_cipher.h" 13 #include "cc_hash.h" 14 #include "cc_aead.h" 15 16 union buffer_array_entry { 17 struct scatterlist *sgl; 18 dma_addr_t buffer_dma; 19 }; 20 21 struct buffer_array { 22 unsigned int num_of_buffers; 23 union buffer_array_entry entry[MAX_NUM_OF_BUFFERS_IN_MLLI]; 24 unsigned int offset[MAX_NUM_OF_BUFFERS_IN_MLLI]; 25 int nents[MAX_NUM_OF_BUFFERS_IN_MLLI]; 26 int total_data_len[MAX_NUM_OF_BUFFERS_IN_MLLI]; 27 bool is_last[MAX_NUM_OF_BUFFERS_IN_MLLI]; 28 u32 *mlli_nents[MAX_NUM_OF_BUFFERS_IN_MLLI]; 29 }; 30 31 static inline char *cc_dma_buf_type(enum cc_req_dma_buf_type type) 32 { 33 switch (type) { 34 case CC_DMA_BUF_NULL: 35 return "BUF_NULL"; 36 case CC_DMA_BUF_DLLI: 37 return "BUF_DLLI"; 38 case CC_DMA_BUF_MLLI: 39 return "BUF_MLLI"; 40 default: 41 return "BUF_INVALID"; 42 } 43 } 44 45 /** 46 * cc_copy_mac() - Copy MAC to temporary location 47 * 48 * @dev: device object 49 * @req: aead request object 50 * @dir: [IN] copy from/to sgl 51 */ 52 static void cc_copy_mac(struct device *dev, struct aead_request *req, 53 enum cc_sg_cpy_direct dir) 54 { 55 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 56 u32 skip = req->assoclen + req->cryptlen; 57 58 cc_copy_sg_portion(dev, areq_ctx->backup_mac, req->src, 59 (skip - areq_ctx->req_authsize), skip, dir); 60 } 61 62 /** 63 * cc_get_sgl_nents() - Get scatterlist number of entries. 64 * 65 * @dev: Device object 66 * @sg_list: SG list 67 * @nbytes: [IN] Total SGL data bytes. 68 * @lbytes: [OUT] Returns the amount of bytes at the last entry 69 * 70 * Return: 71 * Number of entries in the scatterlist 72 */ 73 static unsigned int cc_get_sgl_nents(struct device *dev, 74 struct scatterlist *sg_list, 75 unsigned int nbytes, u32 *lbytes) 76 { 77 unsigned int nents = 0; 78 79 *lbytes = 0; 80 81 while (nbytes && sg_list) { 82 nents++; 83 /* get the number of bytes in the last entry */ 84 *lbytes = nbytes; 85 nbytes -= (sg_list->length > nbytes) ? 86 nbytes : sg_list->length; 87 sg_list = sg_next(sg_list); 88 } 89 90 dev_dbg(dev, "nents %d last bytes %d\n", nents, *lbytes); 91 return nents; 92 } 93 94 /** 95 * cc_copy_sg_portion() - Copy scatter list data, 96 * from to_skip to end, to dest and vice versa 97 * 98 * @dev: Device object 99 * @dest: Buffer to copy to/from 100 * @sg: SG list 101 * @to_skip: Number of bytes to skip before copying 102 * @end: Offset of last byte to copy 103 * @direct: Transfer direction (true == from SG list to buffer, false == from 104 * buffer to SG list) 105 */ 106 void cc_copy_sg_portion(struct device *dev, u8 *dest, struct scatterlist *sg, 107 u32 to_skip, u32 end, enum cc_sg_cpy_direct direct) 108 { 109 u32 nents; 110 111 nents = sg_nents_for_len(sg, end); 112 sg_copy_buffer(sg, nents, dest, (end - to_skip + 1), to_skip, 113 (direct == CC_SG_TO_BUF)); 114 } 115 116 static int cc_render_buff_to_mlli(struct device *dev, dma_addr_t buff_dma, 117 u32 buff_size, u32 *curr_nents, 118 u32 **mlli_entry_pp) 119 { 120 u32 *mlli_entry_p = *mlli_entry_pp; 121 u32 new_nents; 122 123 /* Verify there is no memory overflow*/ 124 new_nents = (*curr_nents + buff_size / CC_MAX_MLLI_ENTRY_SIZE + 1); 125 if (new_nents > MAX_NUM_OF_TOTAL_MLLI_ENTRIES) { 126 dev_err(dev, "Too many mlli entries. current %d max %d\n", 127 new_nents, MAX_NUM_OF_TOTAL_MLLI_ENTRIES); 128 return -ENOMEM; 129 } 130 131 /*handle buffer longer than 64 kbytes */ 132 while (buff_size > CC_MAX_MLLI_ENTRY_SIZE) { 133 cc_lli_set_addr(mlli_entry_p, buff_dma); 134 cc_lli_set_size(mlli_entry_p, CC_MAX_MLLI_ENTRY_SIZE); 135 dev_dbg(dev, "entry[%d]: single_buff=0x%08X size=%08X\n", 136 *curr_nents, mlli_entry_p[LLI_WORD0_OFFSET], 137 mlli_entry_p[LLI_WORD1_OFFSET]); 138 buff_dma += CC_MAX_MLLI_ENTRY_SIZE; 139 buff_size -= CC_MAX_MLLI_ENTRY_SIZE; 140 mlli_entry_p = mlli_entry_p + 2; 141 (*curr_nents)++; 142 } 143 /*Last entry */ 144 cc_lli_set_addr(mlli_entry_p, buff_dma); 145 cc_lli_set_size(mlli_entry_p, buff_size); 146 dev_dbg(dev, "entry[%d]: single_buff=0x%08X size=%08X\n", 147 *curr_nents, mlli_entry_p[LLI_WORD0_OFFSET], 148 mlli_entry_p[LLI_WORD1_OFFSET]); 149 mlli_entry_p = mlli_entry_p + 2; 150 *mlli_entry_pp = mlli_entry_p; 151 (*curr_nents)++; 152 return 0; 153 } 154 155 static int cc_render_sg_to_mlli(struct device *dev, struct scatterlist *sgl, 156 u32 sgl_data_len, u32 sgl_offset, 157 u32 *curr_nents, u32 **mlli_entry_pp) 158 { 159 struct scatterlist *curr_sgl = sgl; 160 u32 *mlli_entry_p = *mlli_entry_pp; 161 s32 rc = 0; 162 163 for ( ; (curr_sgl && sgl_data_len); 164 curr_sgl = sg_next(curr_sgl)) { 165 u32 entry_data_len = 166 (sgl_data_len > sg_dma_len(curr_sgl) - sgl_offset) ? 167 sg_dma_len(curr_sgl) - sgl_offset : 168 sgl_data_len; 169 sgl_data_len -= entry_data_len; 170 rc = cc_render_buff_to_mlli(dev, sg_dma_address(curr_sgl) + 171 sgl_offset, entry_data_len, 172 curr_nents, &mlli_entry_p); 173 if (rc) 174 return rc; 175 176 sgl_offset = 0; 177 } 178 *mlli_entry_pp = mlli_entry_p; 179 return 0; 180 } 181 182 static int cc_generate_mlli(struct device *dev, struct buffer_array *sg_data, 183 struct mlli_params *mlli_params, gfp_t flags) 184 { 185 u32 *mlli_p; 186 u32 total_nents = 0, prev_total_nents = 0; 187 int rc = 0, i; 188 189 dev_dbg(dev, "NUM of SG's = %d\n", sg_data->num_of_buffers); 190 191 /* Allocate memory from the pointed pool */ 192 mlli_params->mlli_virt_addr = 193 dma_pool_alloc(mlli_params->curr_pool, flags, 194 &mlli_params->mlli_dma_addr); 195 if (!mlli_params->mlli_virt_addr) { 196 dev_err(dev, "dma_pool_alloc() failed\n"); 197 rc = -ENOMEM; 198 goto build_mlli_exit; 199 } 200 /* Point to start of MLLI */ 201 mlli_p = mlli_params->mlli_virt_addr; 202 /* go over all SG's and link it to one MLLI table */ 203 for (i = 0; i < sg_data->num_of_buffers; i++) { 204 union buffer_array_entry *entry = &sg_data->entry[i]; 205 u32 tot_len = sg_data->total_data_len[i]; 206 u32 offset = sg_data->offset[i]; 207 208 rc = cc_render_sg_to_mlli(dev, entry->sgl, tot_len, offset, 209 &total_nents, &mlli_p); 210 if (rc) 211 return rc; 212 213 /* set last bit in the current table */ 214 if (sg_data->mlli_nents[i]) { 215 /*Calculate the current MLLI table length for the 216 *length field in the descriptor 217 */ 218 *sg_data->mlli_nents[i] += 219 (total_nents - prev_total_nents); 220 prev_total_nents = total_nents; 221 } 222 } 223 224 /* Set MLLI size for the bypass operation */ 225 mlli_params->mlli_len = (total_nents * LLI_ENTRY_BYTE_SIZE); 226 227 dev_dbg(dev, "MLLI params: virt_addr=%pK dma_addr=%pad mlli_len=0x%X\n", 228 mlli_params->mlli_virt_addr, &mlli_params->mlli_dma_addr, 229 mlli_params->mlli_len); 230 231 build_mlli_exit: 232 return rc; 233 } 234 235 static void cc_add_sg_entry(struct device *dev, struct buffer_array *sgl_data, 236 unsigned int nents, struct scatterlist *sgl, 237 unsigned int data_len, unsigned int data_offset, 238 bool is_last_table, u32 *mlli_nents) 239 { 240 unsigned int index = sgl_data->num_of_buffers; 241 242 dev_dbg(dev, "index=%u nents=%u sgl=%pK data_len=0x%08X is_last=%d\n", 243 index, nents, sgl, data_len, is_last_table); 244 sgl_data->nents[index] = nents; 245 sgl_data->entry[index].sgl = sgl; 246 sgl_data->offset[index] = data_offset; 247 sgl_data->total_data_len[index] = data_len; 248 sgl_data->is_last[index] = is_last_table; 249 sgl_data->mlli_nents[index] = mlli_nents; 250 if (sgl_data->mlli_nents[index]) 251 *sgl_data->mlli_nents[index] = 0; 252 sgl_data->num_of_buffers++; 253 } 254 255 static int cc_map_sg(struct device *dev, struct scatterlist *sg, 256 unsigned int nbytes, int direction, u32 *nents, 257 u32 max_sg_nents, u32 *lbytes, u32 *mapped_nents) 258 { 259 int ret = 0; 260 261 if (!nbytes) { 262 *mapped_nents = 0; 263 *lbytes = 0; 264 *nents = 0; 265 return 0; 266 } 267 268 *nents = cc_get_sgl_nents(dev, sg, nbytes, lbytes); 269 if (*nents > max_sg_nents) { 270 *nents = 0; 271 dev_err(dev, "Too many fragments. current %d max %d\n", 272 *nents, max_sg_nents); 273 return -ENOMEM; 274 } 275 276 ret = dma_map_sg(dev, sg, *nents, direction); 277 if (!ret) { 278 *nents = 0; 279 dev_err(dev, "dma_map_sg() sg buffer failed %d\n", ret); 280 return -ENOMEM; 281 } 282 283 *mapped_nents = ret; 284 285 return 0; 286 } 287 288 static int 289 cc_set_aead_conf_buf(struct device *dev, struct aead_req_ctx *areq_ctx, 290 u8 *config_data, struct buffer_array *sg_data, 291 unsigned int assoclen) 292 { 293 dev_dbg(dev, " handle additional data config set to DLLI\n"); 294 /* create sg for the current buffer */ 295 sg_init_one(&areq_ctx->ccm_adata_sg, config_data, 296 AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size); 297 if (dma_map_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE) != 1) { 298 dev_err(dev, "dma_map_sg() config buffer failed\n"); 299 return -ENOMEM; 300 } 301 dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n", 302 &sg_dma_address(&areq_ctx->ccm_adata_sg), 303 sg_page(&areq_ctx->ccm_adata_sg), 304 sg_virt(&areq_ctx->ccm_adata_sg), 305 areq_ctx->ccm_adata_sg.offset, areq_ctx->ccm_adata_sg.length); 306 /* prepare for case of MLLI */ 307 if (assoclen > 0) { 308 cc_add_sg_entry(dev, sg_data, 1, &areq_ctx->ccm_adata_sg, 309 (AES_BLOCK_SIZE + areq_ctx->ccm_hdr_size), 310 0, false, NULL); 311 } 312 return 0; 313 } 314 315 static int cc_set_hash_buf(struct device *dev, struct ahash_req_ctx *areq_ctx, 316 u8 *curr_buff, u32 curr_buff_cnt, 317 struct buffer_array *sg_data) 318 { 319 dev_dbg(dev, " handle curr buff %x set to DLLI\n", curr_buff_cnt); 320 /* create sg for the current buffer */ 321 sg_init_one(areq_ctx->buff_sg, curr_buff, curr_buff_cnt); 322 if (dma_map_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE) != 1) { 323 dev_err(dev, "dma_map_sg() src buffer failed\n"); 324 return -ENOMEM; 325 } 326 dev_dbg(dev, "Mapped curr_buff: dma_address=%pad page=%p addr=%pK offset=%u length=%u\n", 327 &sg_dma_address(areq_ctx->buff_sg), sg_page(areq_ctx->buff_sg), 328 sg_virt(areq_ctx->buff_sg), areq_ctx->buff_sg->offset, 329 areq_ctx->buff_sg->length); 330 areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI; 331 areq_ctx->curr_sg = areq_ctx->buff_sg; 332 areq_ctx->in_nents = 0; 333 /* prepare for case of MLLI */ 334 cc_add_sg_entry(dev, sg_data, 1, areq_ctx->buff_sg, curr_buff_cnt, 0, 335 false, NULL); 336 return 0; 337 } 338 339 void cc_unmap_cipher_request(struct device *dev, void *ctx, 340 unsigned int ivsize, struct scatterlist *src, 341 struct scatterlist *dst) 342 { 343 struct cipher_req_ctx *req_ctx = (struct cipher_req_ctx *)ctx; 344 345 if (req_ctx->gen_ctx.iv_dma_addr) { 346 dev_dbg(dev, "Unmapped iv: iv_dma_addr=%pad iv_size=%u\n", 347 &req_ctx->gen_ctx.iv_dma_addr, ivsize); 348 dma_unmap_single(dev, req_ctx->gen_ctx.iv_dma_addr, 349 ivsize, DMA_BIDIRECTIONAL); 350 } 351 /* Release pool */ 352 if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI && 353 req_ctx->mlli_params.mlli_virt_addr) { 354 dma_pool_free(req_ctx->mlli_params.curr_pool, 355 req_ctx->mlli_params.mlli_virt_addr, 356 req_ctx->mlli_params.mlli_dma_addr); 357 } 358 359 if (src != dst) { 360 dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_TO_DEVICE); 361 dma_unmap_sg(dev, dst, req_ctx->out_nents, DMA_FROM_DEVICE); 362 dev_dbg(dev, "Unmapped req->dst=%pK\n", sg_virt(dst)); 363 dev_dbg(dev, "Unmapped req->src=%pK\n", sg_virt(src)); 364 } else { 365 dma_unmap_sg(dev, src, req_ctx->in_nents, DMA_BIDIRECTIONAL); 366 dev_dbg(dev, "Unmapped req->src=%pK\n", sg_virt(src)); 367 } 368 } 369 370 int cc_map_cipher_request(struct cc_drvdata *drvdata, void *ctx, 371 unsigned int ivsize, unsigned int nbytes, 372 void *info, struct scatterlist *src, 373 struct scatterlist *dst, gfp_t flags) 374 { 375 struct cipher_req_ctx *req_ctx = (struct cipher_req_ctx *)ctx; 376 struct mlli_params *mlli_params = &req_ctx->mlli_params; 377 struct device *dev = drvdata_to_dev(drvdata); 378 struct buffer_array sg_data; 379 u32 dummy = 0; 380 int rc = 0; 381 u32 mapped_nents = 0; 382 int src_direction = (src != dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL); 383 384 req_ctx->dma_buf_type = CC_DMA_BUF_DLLI; 385 mlli_params->curr_pool = NULL; 386 sg_data.num_of_buffers = 0; 387 388 /* Map IV buffer */ 389 if (ivsize) { 390 dump_byte_array("iv", info, ivsize); 391 req_ctx->gen_ctx.iv_dma_addr = 392 dma_map_single(dev, info, ivsize, DMA_BIDIRECTIONAL); 393 if (dma_mapping_error(dev, req_ctx->gen_ctx.iv_dma_addr)) { 394 dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n", 395 ivsize, info); 396 return -ENOMEM; 397 } 398 dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n", 399 ivsize, info, &req_ctx->gen_ctx.iv_dma_addr); 400 } else { 401 req_ctx->gen_ctx.iv_dma_addr = 0; 402 } 403 404 /* Map the src SGL */ 405 rc = cc_map_sg(dev, src, nbytes, src_direction, &req_ctx->in_nents, 406 LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy, &mapped_nents); 407 if (rc) 408 goto cipher_exit; 409 if (mapped_nents > 1) 410 req_ctx->dma_buf_type = CC_DMA_BUF_MLLI; 411 412 if (src == dst) { 413 /* Handle inplace operation */ 414 if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { 415 req_ctx->out_nents = 0; 416 cc_add_sg_entry(dev, &sg_data, req_ctx->in_nents, src, 417 nbytes, 0, true, 418 &req_ctx->in_mlli_nents); 419 } 420 } else { 421 /* Map the dst sg */ 422 rc = cc_map_sg(dev, dst, nbytes, DMA_FROM_DEVICE, 423 &req_ctx->out_nents, LLI_MAX_NUM_OF_DATA_ENTRIES, 424 &dummy, &mapped_nents); 425 if (rc) 426 goto cipher_exit; 427 if (mapped_nents > 1) 428 req_ctx->dma_buf_type = CC_DMA_BUF_MLLI; 429 430 if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { 431 cc_add_sg_entry(dev, &sg_data, req_ctx->in_nents, src, 432 nbytes, 0, true, 433 &req_ctx->in_mlli_nents); 434 cc_add_sg_entry(dev, &sg_data, req_ctx->out_nents, dst, 435 nbytes, 0, true, 436 &req_ctx->out_mlli_nents); 437 } 438 } 439 440 if (req_ctx->dma_buf_type == CC_DMA_BUF_MLLI) { 441 mlli_params->curr_pool = drvdata->mlli_buffs_pool; 442 rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags); 443 if (rc) 444 goto cipher_exit; 445 } 446 447 dev_dbg(dev, "areq_ctx->dma_buf_type = %s\n", 448 cc_dma_buf_type(req_ctx->dma_buf_type)); 449 450 return 0; 451 452 cipher_exit: 453 cc_unmap_cipher_request(dev, req_ctx, ivsize, src, dst); 454 return rc; 455 } 456 457 void cc_unmap_aead_request(struct device *dev, struct aead_request *req) 458 { 459 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 460 unsigned int hw_iv_size = areq_ctx->hw_iv_size; 461 struct cc_drvdata *drvdata = dev_get_drvdata(dev); 462 int src_direction = (req->src != req->dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL); 463 464 if (areq_ctx->mac_buf_dma_addr) { 465 dma_unmap_single(dev, areq_ctx->mac_buf_dma_addr, 466 MAX_MAC_SIZE, DMA_BIDIRECTIONAL); 467 } 468 469 if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) { 470 if (areq_ctx->hkey_dma_addr) { 471 dma_unmap_single(dev, areq_ctx->hkey_dma_addr, 472 AES_BLOCK_SIZE, DMA_BIDIRECTIONAL); 473 } 474 475 if (areq_ctx->gcm_block_len_dma_addr) { 476 dma_unmap_single(dev, areq_ctx->gcm_block_len_dma_addr, 477 AES_BLOCK_SIZE, DMA_TO_DEVICE); 478 } 479 480 if (areq_ctx->gcm_iv_inc1_dma_addr) { 481 dma_unmap_single(dev, areq_ctx->gcm_iv_inc1_dma_addr, 482 AES_BLOCK_SIZE, DMA_TO_DEVICE); 483 } 484 485 if (areq_ctx->gcm_iv_inc2_dma_addr) { 486 dma_unmap_single(dev, areq_ctx->gcm_iv_inc2_dma_addr, 487 AES_BLOCK_SIZE, DMA_TO_DEVICE); 488 } 489 } 490 491 if (areq_ctx->ccm_hdr_size != ccm_header_size_null) { 492 if (areq_ctx->ccm_iv0_dma_addr) { 493 dma_unmap_single(dev, areq_ctx->ccm_iv0_dma_addr, 494 AES_BLOCK_SIZE, DMA_TO_DEVICE); 495 } 496 497 dma_unmap_sg(dev, &areq_ctx->ccm_adata_sg, 1, DMA_TO_DEVICE); 498 } 499 if (areq_ctx->gen_ctx.iv_dma_addr) { 500 dma_unmap_single(dev, areq_ctx->gen_ctx.iv_dma_addr, 501 hw_iv_size, DMA_BIDIRECTIONAL); 502 kfree_sensitive(areq_ctx->gen_ctx.iv); 503 } 504 505 /* Release pool */ 506 if ((areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || 507 areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) && 508 (areq_ctx->mlli_params.mlli_virt_addr)) { 509 dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n", 510 &areq_ctx->mlli_params.mlli_dma_addr, 511 areq_ctx->mlli_params.mlli_virt_addr); 512 dma_pool_free(areq_ctx->mlli_params.curr_pool, 513 areq_ctx->mlli_params.mlli_virt_addr, 514 areq_ctx->mlli_params.mlli_dma_addr); 515 } 516 517 dev_dbg(dev, "Unmapping src sgl: req->src=%pK areq_ctx->src.nents=%u areq_ctx->assoc.nents=%u assoclen:%u cryptlen=%u\n", 518 sg_virt(req->src), areq_ctx->src.nents, areq_ctx->assoc.nents, 519 areq_ctx->assoclen, req->cryptlen); 520 521 dma_unmap_sg(dev, req->src, areq_ctx->src.mapped_nents, src_direction); 522 if (req->src != req->dst) { 523 dev_dbg(dev, "Unmapping dst sgl: req->dst=%pK\n", 524 sg_virt(req->dst)); 525 dma_unmap_sg(dev, req->dst, areq_ctx->dst.mapped_nents, DMA_FROM_DEVICE); 526 } 527 if (drvdata->coherent && 528 areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT && 529 req->src == req->dst) { 530 /* copy back mac from temporary location to deal with possible 531 * data memory overriding that caused by cache coherence 532 * problem. 533 */ 534 cc_copy_mac(dev, req, CC_SG_FROM_BUF); 535 } 536 } 537 538 static bool cc_is_icv_frag(unsigned int sgl_nents, unsigned int authsize, 539 u32 last_entry_data_size) 540 { 541 return ((sgl_nents > 1) && (last_entry_data_size < authsize)); 542 } 543 544 static int cc_aead_chain_iv(struct cc_drvdata *drvdata, 545 struct aead_request *req, 546 struct buffer_array *sg_data, 547 bool is_last, bool do_chain) 548 { 549 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 550 unsigned int hw_iv_size = areq_ctx->hw_iv_size; 551 struct device *dev = drvdata_to_dev(drvdata); 552 gfp_t flags = cc_gfp_flags(&req->base); 553 int rc = 0; 554 555 if (!req->iv) { 556 areq_ctx->gen_ctx.iv_dma_addr = 0; 557 areq_ctx->gen_ctx.iv = NULL; 558 goto chain_iv_exit; 559 } 560 561 areq_ctx->gen_ctx.iv = kmemdup(req->iv, hw_iv_size, flags); 562 if (!areq_ctx->gen_ctx.iv) 563 return -ENOMEM; 564 565 areq_ctx->gen_ctx.iv_dma_addr = 566 dma_map_single(dev, areq_ctx->gen_ctx.iv, hw_iv_size, 567 DMA_BIDIRECTIONAL); 568 if (dma_mapping_error(dev, areq_ctx->gen_ctx.iv_dma_addr)) { 569 dev_err(dev, "Mapping iv %u B at va=%pK for DMA failed\n", 570 hw_iv_size, req->iv); 571 kfree_sensitive(areq_ctx->gen_ctx.iv); 572 areq_ctx->gen_ctx.iv = NULL; 573 rc = -ENOMEM; 574 goto chain_iv_exit; 575 } 576 577 dev_dbg(dev, "Mapped iv %u B at va=%pK to dma=%pad\n", 578 hw_iv_size, req->iv, &areq_ctx->gen_ctx.iv_dma_addr); 579 580 chain_iv_exit: 581 return rc; 582 } 583 584 static int cc_aead_chain_assoc(struct cc_drvdata *drvdata, 585 struct aead_request *req, 586 struct buffer_array *sg_data, 587 bool is_last, bool do_chain) 588 { 589 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 590 int rc = 0; 591 int mapped_nents = 0; 592 struct device *dev = drvdata_to_dev(drvdata); 593 594 if (!sg_data) { 595 rc = -EINVAL; 596 goto chain_assoc_exit; 597 } 598 599 if (areq_ctx->assoclen == 0) { 600 areq_ctx->assoc_buff_type = CC_DMA_BUF_NULL; 601 areq_ctx->assoc.nents = 0; 602 areq_ctx->assoc.mlli_nents = 0; 603 dev_dbg(dev, "Chain assoc of length 0: buff_type=%s nents=%u\n", 604 cc_dma_buf_type(areq_ctx->assoc_buff_type), 605 areq_ctx->assoc.nents); 606 goto chain_assoc_exit; 607 } 608 609 mapped_nents = sg_nents_for_len(req->src, areq_ctx->assoclen); 610 if (mapped_nents < 0) 611 return mapped_nents; 612 613 if (mapped_nents > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) { 614 dev_err(dev, "Too many fragments. current %d max %d\n", 615 mapped_nents, LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES); 616 return -ENOMEM; 617 } 618 areq_ctx->assoc.nents = mapped_nents; 619 620 /* in CCM case we have additional entry for 621 * ccm header configurations 622 */ 623 if (areq_ctx->ccm_hdr_size != ccm_header_size_null) { 624 if ((mapped_nents + 1) > LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES) { 625 dev_err(dev, "CCM case.Too many fragments. Current %d max %d\n", 626 (areq_ctx->assoc.nents + 1), 627 LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES); 628 rc = -ENOMEM; 629 goto chain_assoc_exit; 630 } 631 } 632 633 if (mapped_nents == 1 && areq_ctx->ccm_hdr_size == ccm_header_size_null) 634 areq_ctx->assoc_buff_type = CC_DMA_BUF_DLLI; 635 else 636 areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI; 637 638 if (do_chain || areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) { 639 dev_dbg(dev, "Chain assoc: buff_type=%s nents=%u\n", 640 cc_dma_buf_type(areq_ctx->assoc_buff_type), 641 areq_ctx->assoc.nents); 642 cc_add_sg_entry(dev, sg_data, areq_ctx->assoc.nents, req->src, 643 areq_ctx->assoclen, 0, is_last, 644 &areq_ctx->assoc.mlli_nents); 645 areq_ctx->assoc_buff_type = CC_DMA_BUF_MLLI; 646 } 647 648 chain_assoc_exit: 649 return rc; 650 } 651 652 static void cc_prepare_aead_data_dlli(struct aead_request *req, 653 u32 *src_last_bytes, u32 *dst_last_bytes) 654 { 655 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 656 enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type; 657 unsigned int authsize = areq_ctx->req_authsize; 658 struct scatterlist *sg; 659 ssize_t offset; 660 661 areq_ctx->is_icv_fragmented = false; 662 663 if ((req->src == req->dst) || direct == DRV_CRYPTO_DIRECTION_DECRYPT) { 664 sg = areq_ctx->src_sgl; 665 offset = *src_last_bytes - authsize; 666 } else { 667 sg = areq_ctx->dst_sgl; 668 offset = *dst_last_bytes - authsize; 669 } 670 671 areq_ctx->icv_dma_addr = sg_dma_address(sg) + offset; 672 areq_ctx->icv_virt_addr = sg_virt(sg) + offset; 673 } 674 675 static void cc_prepare_aead_data_mlli(struct cc_drvdata *drvdata, 676 struct aead_request *req, 677 struct buffer_array *sg_data, 678 u32 *src_last_bytes, u32 *dst_last_bytes, 679 bool is_last_table) 680 { 681 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 682 enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type; 683 unsigned int authsize = areq_ctx->req_authsize; 684 struct device *dev = drvdata_to_dev(drvdata); 685 struct scatterlist *sg; 686 687 if (req->src == req->dst) { 688 /*INPLACE*/ 689 cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents, 690 areq_ctx->src_sgl, areq_ctx->cryptlen, 691 areq_ctx->src_offset, is_last_table, 692 &areq_ctx->src.mlli_nents); 693 694 areq_ctx->is_icv_fragmented = 695 cc_is_icv_frag(areq_ctx->src.nents, authsize, 696 *src_last_bytes); 697 698 if (areq_ctx->is_icv_fragmented) { 699 /* Backup happens only when ICV is fragmented, ICV 700 * verification is made by CPU compare in order to 701 * simplify MAC verification upon request completion 702 */ 703 if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) { 704 /* In coherent platforms (e.g. ACP) 705 * already copying ICV for any 706 * INPLACE-DECRYPT operation, hence 707 * we must neglect this code. 708 */ 709 if (!drvdata->coherent) 710 cc_copy_mac(dev, req, CC_SG_TO_BUF); 711 712 areq_ctx->icv_virt_addr = areq_ctx->backup_mac; 713 } else { 714 areq_ctx->icv_virt_addr = areq_ctx->mac_buf; 715 areq_ctx->icv_dma_addr = 716 areq_ctx->mac_buf_dma_addr; 717 } 718 } else { /* Contig. ICV */ 719 sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1]; 720 /*Should hanlde if the sg is not contig.*/ 721 areq_ctx->icv_dma_addr = sg_dma_address(sg) + 722 (*src_last_bytes - authsize); 723 areq_ctx->icv_virt_addr = sg_virt(sg) + 724 (*src_last_bytes - authsize); 725 } 726 727 } else if (direct == DRV_CRYPTO_DIRECTION_DECRYPT) { 728 /*NON-INPLACE and DECRYPT*/ 729 cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents, 730 areq_ctx->src_sgl, areq_ctx->cryptlen, 731 areq_ctx->src_offset, is_last_table, 732 &areq_ctx->src.mlli_nents); 733 cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents, 734 areq_ctx->dst_sgl, areq_ctx->cryptlen, 735 areq_ctx->dst_offset, is_last_table, 736 &areq_ctx->dst.mlli_nents); 737 738 areq_ctx->is_icv_fragmented = 739 cc_is_icv_frag(areq_ctx->src.nents, authsize, 740 *src_last_bytes); 741 /* Backup happens only when ICV is fragmented, ICV 742 743 * verification is made by CPU compare in order to simplify 744 * MAC verification upon request completion 745 */ 746 if (areq_ctx->is_icv_fragmented) { 747 cc_copy_mac(dev, req, CC_SG_TO_BUF); 748 areq_ctx->icv_virt_addr = areq_ctx->backup_mac; 749 750 } else { /* Contig. ICV */ 751 sg = &areq_ctx->src_sgl[areq_ctx->src.nents - 1]; 752 /*Should hanlde if the sg is not contig.*/ 753 areq_ctx->icv_dma_addr = sg_dma_address(sg) + 754 (*src_last_bytes - authsize); 755 areq_ctx->icv_virt_addr = sg_virt(sg) + 756 (*src_last_bytes - authsize); 757 } 758 759 } else { 760 /*NON-INPLACE and ENCRYPT*/ 761 cc_add_sg_entry(dev, sg_data, areq_ctx->dst.nents, 762 areq_ctx->dst_sgl, areq_ctx->cryptlen, 763 areq_ctx->dst_offset, is_last_table, 764 &areq_ctx->dst.mlli_nents); 765 cc_add_sg_entry(dev, sg_data, areq_ctx->src.nents, 766 areq_ctx->src_sgl, areq_ctx->cryptlen, 767 areq_ctx->src_offset, is_last_table, 768 &areq_ctx->src.mlli_nents); 769 770 areq_ctx->is_icv_fragmented = 771 cc_is_icv_frag(areq_ctx->dst.nents, authsize, 772 *dst_last_bytes); 773 774 if (!areq_ctx->is_icv_fragmented) { 775 sg = &areq_ctx->dst_sgl[areq_ctx->dst.nents - 1]; 776 /* Contig. ICV */ 777 areq_ctx->icv_dma_addr = sg_dma_address(sg) + 778 (*dst_last_bytes - authsize); 779 areq_ctx->icv_virt_addr = sg_virt(sg) + 780 (*dst_last_bytes - authsize); 781 } else { 782 areq_ctx->icv_dma_addr = areq_ctx->mac_buf_dma_addr; 783 areq_ctx->icv_virt_addr = areq_ctx->mac_buf; 784 } 785 } 786 } 787 788 static int cc_aead_chain_data(struct cc_drvdata *drvdata, 789 struct aead_request *req, 790 struct buffer_array *sg_data, 791 bool is_last_table, bool do_chain) 792 { 793 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 794 struct device *dev = drvdata_to_dev(drvdata); 795 enum drv_crypto_direction direct = areq_ctx->gen_ctx.op_type; 796 unsigned int authsize = areq_ctx->req_authsize; 797 unsigned int src_last_bytes = 0, dst_last_bytes = 0; 798 int rc = 0; 799 u32 src_mapped_nents = 0, dst_mapped_nents = 0; 800 u32 offset = 0; 801 /* non-inplace mode */ 802 unsigned int size_for_map = req->assoclen + req->cryptlen; 803 u32 sg_index = 0; 804 u32 size_to_skip = req->assoclen; 805 struct scatterlist *sgl; 806 807 offset = size_to_skip; 808 809 if (!sg_data) 810 return -EINVAL; 811 812 areq_ctx->src_sgl = req->src; 813 areq_ctx->dst_sgl = req->dst; 814 815 size_for_map += (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) ? 816 authsize : 0; 817 src_mapped_nents = cc_get_sgl_nents(dev, req->src, size_for_map, 818 &src_last_bytes); 819 sg_index = areq_ctx->src_sgl->length; 820 //check where the data starts 821 while (src_mapped_nents && (sg_index <= size_to_skip)) { 822 src_mapped_nents--; 823 offset -= areq_ctx->src_sgl->length; 824 sgl = sg_next(areq_ctx->src_sgl); 825 if (!sgl) 826 break; 827 areq_ctx->src_sgl = sgl; 828 sg_index += areq_ctx->src_sgl->length; 829 } 830 if (src_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) { 831 dev_err(dev, "Too many fragments. current %d max %d\n", 832 src_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES); 833 return -ENOMEM; 834 } 835 836 areq_ctx->src.nents = src_mapped_nents; 837 838 areq_ctx->src_offset = offset; 839 840 if (req->src != req->dst) { 841 size_for_map = req->assoclen + req->cryptlen; 842 843 if (direct == DRV_CRYPTO_DIRECTION_ENCRYPT) 844 size_for_map += authsize; 845 else 846 size_for_map -= authsize; 847 848 rc = cc_map_sg(dev, req->dst, size_for_map, DMA_FROM_DEVICE, 849 &areq_ctx->dst.mapped_nents, 850 LLI_MAX_NUM_OF_DATA_ENTRIES, &dst_last_bytes, 851 &dst_mapped_nents); 852 if (rc) 853 goto chain_data_exit; 854 } 855 856 dst_mapped_nents = cc_get_sgl_nents(dev, req->dst, size_for_map, 857 &dst_last_bytes); 858 sg_index = areq_ctx->dst_sgl->length; 859 offset = size_to_skip; 860 861 //check where the data starts 862 while (dst_mapped_nents && sg_index <= size_to_skip) { 863 dst_mapped_nents--; 864 offset -= areq_ctx->dst_sgl->length; 865 sgl = sg_next(areq_ctx->dst_sgl); 866 if (!sgl) 867 break; 868 areq_ctx->dst_sgl = sgl; 869 sg_index += areq_ctx->dst_sgl->length; 870 } 871 if (dst_mapped_nents > LLI_MAX_NUM_OF_DATA_ENTRIES) { 872 dev_err(dev, "Too many fragments. current %d max %d\n", 873 dst_mapped_nents, LLI_MAX_NUM_OF_DATA_ENTRIES); 874 return -ENOMEM; 875 } 876 areq_ctx->dst.nents = dst_mapped_nents; 877 areq_ctx->dst_offset = offset; 878 if (src_mapped_nents > 1 || 879 dst_mapped_nents > 1 || 880 do_chain) { 881 areq_ctx->data_buff_type = CC_DMA_BUF_MLLI; 882 cc_prepare_aead_data_mlli(drvdata, req, sg_data, 883 &src_last_bytes, &dst_last_bytes, 884 is_last_table); 885 } else { 886 areq_ctx->data_buff_type = CC_DMA_BUF_DLLI; 887 cc_prepare_aead_data_dlli(req, &src_last_bytes, 888 &dst_last_bytes); 889 } 890 891 chain_data_exit: 892 return rc; 893 } 894 895 static void cc_update_aead_mlli_nents(struct cc_drvdata *drvdata, 896 struct aead_request *req) 897 { 898 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 899 u32 curr_mlli_size = 0; 900 901 if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI) { 902 areq_ctx->assoc.sram_addr = drvdata->mlli_sram_addr; 903 curr_mlli_size = areq_ctx->assoc.mlli_nents * 904 LLI_ENTRY_BYTE_SIZE; 905 } 906 907 if (areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) { 908 /*Inplace case dst nents equal to src nents*/ 909 if (req->src == req->dst) { 910 areq_ctx->dst.mlli_nents = areq_ctx->src.mlli_nents; 911 areq_ctx->src.sram_addr = drvdata->mlli_sram_addr + 912 curr_mlli_size; 913 areq_ctx->dst.sram_addr = areq_ctx->src.sram_addr; 914 if (!areq_ctx->is_single_pass) 915 areq_ctx->assoc.mlli_nents += 916 areq_ctx->src.mlli_nents; 917 } else { 918 if (areq_ctx->gen_ctx.op_type == 919 DRV_CRYPTO_DIRECTION_DECRYPT) { 920 areq_ctx->src.sram_addr = 921 drvdata->mlli_sram_addr + 922 curr_mlli_size; 923 areq_ctx->dst.sram_addr = 924 areq_ctx->src.sram_addr + 925 areq_ctx->src.mlli_nents * 926 LLI_ENTRY_BYTE_SIZE; 927 if (!areq_ctx->is_single_pass) 928 areq_ctx->assoc.mlli_nents += 929 areq_ctx->src.mlli_nents; 930 } else { 931 areq_ctx->dst.sram_addr = 932 drvdata->mlli_sram_addr + 933 curr_mlli_size; 934 areq_ctx->src.sram_addr = 935 areq_ctx->dst.sram_addr + 936 areq_ctx->dst.mlli_nents * 937 LLI_ENTRY_BYTE_SIZE; 938 if (!areq_ctx->is_single_pass) 939 areq_ctx->assoc.mlli_nents += 940 areq_ctx->dst.mlli_nents; 941 } 942 } 943 } 944 } 945 946 int cc_map_aead_request(struct cc_drvdata *drvdata, struct aead_request *req) 947 { 948 struct aead_req_ctx *areq_ctx = aead_request_ctx_dma(req); 949 struct mlli_params *mlli_params = &areq_ctx->mlli_params; 950 struct device *dev = drvdata_to_dev(drvdata); 951 struct buffer_array sg_data; 952 unsigned int authsize = areq_ctx->req_authsize; 953 int rc = 0; 954 dma_addr_t dma_addr; 955 u32 mapped_nents = 0; 956 u32 dummy = 0; /*used for the assoc data fragments */ 957 u32 size_to_map; 958 gfp_t flags = cc_gfp_flags(&req->base); 959 960 mlli_params->curr_pool = NULL; 961 sg_data.num_of_buffers = 0; 962 963 /* copy mac to a temporary location to deal with possible 964 * data memory overriding that caused by cache coherence problem. 965 */ 966 if (drvdata->coherent && 967 areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_DECRYPT && 968 req->src == req->dst) 969 cc_copy_mac(dev, req, CC_SG_TO_BUF); 970 971 /* cacluate the size for cipher remove ICV in decrypt*/ 972 areq_ctx->cryptlen = (areq_ctx->gen_ctx.op_type == 973 DRV_CRYPTO_DIRECTION_ENCRYPT) ? 974 req->cryptlen : 975 (req->cryptlen - authsize); 976 977 dma_addr = dma_map_single(dev, areq_ctx->mac_buf, MAX_MAC_SIZE, 978 DMA_BIDIRECTIONAL); 979 if (dma_mapping_error(dev, dma_addr)) { 980 dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n", 981 MAX_MAC_SIZE, areq_ctx->mac_buf); 982 rc = -ENOMEM; 983 goto aead_map_failure; 984 } 985 areq_ctx->mac_buf_dma_addr = dma_addr; 986 987 if (areq_ctx->ccm_hdr_size != ccm_header_size_null) { 988 void *addr = areq_ctx->ccm_config + CCM_CTR_COUNT_0_OFFSET; 989 990 dma_addr = dma_map_single(dev, addr, AES_BLOCK_SIZE, 991 DMA_TO_DEVICE); 992 993 if (dma_mapping_error(dev, dma_addr)) { 994 dev_err(dev, "Mapping mac_buf %u B at va=%pK for DMA failed\n", 995 AES_BLOCK_SIZE, addr); 996 areq_ctx->ccm_iv0_dma_addr = 0; 997 rc = -ENOMEM; 998 goto aead_map_failure; 999 } 1000 areq_ctx->ccm_iv0_dma_addr = dma_addr; 1001 1002 rc = cc_set_aead_conf_buf(dev, areq_ctx, areq_ctx->ccm_config, 1003 &sg_data, areq_ctx->assoclen); 1004 if (rc) 1005 goto aead_map_failure; 1006 } 1007 1008 if (areq_ctx->cipher_mode == DRV_CIPHER_GCTR) { 1009 dma_addr = dma_map_single(dev, areq_ctx->hkey, AES_BLOCK_SIZE, 1010 DMA_BIDIRECTIONAL); 1011 if (dma_mapping_error(dev, dma_addr)) { 1012 dev_err(dev, "Mapping hkey %u B at va=%pK for DMA failed\n", 1013 AES_BLOCK_SIZE, areq_ctx->hkey); 1014 rc = -ENOMEM; 1015 goto aead_map_failure; 1016 } 1017 areq_ctx->hkey_dma_addr = dma_addr; 1018 1019 dma_addr = dma_map_single(dev, &areq_ctx->gcm_len_block, 1020 AES_BLOCK_SIZE, DMA_TO_DEVICE); 1021 if (dma_mapping_error(dev, dma_addr)) { 1022 dev_err(dev, "Mapping gcm_len_block %u B at va=%pK for DMA failed\n", 1023 AES_BLOCK_SIZE, &areq_ctx->gcm_len_block); 1024 rc = -ENOMEM; 1025 goto aead_map_failure; 1026 } 1027 areq_ctx->gcm_block_len_dma_addr = dma_addr; 1028 1029 dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc1, 1030 AES_BLOCK_SIZE, DMA_TO_DEVICE); 1031 1032 if (dma_mapping_error(dev, dma_addr)) { 1033 dev_err(dev, "Mapping gcm_iv_inc1 %u B at va=%pK for DMA failed\n", 1034 AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc1)); 1035 areq_ctx->gcm_iv_inc1_dma_addr = 0; 1036 rc = -ENOMEM; 1037 goto aead_map_failure; 1038 } 1039 areq_ctx->gcm_iv_inc1_dma_addr = dma_addr; 1040 1041 dma_addr = dma_map_single(dev, areq_ctx->gcm_iv_inc2, 1042 AES_BLOCK_SIZE, DMA_TO_DEVICE); 1043 1044 if (dma_mapping_error(dev, dma_addr)) { 1045 dev_err(dev, "Mapping gcm_iv_inc2 %u B at va=%pK for DMA failed\n", 1046 AES_BLOCK_SIZE, (areq_ctx->gcm_iv_inc2)); 1047 areq_ctx->gcm_iv_inc2_dma_addr = 0; 1048 rc = -ENOMEM; 1049 goto aead_map_failure; 1050 } 1051 areq_ctx->gcm_iv_inc2_dma_addr = dma_addr; 1052 } 1053 1054 size_to_map = req->cryptlen + req->assoclen; 1055 /* If we do in-place encryption, we also need the auth tag */ 1056 if ((areq_ctx->gen_ctx.op_type == DRV_CRYPTO_DIRECTION_ENCRYPT) && 1057 (req->src == req->dst)) { 1058 size_to_map += authsize; 1059 } 1060 1061 rc = cc_map_sg(dev, req->src, size_to_map, 1062 (req->src != req->dst ? DMA_TO_DEVICE : DMA_BIDIRECTIONAL), 1063 &areq_ctx->src.mapped_nents, 1064 (LLI_MAX_NUM_OF_ASSOC_DATA_ENTRIES + 1065 LLI_MAX_NUM_OF_DATA_ENTRIES), 1066 &dummy, &mapped_nents); 1067 if (rc) 1068 goto aead_map_failure; 1069 1070 if (areq_ctx->is_single_pass) { 1071 /* 1072 * Create MLLI table for: 1073 * (1) Assoc. data 1074 * (2) Src/Dst SGLs 1075 * Note: IV is contg. buffer (not an SGL) 1076 */ 1077 rc = cc_aead_chain_assoc(drvdata, req, &sg_data, true, false); 1078 if (rc) 1079 goto aead_map_failure; 1080 rc = cc_aead_chain_iv(drvdata, req, &sg_data, true, false); 1081 if (rc) 1082 goto aead_map_failure; 1083 rc = cc_aead_chain_data(drvdata, req, &sg_data, true, false); 1084 if (rc) 1085 goto aead_map_failure; 1086 } else { /* DOUBLE-PASS flow */ 1087 /* 1088 * Prepare MLLI table(s) in this order: 1089 * 1090 * If ENCRYPT/DECRYPT (inplace): 1091 * (1) MLLI table for assoc 1092 * (2) IV entry (chained right after end of assoc) 1093 * (3) MLLI for src/dst (inplace operation) 1094 * 1095 * If ENCRYPT (non-inplace) 1096 * (1) MLLI table for assoc 1097 * (2) IV entry (chained right after end of assoc) 1098 * (3) MLLI for dst 1099 * (4) MLLI for src 1100 * 1101 * If DECRYPT (non-inplace) 1102 * (1) MLLI table for assoc 1103 * (2) IV entry (chained right after end of assoc) 1104 * (3) MLLI for src 1105 * (4) MLLI for dst 1106 */ 1107 rc = cc_aead_chain_assoc(drvdata, req, &sg_data, false, true); 1108 if (rc) 1109 goto aead_map_failure; 1110 rc = cc_aead_chain_iv(drvdata, req, &sg_data, false, true); 1111 if (rc) 1112 goto aead_map_failure; 1113 rc = cc_aead_chain_data(drvdata, req, &sg_data, true, true); 1114 if (rc) 1115 goto aead_map_failure; 1116 } 1117 1118 /* Mlli support -start building the MLLI according to the above 1119 * results 1120 */ 1121 if (areq_ctx->assoc_buff_type == CC_DMA_BUF_MLLI || 1122 areq_ctx->data_buff_type == CC_DMA_BUF_MLLI) { 1123 mlli_params->curr_pool = drvdata->mlli_buffs_pool; 1124 rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags); 1125 if (rc) 1126 goto aead_map_failure; 1127 1128 cc_update_aead_mlli_nents(drvdata, req); 1129 dev_dbg(dev, "assoc params mn %d\n", 1130 areq_ctx->assoc.mlli_nents); 1131 dev_dbg(dev, "src params mn %d\n", areq_ctx->src.mlli_nents); 1132 dev_dbg(dev, "dst params mn %d\n", areq_ctx->dst.mlli_nents); 1133 } 1134 return 0; 1135 1136 aead_map_failure: 1137 cc_unmap_aead_request(dev, req); 1138 return rc; 1139 } 1140 1141 int cc_map_hash_request_final(struct cc_drvdata *drvdata, void *ctx, 1142 struct scatterlist *src, unsigned int nbytes, 1143 bool do_update, gfp_t flags) 1144 { 1145 struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx; 1146 struct device *dev = drvdata_to_dev(drvdata); 1147 u8 *curr_buff = cc_hash_buf(areq_ctx); 1148 u32 *curr_buff_cnt = cc_hash_buf_cnt(areq_ctx); 1149 struct mlli_params *mlli_params = &areq_ctx->mlli_params; 1150 struct buffer_array sg_data; 1151 int rc = 0; 1152 u32 dummy = 0; 1153 u32 mapped_nents = 0; 1154 1155 dev_dbg(dev, "final params : curr_buff=%pK curr_buff_cnt=0x%X nbytes = 0x%X src=%pK curr_index=%u\n", 1156 curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index); 1157 /* Init the type of the dma buffer */ 1158 areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL; 1159 mlli_params->curr_pool = NULL; 1160 sg_data.num_of_buffers = 0; 1161 areq_ctx->in_nents = 0; 1162 1163 if (nbytes == 0 && *curr_buff_cnt == 0) { 1164 /* nothing to do */ 1165 return 0; 1166 } 1167 1168 /* map the previous buffer */ 1169 if (*curr_buff_cnt) { 1170 rc = cc_set_hash_buf(dev, areq_ctx, curr_buff, *curr_buff_cnt, 1171 &sg_data); 1172 if (rc) 1173 return rc; 1174 } 1175 1176 if (src && nbytes > 0 && do_update) { 1177 rc = cc_map_sg(dev, src, nbytes, DMA_TO_DEVICE, 1178 &areq_ctx->in_nents, LLI_MAX_NUM_OF_DATA_ENTRIES, 1179 &dummy, &mapped_nents); 1180 if (rc) 1181 goto unmap_curr_buff; 1182 if (src && mapped_nents == 1 && 1183 areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) { 1184 memcpy(areq_ctx->buff_sg, src, 1185 sizeof(struct scatterlist)); 1186 areq_ctx->buff_sg->length = nbytes; 1187 areq_ctx->curr_sg = areq_ctx->buff_sg; 1188 areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI; 1189 } else { 1190 areq_ctx->data_dma_buf_type = CC_DMA_BUF_MLLI; 1191 } 1192 } 1193 1194 /*build mlli */ 1195 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_MLLI) { 1196 mlli_params->curr_pool = drvdata->mlli_buffs_pool; 1197 /* add the src data to the sg_data */ 1198 cc_add_sg_entry(dev, &sg_data, areq_ctx->in_nents, src, nbytes, 1199 0, true, &areq_ctx->mlli_nents); 1200 rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags); 1201 if (rc) 1202 goto fail_unmap_din; 1203 } 1204 /* change the buffer index for the unmap function */ 1205 areq_ctx->buff_index = (areq_ctx->buff_index ^ 1); 1206 dev_dbg(dev, "areq_ctx->data_dma_buf_type = %s\n", 1207 cc_dma_buf_type(areq_ctx->data_dma_buf_type)); 1208 return 0; 1209 1210 fail_unmap_din: 1211 dma_unmap_sg(dev, src, areq_ctx->in_nents, DMA_TO_DEVICE); 1212 1213 unmap_curr_buff: 1214 if (*curr_buff_cnt) 1215 dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE); 1216 1217 return rc; 1218 } 1219 1220 int cc_map_hash_request_update(struct cc_drvdata *drvdata, void *ctx, 1221 struct scatterlist *src, unsigned int nbytes, 1222 unsigned int block_size, gfp_t flags) 1223 { 1224 struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx; 1225 struct device *dev = drvdata_to_dev(drvdata); 1226 u8 *curr_buff = cc_hash_buf(areq_ctx); 1227 u32 *curr_buff_cnt = cc_hash_buf_cnt(areq_ctx); 1228 u8 *next_buff = cc_next_buf(areq_ctx); 1229 u32 *next_buff_cnt = cc_next_buf_cnt(areq_ctx); 1230 struct mlli_params *mlli_params = &areq_ctx->mlli_params; 1231 unsigned int update_data_len; 1232 u32 total_in_len = nbytes + *curr_buff_cnt; 1233 struct buffer_array sg_data; 1234 unsigned int swap_index = 0; 1235 int rc = 0; 1236 u32 dummy = 0; 1237 u32 mapped_nents = 0; 1238 1239 dev_dbg(dev, " update params : curr_buff=%pK curr_buff_cnt=0x%X nbytes=0x%X src=%pK curr_index=%u\n", 1240 curr_buff, *curr_buff_cnt, nbytes, src, areq_ctx->buff_index); 1241 /* Init the type of the dma buffer */ 1242 areq_ctx->data_dma_buf_type = CC_DMA_BUF_NULL; 1243 mlli_params->curr_pool = NULL; 1244 areq_ctx->curr_sg = NULL; 1245 sg_data.num_of_buffers = 0; 1246 areq_ctx->in_nents = 0; 1247 1248 if (total_in_len < block_size) { 1249 dev_dbg(dev, " less than one block: curr_buff=%pK *curr_buff_cnt=0x%X copy_to=%pK\n", 1250 curr_buff, *curr_buff_cnt, &curr_buff[*curr_buff_cnt]); 1251 areq_ctx->in_nents = sg_nents_for_len(src, nbytes); 1252 sg_copy_to_buffer(src, areq_ctx->in_nents, 1253 &curr_buff[*curr_buff_cnt], nbytes); 1254 *curr_buff_cnt += nbytes; 1255 return 1; 1256 } 1257 1258 /* Calculate the residue size*/ 1259 *next_buff_cnt = total_in_len & (block_size - 1); 1260 /* update data len */ 1261 update_data_len = total_in_len - *next_buff_cnt; 1262 1263 dev_dbg(dev, " temp length : *next_buff_cnt=0x%X update_data_len=0x%X\n", 1264 *next_buff_cnt, update_data_len); 1265 1266 /* Copy the new residue to next buffer */ 1267 if (*next_buff_cnt) { 1268 dev_dbg(dev, " handle residue: next buff %pK skip data %u residue %u\n", 1269 next_buff, (update_data_len - *curr_buff_cnt), 1270 *next_buff_cnt); 1271 cc_copy_sg_portion(dev, next_buff, src, 1272 (update_data_len - *curr_buff_cnt), 1273 nbytes, CC_SG_TO_BUF); 1274 /* change the buffer index for next operation */ 1275 swap_index = 1; 1276 } 1277 1278 if (*curr_buff_cnt) { 1279 rc = cc_set_hash_buf(dev, areq_ctx, curr_buff, *curr_buff_cnt, 1280 &sg_data); 1281 if (rc) 1282 return rc; 1283 /* change the buffer index for next operation */ 1284 swap_index = 1; 1285 } 1286 1287 if (update_data_len > *curr_buff_cnt) { 1288 rc = cc_map_sg(dev, src, (update_data_len - *curr_buff_cnt), 1289 DMA_TO_DEVICE, &areq_ctx->in_nents, 1290 LLI_MAX_NUM_OF_DATA_ENTRIES, &dummy, 1291 &mapped_nents); 1292 if (rc) 1293 goto unmap_curr_buff; 1294 if (mapped_nents == 1 && 1295 areq_ctx->data_dma_buf_type == CC_DMA_BUF_NULL) { 1296 /* only one entry in the SG and no previous data */ 1297 memcpy(areq_ctx->buff_sg, src, 1298 sizeof(struct scatterlist)); 1299 areq_ctx->buff_sg->length = update_data_len; 1300 areq_ctx->data_dma_buf_type = CC_DMA_BUF_DLLI; 1301 areq_ctx->curr_sg = areq_ctx->buff_sg; 1302 } else { 1303 areq_ctx->data_dma_buf_type = CC_DMA_BUF_MLLI; 1304 } 1305 } 1306 1307 if (areq_ctx->data_dma_buf_type == CC_DMA_BUF_MLLI) { 1308 mlli_params->curr_pool = drvdata->mlli_buffs_pool; 1309 /* add the src data to the sg_data */ 1310 cc_add_sg_entry(dev, &sg_data, areq_ctx->in_nents, src, 1311 (update_data_len - *curr_buff_cnt), 0, true, 1312 &areq_ctx->mlli_nents); 1313 rc = cc_generate_mlli(dev, &sg_data, mlli_params, flags); 1314 if (rc) 1315 goto fail_unmap_din; 1316 } 1317 areq_ctx->buff_index = (areq_ctx->buff_index ^ swap_index); 1318 1319 return 0; 1320 1321 fail_unmap_din: 1322 dma_unmap_sg(dev, src, areq_ctx->in_nents, DMA_TO_DEVICE); 1323 1324 unmap_curr_buff: 1325 if (*curr_buff_cnt) 1326 dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE); 1327 1328 return rc; 1329 } 1330 1331 void cc_unmap_hash_request(struct device *dev, void *ctx, 1332 struct scatterlist *src, bool do_revert) 1333 { 1334 struct ahash_req_ctx *areq_ctx = (struct ahash_req_ctx *)ctx; 1335 u32 *prev_len = cc_next_buf_cnt(areq_ctx); 1336 1337 /*In case a pool was set, a table was 1338 *allocated and should be released 1339 */ 1340 if (areq_ctx->mlli_params.curr_pool) { 1341 dev_dbg(dev, "free MLLI buffer: dma=%pad virt=%pK\n", 1342 &areq_ctx->mlli_params.mlli_dma_addr, 1343 areq_ctx->mlli_params.mlli_virt_addr); 1344 dma_pool_free(areq_ctx->mlli_params.curr_pool, 1345 areq_ctx->mlli_params.mlli_virt_addr, 1346 areq_ctx->mlli_params.mlli_dma_addr); 1347 } 1348 1349 if (src && areq_ctx->in_nents) { 1350 dev_dbg(dev, "Unmapped sg src: virt=%pK dma=%pad len=0x%X\n", 1351 sg_virt(src), &sg_dma_address(src), sg_dma_len(src)); 1352 dma_unmap_sg(dev, src, 1353 areq_ctx->in_nents, DMA_TO_DEVICE); 1354 } 1355 1356 if (*prev_len) { 1357 dev_dbg(dev, "Unmapped buffer: areq_ctx->buff_sg=%pK dma=%pad len 0x%X\n", 1358 sg_virt(areq_ctx->buff_sg), 1359 &sg_dma_address(areq_ctx->buff_sg), 1360 sg_dma_len(areq_ctx->buff_sg)); 1361 dma_unmap_sg(dev, areq_ctx->buff_sg, 1, DMA_TO_DEVICE); 1362 if (!do_revert) { 1363 /* clean the previous data length for update 1364 * operation 1365 */ 1366 *prev_len = 0; 1367 } else { 1368 areq_ctx->buff_index ^= 1; 1369 } 1370 } 1371 } 1372 1373 int cc_buffer_mgr_init(struct cc_drvdata *drvdata) 1374 { 1375 struct device *dev = drvdata_to_dev(drvdata); 1376 1377 drvdata->mlli_buffs_pool = 1378 dma_pool_create("dx_single_mlli_tables", dev, 1379 MAX_NUM_OF_TOTAL_MLLI_ENTRIES * 1380 LLI_ENTRY_BYTE_SIZE, 1381 MLLI_TABLE_MIN_ALIGNMENT, 0); 1382 1383 if (!drvdata->mlli_buffs_pool) 1384 return -ENOMEM; 1385 1386 return 0; 1387 } 1388 1389 int cc_buffer_mgr_fini(struct cc_drvdata *drvdata) 1390 { 1391 dma_pool_destroy(drvdata->mlli_buffs_pool); 1392 return 0; 1393 } 1394