Lines Matching +full:inline +full:- +full:crypto +full:- +full:engine
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Intel Keem Bay OCS AES Crypto Driver.
5 * Copyright (C) 2018-2020 Intel Corporation
8 #include <linux/dma-mapping.h>
17 #include <crypto/aes.h>
18 #include <crypto/gcm.h>
20 #include "ocs-aes.h"
131 * 11-bit value, but it is actually 10-bits.
137 * before the tag is written. For 128-bit mode this required delay is 28 OCS
138 * clock cycles. For 256-bit mode it is 36 OCS clock cycles.
156 * ------------ ---------
158 * 1 ... 15-L Nonce N
159 * 16-L ... 15 Counter i
161 * Flags = L' = L - 1
164 #define COUNTER_START(lprime) (16 - ((lprime) + 1))
175 * struct ocs_dma_linked_list - OCS DMA linked list entry.
179 * @ll_flags: Flags (Freeze @ terminate) for the DMA engine.
192 * bit [10] - KEY_HI_LO_SWAP
193 * bit [9] - KEY_HI_SWAP_DWORDS_IN_OCTWORD
194 * bit [8] - KEY_HI_SWAP_BYTES_IN_DWORD
195 * bit [7] - KEY_LO_SWAP_DWORDS_IN_OCTWORD
196 * bit [6] - KEY_LO_SWAP_BYTES_IN_DWORD
197 * bit [5] - IV_SWAP_DWORDS_IN_OCTWORD
198 * bit [4] - IV_SWAP_BYTES_IN_DWORD
199 * bit [3] - DOUT_SWAP_DWORDS_IN_OCTWORD
200 * bit [2] - DOUT_SWAP_BYTES_IN_DWORD
201 * bit [1] - DOUT_SWAP_DWORDS_IN_OCTWORD
202 * bit [0] - DOUT_SWAP_BYTES_IN_DWORD
204 static inline void aes_a_set_endianness(const struct ocs_aes_dev *aes_dev) in aes_a_set_endianness()
206 iowrite32(0x7FF, aes_dev->base_reg + AES_BYTE_ORDER_CFG_OFFSET); in aes_a_set_endianness()
210 static inline void aes_a_op_trigger(const struct ocs_aes_dev *aes_dev) in aes_a_op_trigger()
212 iowrite32(AES_ACTIVE_TRIGGER, aes_dev->base_reg + AES_ACTIVE_OFFSET); in aes_a_op_trigger()
216 static inline void aes_a_op_termination(const struct ocs_aes_dev *aes_dev) in aes_a_op_termination()
219 aes_dev->base_reg + AES_ACTIVE_OFFSET); in aes_a_op_termination()
226 * - For AES-CCM it is called for the last batch of Payload data and Ciphertext
228 * - For AES-GCM, it is called for the last batch of Plaintext data and
231 static inline void aes_a_set_last_gcx(const struct ocs_aes_dev *aes_dev) in aes_a_set_last_gcx()
234 aes_dev->base_reg + AES_ACTIVE_OFFSET); in aes_a_set_last_gcx()
238 static inline void aes_a_wait_last_gcx(const struct ocs_aes_dev *aes_dev) in aes_a_wait_last_gcx()
243 aes_active_reg = ioread32(aes_dev->base_reg + in aes_a_wait_last_gcx()
254 reg = ioread32(aes_dev->base_reg + AES_A_DMA_STATUS_OFFSET); in aes_a_dma_wait_input_buffer_occupancy()
265 static inline void aes_a_set_last_gcx_and_adata(const struct ocs_aes_dev *aes_dev) in aes_a_set_last_gcx_and_adata()
268 aes_dev->base_reg + AES_ACTIVE_OFFSET); in aes_a_set_last_gcx_and_adata()
272 static inline void aes_a_dma_set_xfer_size_zero(const struct ocs_aes_dev *aes_dev) in aes_a_dma_set_xfer_size_zero()
274 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); in aes_a_dma_set_xfer_size_zero()
275 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); in aes_a_dma_set_xfer_size_zero()
278 /* Activate DMA for zero-byte transfer case. */
279 static inline void aes_a_dma_active(const struct ocs_aes_dev *aes_dev) in aes_a_dma_active()
282 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); in aes_a_dma_active()
286 static inline void aes_a_dma_active_src_ll_en(const struct ocs_aes_dev *aes_dev) in aes_a_dma_active_src_ll_en()
290 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); in aes_a_dma_active_src_ll_en()
294 static inline void aes_a_dma_active_dst_ll_en(const struct ocs_aes_dev *aes_dev) in aes_a_dma_active_dst_ll_en()
298 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); in aes_a_dma_active_dst_ll_en()
302 static inline void aes_a_dma_active_src_dst_ll_en(const struct ocs_aes_dev *aes_dev) in aes_a_dma_active_src_dst_ll_en()
307 aes_dev->base_reg + AES_A_DMA_DMA_MODE_OFFSET); in aes_a_dma_active_src_dst_ll_en()
311 static inline void aes_a_dma_reset_and_activate_perf_cntr(const struct ocs_aes_dev *aes_dev) in aes_a_dma_reset_and_activate_perf_cntr()
313 iowrite32(0x00000000, aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET); in aes_a_dma_reset_and_activate_perf_cntr()
315 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); in aes_a_dma_reset_and_activate_perf_cntr()
319 static inline void aes_a_dma_wait_and_deactivate_perf_cntr(const struct ocs_aes_dev *aes_dev, in aes_a_dma_wait_and_deactivate_perf_cntr()
322 while (ioread32(aes_dev->base_reg + AES_A_DMA_PERF_CNTR_OFFSET) < delay) in aes_a_dma_wait_and_deactivate_perf_cntr()
325 aes_dev->base_reg + AES_A_DMA_WHILE_ACTIVE_MODE_OFFSET); in aes_a_dma_wait_and_deactivate_perf_cntr()
335 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); in aes_irq_disable()
336 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); in aes_irq_disable()
339 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); in aes_irq_disable()
342 aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); in aes_irq_disable()
344 isr_val = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); in aes_irq_disable()
347 aes_dev->base_reg + AES_A_DMA_MSI_MASK_OFFSET); in aes_irq_disable()
349 isr_val = ioread32(aes_dev->base_reg + AES_ISR_OFFSET); in aes_irq_disable()
351 iowrite32(isr_val, aes_dev->base_reg + AES_ISR_OFFSET); in aes_irq_disable()
366 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); in aes_irq_enable()
370 * bits [31:3] - reserved in aes_irq_enable()
371 * bit [2] - EN_SKS_ERR in aes_irq_enable()
372 * bit [1] - EN_AES_COMPLETE in aes_irq_enable()
373 * bit [0] - reserved in aes_irq_enable()
375 iowrite32(AES_COMPLETE_INT, aes_dev->base_reg + AES_IER_OFFSET); in aes_irq_enable()
380 iowrite32(AES_DISABLE_INT, aes_dev->base_reg + AES_IER_OFFSET); in aes_irq_enable()
384 * bits [31:9] - reserved in aes_irq_enable()
385 * bit [8] - CPD_ERR_INT_EN in aes_irq_enable()
386 * bit [7] - OUTBUF_RD_ERR_INT_EN in aes_irq_enable()
387 * bit [6] - OUTBUF_WR_ERR_INT_EN in aes_irq_enable()
388 * bit [5] - INBUF_RD_ERR_INT_EN in aes_irq_enable()
389 * bit [4] - INBUF_WR_ERR_INT_EN in aes_irq_enable()
390 * bit [3] - BAD_COMP_INT_EN in aes_irq_enable()
391 * bit [2] - SAI_INT_EN in aes_irq_enable()
392 * bit [1] - DST_DONE_INT_EN in aes_irq_enable()
393 * bit [0] - SRC_DONE_INT_EN in aes_irq_enable()
403 aes_dev->base_reg + AES_A_DMA_MSI_IER_OFFSET); in aes_irq_enable()
407 /* Enable and wait for IRQ (either from OCS AES engine or DMA) */
412 reinit_completion(&aes_dev->irq_completion); in ocs_aes_irq_enable_and_wait()
414 rc = wait_for_completion_interruptible(&aes_dev->irq_completion); in ocs_aes_irq_enable_and_wait()
418 return aes_dev->dma_err_mask ? -EIO : 0; in ocs_aes_irq_enable_and_wait()
422 static inline void dma_to_ocs_aes_ll(struct ocs_aes_dev *aes_dev, in dma_to_ocs_aes_ll()
425 iowrite32(0, aes_dev->base_reg + AES_A_DMA_SRC_SIZE_OFFSET); in dma_to_ocs_aes_ll()
427 aes_dev->base_reg + AES_A_DMA_NEXT_SRC_DESCR_OFFSET); in dma_to_ocs_aes_ll()
431 static inline void dma_from_ocs_aes_ll(struct ocs_aes_dev *aes_dev, in dma_from_ocs_aes_ll()
434 iowrite32(0, aes_dev->base_reg + AES_A_DMA_DST_SIZE_OFFSET); in dma_from_ocs_aes_ll()
436 aes_dev->base_reg + AES_A_DMA_NEXT_DST_DESCR_OFFSET); in dma_from_ocs_aes_ll()
445 aes_dma_isr = ioread32(aes_dev->base_reg + AES_A_DMA_MSI_ISR_OFFSET); in ocs_aes_irq_handler()
451 aes_dev->dma_err_mask = aes_dma_isr & in ocs_aes_irq_handler()
461 complete(&aes_dev->irq_completion); in ocs_aes_irq_handler()
467 * ocs_aes_set_key() - Write key into OCS AES hardware.
484 /* OCS AES supports 128-bit and 256-bit keys only. */ in ocs_aes_set_key()
486 dev_err(aes_dev->dev, in ocs_aes_set_key()
487 "%d-bit keys not supported by AES cipher\n", in ocs_aes_set_key()
489 return -EINVAL; in ocs_aes_set_key()
491 /* OCS SM4 supports 128-bit keys only. */ in ocs_aes_set_key()
493 dev_err(aes_dev->dev, in ocs_aes_set_key()
494 "%d-bit keys not supported for SM4 cipher\n", in ocs_aes_set_key()
496 return -EINVAL; in ocs_aes_set_key()
500 return -EINVAL; in ocs_aes_set_key()
504 /* Write key to AES_KEY[0-7] registers */ in ocs_aes_set_key()
507 aes_dev->base_reg + AES_KEY_0_OFFSET + in ocs_aes_set_key()
512 * bits [31:1] - reserved in ocs_aes_set_key()
513 * bit [0] - AES_KEY_SIZE in ocs_aes_set_key()
514 * 0 - 128 bit key in ocs_aes_set_key()
515 * 1 - 256 bit key in ocs_aes_set_key()
518 iowrite32(val, aes_dev->base_reg + AES_KEY_SIZE_OFFSET); in ocs_aes_set_key()
524 static inline void set_ocs_aes_command(struct ocs_aes_dev *aes_dev, in set_ocs_aes_command()
533 * bit [14] - CIPHER_SELECT in set_ocs_aes_command()
534 * 0 - AES in set_ocs_aes_command()
535 * 1 - SM4 in set_ocs_aes_command()
536 * bits [11:8] - OCS_AES_MODE in set_ocs_aes_command()
537 * 0000 - ECB in set_ocs_aes_command()
538 * 0001 - CBC in set_ocs_aes_command()
539 * 0010 - CTR in set_ocs_aes_command()
540 * 0110 - CCM in set_ocs_aes_command()
541 * 0111 - GCM in set_ocs_aes_command()
542 * 1001 - CTS in set_ocs_aes_command()
543 * bits [7:6] - AES_INSTRUCTION in set_ocs_aes_command()
544 * 00 - ENCRYPT in set_ocs_aes_command()
545 * 01 - DECRYPT in set_ocs_aes_command()
546 * 10 - EXPAND in set_ocs_aes_command()
547 * 11 - BYPASS in set_ocs_aes_command()
548 * bits [3:2] - CTR_M_BITS in set_ocs_aes_command()
549 * 00 - No increment in set_ocs_aes_command()
550 * 01 - Least significant 32 bits are incremented in set_ocs_aes_command()
551 * 10 - Least significant 64 bits are incremented in set_ocs_aes_command()
552 * 11 - Full 128 bits are incremented in set_ocs_aes_command()
556 iowrite32(val, aes_dev->base_reg + AES_COMMAND_OFFSET); in set_ocs_aes_command()
567 /* Set endianness recommended by data-sheet. */ in ocs_aes_init()
578 static inline void ocs_aes_write_last_data_blk_len(struct ocs_aes_dev *aes_dev, in ocs_aes_write_last_data_blk_len()
593 iowrite32(val, aes_dev->base_reg + AES_PLEN_OFFSET); in ocs_aes_write_last_data_blk_len()
598 * If OK return 0; else return -EINVAL.
610 return -EINVAL; in ocs_aes_validate_inputs()
615 return -EINVAL; in ocs_aes_validate_inputs()
619 return -EINVAL; in ocs_aes_validate_inputs()
626 * linked-list must be defined. in ocs_aes_validate_inputs()
631 return -EINVAL; in ocs_aes_validate_inputs()
644 return -EINVAL; in ocs_aes_validate_inputs()
649 return -EINVAL; in ocs_aes_validate_inputs()
656 return -EINVAL; in ocs_aes_validate_inputs()
661 return -EINVAL; in ocs_aes_validate_inputs()
665 return -EINVAL; in ocs_aes_validate_inputs()
672 return -EINVAL; in ocs_aes_validate_inputs()
677 return -EINVAL; in ocs_aes_validate_inputs()
681 return -EINVAL; in ocs_aes_validate_inputs()
688 return -EINVAL; in ocs_aes_validate_inputs()
693 return -EINVAL; in ocs_aes_validate_inputs()
697 return -EINVAL; in ocs_aes_validate_inputs()
704 return -EINVAL; in ocs_aes_validate_inputs()
712 return -EINVAL; in ocs_aes_validate_inputs()
716 return -EINVAL; in ocs_aes_validate_inputs()
720 return -EINVAL; in ocs_aes_validate_inputs()
724 return -EINVAL; in ocs_aes_validate_inputs()
731 return -EINVAL; in ocs_aes_validate_inputs()
736 return -EINVAL; in ocs_aes_validate_inputs()
740 return -EINVAL; in ocs_aes_validate_inputs()
744 return -EINVAL; in ocs_aes_validate_inputs()
753 return -EINVAL; in ocs_aes_validate_inputs()
757 return -EINVAL; in ocs_aes_validate_inputs()
769 return -EINVAL; in ocs_aes_validate_inputs()
773 return -EINVAL; in ocs_aes_validate_inputs()
778 return -EINVAL; in ocs_aes_validate_inputs()
783 * ocs_aes_op() - Perform AES/SM4 operation.
819 return -EINVAL; in ocs_aes_op()
827 /* Write the byte length of the last data block to engine. */ in ocs_aes_op()
833 iowrite32(iv32[0], aes_dev->base_reg + AES_IV_0_OFFSET); in ocs_aes_op()
834 iowrite32(iv32[1], aes_dev->base_reg + AES_IV_1_OFFSET); in ocs_aes_op()
835 iowrite32(iv32[2], aes_dev->base_reg + AES_IV_2_OFFSET); in ocs_aes_op()
836 iowrite32(iv32[3], aes_dev->base_reg + AES_IV_3_OFFSET); in ocs_aes_op()
849 * For CTS mode, instruct engine to activate ciphertext in ocs_aes_op()
858 /* Wait for engine to complete processing. */ in ocs_aes_op()
865 iv32[0] = ioread32(aes_dev->base_reg + AES_IV_0_OFFSET); in ocs_aes_op()
866 iv32[1] = ioread32(aes_dev->base_reg + AES_IV_1_OFFSET); in ocs_aes_op()
867 iv32[2] = ioread32(aes_dev->base_reg + AES_IV_2_OFFSET); in ocs_aes_op()
868 iv32[3] = ioread32(aes_dev->base_reg + AES_IV_3_OFFSET); in ocs_aes_op()
874 /* Compute and write J0 to engine registers. */
881 * IV must be 12 bytes; Other sizes not supported as Linux crypto API in ocs_aes_gcm_write_j0()
884 iowrite32(0x00000001, aes_dev->base_reg + AES_IV_0_OFFSET); in ocs_aes_gcm_write_j0()
885 iowrite32(__swab32(j0[2]), aes_dev->base_reg + AES_IV_1_OFFSET); in ocs_aes_gcm_write_j0()
886 iowrite32(__swab32(j0[1]), aes_dev->base_reg + AES_IV_2_OFFSET); in ocs_aes_gcm_write_j0()
887 iowrite32(__swab32(j0[0]), aes_dev->base_reg + AES_IV_3_OFFSET); in ocs_aes_gcm_write_j0()
890 /* Read GCM tag from engine registers. */
891 static inline void ocs_aes_gcm_read_tag(struct ocs_aes_dev *aes_dev, in ocs_aes_gcm_read_tag()
901 tag_u32[0] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_3_OFFSET)); in ocs_aes_gcm_read_tag()
902 tag_u32[1] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_2_OFFSET)); in ocs_aes_gcm_read_tag()
903 tag_u32[2] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_1_OFFSET)); in ocs_aes_gcm_read_tag()
904 tag_u32[3] = __swab32(ioread32(aes_dev->base_reg + AES_T_MAC_0_OFFSET)); in ocs_aes_gcm_read_tag()
910 * ocs_aes_gcm_op() - Perform GCM operation.
955 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); in ocs_aes_gcm_op()
963 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_0_OFFSET); in ocs_aes_gcm_op()
965 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_1_OFFSET); in ocs_aes_gcm_op()
970 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_2_OFFSET); in ocs_aes_gcm_op()
972 iowrite32(val, aes_dev->base_reg + AES_MULTIPURPOSE2_3_OFFSET); in ocs_aes_gcm_op()
979 /* If aad present, configure DMA to feed it to the engine. */ in ocs_aes_gcm_op()
983 /* Instructs engine to pad last block of aad, if needed. */ in ocs_aes_gcm_op()
1009 /* Instruct AES/SMA4 engine payload processing is over. */ in ocs_aes_gcm_op()
1012 /* Wait for OCS AES engine to complete processing. */ in ocs_aes_gcm_op()
1022 /* Write encrypted tag to AES/SM4 engine. */
1040 /* Write encrypted tag to AES/SM4 engine. */ in ocs_aes_ccm_write_encrypted_tag()
1042 iowrite8(in_tag[i], aes_dev->base_reg + in ocs_aes_ccm_write_encrypted_tag()
1050 * Note: B0 format is documented in NIST Special Publication 800-38C
1051 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1068 * bit 5-3: t value encoded as (t-2)/2 in ocs_aes_ccm_write_b0()
1069 * bit 2-0: q value encoded as q - 1 in ocs_aes_ccm_write_b0()
1077 * encoded as (t - 2) / 2 in ocs_aes_ccm_write_b0()
1079 b0[0] |= (((tag_size - 2) / 2) & 0x7) << 3; in ocs_aes_ccm_write_b0()
1083 * q - 1 == iv[0] & 0x7; in ocs_aes_ccm_write_b0()
1087 * Copy the Nonce N from IV to B0; N is located in iv[1]..iv[15 - q] in ocs_aes_ccm_write_b0()
1088 * and must be copied to b0[1]..b0[15-q]. in ocs_aes_ccm_write_b0()
1092 for (i = 1; i <= 15 - q; i++) in ocs_aes_ccm_write_b0()
1096 * Q is encoded in q octets, in big-endian order, so to write it, we in ocs_aes_ccm_write_b0()
1099 i = sizeof(b0) - 1; in ocs_aes_ccm_write_b0()
1103 i--; in ocs_aes_ccm_write_b0()
1104 q--; in ocs_aes_ccm_write_b0()
1111 return -EOVERFLOW; in ocs_aes_ccm_write_b0()
1114 iowrite8(b0[i], aes_dev->base_reg + in ocs_aes_ccm_write_b0()
1122 * Note: adata len encoding is documented in NIST Special Publication 800-38C
1123 * https://nvlpubs.nist.gov/nistpubs/Legacy/SP/nistspecialpublication800-38c.pdf
1134 * If 0 < a < 2^16 - 2^8 ==> 'a' encoded as [a]16, i.e., two octets in ocs_aes_ccm_write_adata_len()
1136 * If 2^16 - 2^8 ≤ a < 2^32 ==> 'a' encoded as 0xff || 0xfe || [a]32, in ocs_aes_ccm_write_adata_len()
1155 aes_dev->base_reg + in ocs_aes_ccm_write_adata_len()
1221 * AES/SM4 engine to pad the last block of data. in ocs_aes_ccm_encrypt_do_payload()
1235 /* Let engine process 0-length input. */ in ocs_aes_ccm_decrypt_do_payload()
1252 * AES/SM4 engine to differentiate between encrypted data and in ocs_aes_ccm_decrypt_do_payload()
1269 static inline int ccm_compare_tag_to_yr(struct ocs_aes_dev *aes_dev, in ccm_compare_tag_to_yr()
1278 tag[i] = ioread32(aes_dev->base_reg + in ccm_compare_tag_to_yr()
1280 yr[i] = ioread32(aes_dev->base_reg + in ccm_compare_tag_to_yr()
1285 return memcmp(tag, yr, tag_size_bytes) ? -EBADMSG : 0; in ccm_compare_tag_to_yr()
1289 * ocs_aes_ccm_op() - Perform CCM operation.
1297 * @adata_dma_list: The OCS DMA list mapping input A-data.
1333 * Note: rfc 3610 and NIST 800-38C require counter of zero to encrypt in ocs_aes_ccm_op()
1345 aes_dev->base_reg + AES_MULTIPURPOSE1_3_OFFSET); in ocs_aes_ccm_op()
1347 aes_dev->base_reg + AES_MULTIPURPOSE1_2_OFFSET); in ocs_aes_ccm_op()
1349 aes_dev->base_reg + AES_MULTIPURPOSE1_1_OFFSET); in ocs_aes_ccm_op()
1351 aes_dev->base_reg + AES_MULTIPURPOSE1_0_OFFSET); in ocs_aes_ccm_op()
1354 iowrite32(tag_size, aes_dev->base_reg + AES_TLEN_OFFSET); in ocs_aes_ccm_op()
1392 /* Process MAC/tag directly: feed tag to engine and wait for IRQ. */ in ocs_aes_ccm_op()
1402 * ocs_create_linked_list_from_sg() - Create OCS DMA linked list from SG list.
1407 * @sg_dma_count: The number of DMA-mapped entries in @sg. This must be the
1430 return -EINVAL; in ocs_create_linked_list_from_sg()
1433 dll_desc->vaddr = NULL; in ocs_create_linked_list_from_sg()
1434 dll_desc->dma_addr = DMA_MAPPING_ERROR; in ocs_create_linked_list_from_sg()
1435 dll_desc->size = 0; in ocs_create_linked_list_from_sg()
1442 data_offset -= sg_dma_len(sg); in ocs_create_linked_list_from_sg()
1443 sg_dma_count--; in ocs_create_linked_list_from_sg()
1447 return -EINVAL; in ocs_create_linked_list_from_sg()
1450 /* Compute number of DMA-mapped SG entries to add into OCS DMA list. */ in ocs_create_linked_list_from_sg()
1457 return -EINVAL; in ocs_create_linked_list_from_sg()
1463 return -EINVAL; in ocs_create_linked_list_from_sg()
1466 dll_desc->size = sizeof(struct ocs_dma_linked_list) * dma_nents; in ocs_create_linked_list_from_sg()
1467 dll_desc->vaddr = dma_alloc_coherent(aes_dev->dev, dll_desc->size, in ocs_create_linked_list_from_sg()
1468 &dll_desc->dma_addr, GFP_KERNEL); in ocs_create_linked_list_from_sg()
1469 if (!dll_desc->vaddr) in ocs_create_linked_list_from_sg()
1470 return -ENOMEM; in ocs_create_linked_list_from_sg()
1473 ll = dll_desc->vaddr; in ocs_create_linked_list_from_sg()
1476 ll[i].src_len = (sg_dma_len(sg) - data_offset) < data_size ? in ocs_create_linked_list_from_sg()
1477 (sg_dma_len(sg) - data_offset) : data_size; in ocs_create_linked_list_from_sg()
1479 data_size -= ll[i].src_len; in ocs_create_linked_list_from_sg()
1481 ll[i].next = dll_desc->dma_addr + (sizeof(*ll) * (i + 1)); in ocs_create_linked_list_from_sg()
1485 ll[i - 1].next = 0; in ocs_create_linked_list_from_sg()
1486 ll[i - 1].ll_flags = OCS_LL_DMA_FLAG_TERMINATE; in ocs_create_linked_list_from_sg()