478 	struct ccp_aes_engine *aes = &cmd->u.aes;  in ccp_run_aes_cmac_cmd()  local
485 	if (!((aes->key_len == AES_KEYSIZE_128) ||  in ccp_run_aes_cmac_cmd()
486 	      (aes->key_len == AES_KEYSIZE_192) ||  in ccp_run_aes_cmac_cmd()
487 	      (aes->key_len == AES_KEYSIZE_256)))  in ccp_run_aes_cmac_cmd()
490 	if (aes->src_len & (AES_BLOCK_SIZE - 1))  in ccp_run_aes_cmac_cmd()
493 	if (aes->iv_len != AES_BLOCK_SIZE)  in ccp_run_aes_cmac_cmd()
496 	if (!aes->key || !aes->iv || !aes->src)  in ccp_run_aes_cmac_cmd()
499 	if (aes->cmac_final) {  in ccp_run_aes_cmac_cmd()
500 		if (aes->cmac_key_len != AES_BLOCK_SIZE)  in ccp_run_aes_cmac_cmd()
503 		if (!aes->cmac_key)  in ccp_run_aes_cmac_cmd()
517 	op.u.aes.type = aes->type;  in ccp_run_aes_cmac_cmd()
518 	op.u.aes.mode = aes->mode;  in ccp_run_aes_cmac_cmd()
519 	op.u.aes.action = aes->action;  in ccp_run_aes_cmac_cmd()
532 	dm_offset = CCP_SB_BYTES - aes->key_len;  in ccp_run_aes_cmac_cmd()
533 	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);  in ccp_run_aes_cmac_cmd()
543 	/* The AES context fits in a single (32-byte) SB entry and  in ccp_run_aes_cmac_cmd()
554 	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmac_cmd()
564 	/* Send data to the CCP AES engine */  in ccp_run_aes_cmac_cmd()
565 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmac_cmd()
572 		if (aes->cmac_final && !src.sg_wa.bytes_left) {  in ccp_run_aes_cmac_cmd()
584 			ret = ccp_set_dm_area(&ctx, 0, aes->cmac_key, 0,  in ccp_run_aes_cmac_cmd()
585 					      aes->cmac_key_len);  in ccp_run_aes_cmac_cmd()
596 		ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_cmac_cmd()
605 	/* Retrieve the AES context - convert from LE to BE using  in ccp_run_aes_cmac_cmd()
617 	ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmac_cmd()
634 	struct ccp_aes_engine *aes = &cmd->u.aes;  in ccp_run_aes_gcm_cmd()  local
652 	if (!aes->iv)  in ccp_run_aes_gcm_cmd()
655 	if (!((aes->key_len == AES_KEYSIZE_128) ||  in ccp_run_aes_gcm_cmd()
656 		(aes->key_len == AES_KEYSIZE_192) ||  in ccp_run_aes_gcm_cmd()
657 		(aes->key_len == AES_KEYSIZE_256)))  in ccp_run_aes_gcm_cmd()
660 	if (!aes->key) /* Gotta have a key SGL */  in ccp_run_aes_gcm_cmd()
664 	authsize = aes->authsize ? aes->authsize : AES_BLOCK_SIZE;  in ccp_run_aes_gcm_cmd()
684 	p_aad = aes->src;  in ccp_run_aes_gcm_cmd()
685 	p_inp = scatterwalk_ffwd(sg_inp, aes->src, aes->aad_len);  in ccp_run_aes_gcm_cmd()
686 	p_outp = scatterwalk_ffwd(sg_outp, aes->dst, aes->aad_len);  in ccp_run_aes_gcm_cmd()
687 	if (aes->action == CCP_AES_ACTION_ENCRYPT) {  in ccp_run_aes_gcm_cmd()
688 		ilen = aes->src_len;  in ccp_run_aes_gcm_cmd()
692 		ilen = aes->src_len - authsize;  in ccp_run_aes_gcm_cmd()
704 	op.u.aes.type = aes->type;  in ccp_run_aes_gcm_cmd()
713 	dm_offset = CCP_SB_BYTES - aes->key_len;  in ccp_run_aes_gcm_cmd()
714 	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);  in ccp_run_aes_gcm_cmd()
734 	dm_offset = CCP_AES_CTX_SB_COUNT * CCP_SB_BYTES - aes->iv_len;  in ccp_run_aes_gcm_cmd()
735 	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_gcm_cmd()
747 	if (aes->aad_len > 0) {  in ccp_run_aes_gcm_cmd()
749 		ret = ccp_init_data(&aad, cmd_q, p_aad, aes->aad_len,  in ccp_run_aes_gcm_cmd()
755 		op.u.aes.mode = CCP_AES_MODE_GHASH;  in ccp_run_aes_gcm_cmd()
756 		op.u.aes.action = CCP_AES_GHASHAAD;  in ccp_run_aes_gcm_cmd()
761 			ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_gcm_cmd()
772 	op.u.aes.mode = CCP_AES_MODE_GCTR;  in ccp_run_aes_gcm_cmd()
773 	op.u.aes.action = aes->action;  in ccp_run_aes_gcm_cmd()
805 					op.u.aes.size = (nbytes * 8) - 1;  in ccp_run_aes_gcm_cmd()
809 			ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_gcm_cmd()
828 	ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_gcm_cmd()
847 	final[0] = cpu_to_be64(aes->aad_len * 8);  in ccp_run_aes_gcm_cmd()
856 	op.u.aes.type = aes->type;  in ccp_run_aes_gcm_cmd()
857 	op.u.aes.mode = CCP_AES_MODE_GHASH;  in ccp_run_aes_gcm_cmd()
858 	op.u.aes.action = CCP_AES_GHASHFINAL;  in ccp_run_aes_gcm_cmd()
866 	op.u.aes.size = 0;  in ccp_run_aes_gcm_cmd()
867 	ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_gcm_cmd()
871 	if (aes->action == CCP_AES_ACTION_ENCRYPT) {  in ccp_run_aes_gcm_cmd()
903 	if (aes->aad_len)  in ccp_run_aes_gcm_cmd()
918 	struct ccp_aes_engine *aes = &cmd->u.aes;  in ccp_run_aes_cmd()  local
926 	if (!((aes->key_len == AES_KEYSIZE_128) ||  in ccp_run_aes_cmd()
927 	      (aes->key_len == AES_KEYSIZE_192) ||  in ccp_run_aes_cmd()
928 	      (aes->key_len == AES_KEYSIZE_256)))  in ccp_run_aes_cmd()
931 	if (((aes->mode == CCP_AES_MODE_ECB) ||  in ccp_run_aes_cmd()
932 	     (aes->mode == CCP_AES_MODE_CBC)) &&  in ccp_run_aes_cmd()
933 	    (aes->src_len & (AES_BLOCK_SIZE - 1)))  in ccp_run_aes_cmd()
936 	if (!aes->key || !aes->src || !aes->dst)  in ccp_run_aes_cmd()
939 	if (aes->mode != CCP_AES_MODE_ECB) {  in ccp_run_aes_cmd()
940 		if (aes->iv_len != AES_BLOCK_SIZE)  in ccp_run_aes_cmd()
943 		if (!aes->iv)  in ccp_run_aes_cmd()
956 	op.init = (aes->mode == CCP_AES_MODE_ECB) ? 0 : 1;  in ccp_run_aes_cmd()
957 	op.u.aes.type = aes->type;  in ccp_run_aes_cmd()
958 	op.u.aes.mode = aes->mode;  in ccp_run_aes_cmd()
959 	op.u.aes.action = aes->action;  in ccp_run_aes_cmd()
972 	dm_offset = CCP_SB_BYTES - aes->key_len;  in ccp_run_aes_cmd()
973 	ret = ccp_set_dm_area(&key, dm_offset, aes->key, 0, aes->key_len);  in ccp_run_aes_cmd()
983 	/* The AES context fits in a single (32-byte) SB entry and  in ccp_run_aes_cmd()
993 	if (aes->mode != CCP_AES_MODE_ECB) {  in ccp_run_aes_cmd()
994 		/* Load the AES context - convert to LE */  in ccp_run_aes_cmd()
996 		ret = ccp_set_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmd()
1006 	switch (aes->mode) {  in ccp_run_aes_cmd()
1009 		op.u.aes.size = AES_BLOCK_SIZE * BITS_PER_BYTE - 1;  in ccp_run_aes_cmd()
1012 		op.u.aes.size = 0;  in ccp_run_aes_cmd()
1019 	if (sg_virt(aes->src) == sg_virt(aes->dst))  in ccp_run_aes_cmd()
1022 	ret = ccp_init_data(&src, cmd_q, aes->src, aes->src_len,  in ccp_run_aes_cmd()
1031 		ret = ccp_init_data(&dst, cmd_q, aes->dst, aes->src_len,  in ccp_run_aes_cmd()
1037 	/* Send data to the CCP AES engine */  in ccp_run_aes_cmd()
1043 			/* Since we don't retrieve the AES context in ECB  in ccp_run_aes_cmd()
1047 			if (aes->mode == CCP_AES_MODE_ECB)  in ccp_run_aes_cmd()
1051 		ret = cmd_q->ccp->vdata->perform->aes(&op);  in ccp_run_aes_cmd()
1060 	if (aes->mode != CCP_AES_MODE_ECB) {  in ccp_run_aes_cmd()
1061 		/* Retrieve the AES context - convert from LE to BE using  in ccp_run_aes_cmd()
1073 		ccp_get_dm_area(&ctx, dm_offset, aes->iv, 0, aes->iv_len);  in ccp_run_aes_cmd()
1205 	/* The AES context fits in a single (32-byte) SB entry and  in ccp_run_xts_aes_cmd()
1247 	/* Send data to the CCP AES engine */  in ccp_run_xts_aes_cmd()
1262 	/* Retrieve the AES context - convert from LE to BE using  in ccp_run_xts_aes_cmd()
1346 	 * (like AES) must be in little endian format. Use the 256-bit byte  in ccp_run_des3_cmd()
2477 		switch (cmd->u.aes.mode) {  in ccp_run_cmd()