43 #define _calc_walked(inout) (dd->inout##_walk.offset - dd->inout##_sg->offset)
45 #define DES_REG_KEY(dd, x)		((dd)->pdata->key_ofs - \  argument
48 #define DES_REG_IV(dd, x)		((dd)->pdata->iv_ofs + ((x) * 0x04))  argument
50 #define DES_REG_CTRL(dd)		((dd)->pdata->ctrl_ofs)  argument
57 #define DES_REG_DATA_N(dd, x)		((dd)->pdata->data_ofs + ((x) * 0x04))  argument
59 #define DES_REG_REV(dd)			((dd)->pdata->rev_ofs)  argument
61 #define DES_REG_MASK(dd)		((dd)->pdata->mask_ofs)  argument
65 #define DES_REG_IRQ_STATUS(dd)         ((dd)->pdata->irq_status_ofs)  argument
66 #define DES_REG_IRQ_ENABLE(dd)         ((dd)->pdata->irq_enable_ofs)  argument
82 	struct omap_des_dev *dd;  member
106 	void		(*trigger)(struct omap_des_dev *dd, int length);
170 #define omap_des_read(dd, offset)                               \  argument
173 	 _read_ret = __raw_readl(dd->io_base + offset);          \
179 static inline u32 omap_des_read(struct omap_des_dev *dd, u32 offset)  in omap_des_read()  argument
181 	return __raw_readl(dd->io_base + offset);  in omap_des_read()
186 #define omap_des_write(dd, offset, value)                               \  argument
190 		__raw_writel(value, dd->io_base + offset);              \
193 static inline void omap_des_write(struct omap_des_dev *dd, u32 offset,  in omap_des_write()  argument
196 	__raw_writel(value, dd->io_base + offset);  in omap_des_write()
200 static inline void omap_des_write_mask(struct omap_des_dev *dd, u32 offset,  in omap_des_write_mask()  argument
205 	val = omap_des_read(dd, offset);  in omap_des_write_mask()
208 	omap_des_write(dd, offset, val);  in omap_des_write_mask()
211 static void omap_des_write_n(struct omap_des_dev *dd, u32 offset,  in omap_des_write_n()  argument
215 		omap_des_write(dd, offset, *value);  in omap_des_write_n()
218 static int omap_des_hw_init(struct omap_des_dev *dd)  in omap_des_hw_init()  argument
227 	err = pm_runtime_resume_and_get(dd->dev);  in omap_des_hw_init()
229 		dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);  in omap_des_hw_init()
233 	if (!(dd->flags & FLAGS_INIT)) {  in omap_des_hw_init()
234 		dd->flags |= FLAGS_INIT;  in omap_des_hw_init()
235 		dd->err = 0;  in omap_des_hw_init()
241 static int omap_des_write_ctrl(struct omap_des_dev *dd)  in omap_des_write_ctrl()  argument
247 	err = omap_des_hw_init(dd);  in omap_des_write_ctrl()
251 	key32 = dd->ctx->keylen / sizeof(u32);  in omap_des_write_ctrl()
255 		omap_des_write(dd, DES_REG_KEY(dd, i),  in omap_des_write_ctrl()
256 			       __le32_to_cpu(dd->ctx->key[i]));  in omap_des_write_ctrl()
259 	if ((dd->flags & FLAGS_CBC) && dd->req->iv)  in omap_des_write_ctrl()
260 		omap_des_write_n(dd, DES_REG_IV(dd, 0), (void *)dd->req->iv, 2);  in omap_des_write_ctrl()
262 	if (dd->flags & FLAGS_CBC)  in omap_des_write_ctrl()
264 	if (dd->flags & FLAGS_ENCRYPT)  in omap_des_write_ctrl()
271 	omap_des_write_mask(dd, DES_REG_CTRL(dd), val, mask);  in omap_des_write_ctrl()
276 static void omap_des_dma_trigger_omap4(struct omap_des_dev *dd, int length)  in omap_des_dma_trigger_omap4()  argument
280 	omap_des_write(dd, DES_REG_LENGTH_N(0), length);  in omap_des_dma_trigger_omap4()
282 	val = dd->pdata->dma_start;  in omap_des_dma_trigger_omap4()
284 	if (dd->dma_lch_out != NULL)  in omap_des_dma_trigger_omap4()
285 		val |= dd->pdata->dma_enable_out;  in omap_des_dma_trigger_omap4()
286 	if (dd->dma_lch_in != NULL)  in omap_des_dma_trigger_omap4()
287 		val |= dd->pdata->dma_enable_in;  in omap_des_dma_trigger_omap4()
289 	mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |  in omap_des_dma_trigger_omap4()
290 	       dd->pdata->dma_start;  in omap_des_dma_trigger_omap4()
292 	omap_des_write_mask(dd, DES_REG_MASK(dd), val, mask);  in omap_des_dma_trigger_omap4()
295 static void omap_des_dma_stop(struct omap_des_dev *dd)  in omap_des_dma_stop()  argument
299 	mask = dd->pdata->dma_enable_out | dd->pdata->dma_enable_in |  in omap_des_dma_stop()
300 	       dd->pdata->dma_start;  in omap_des_dma_stop()
302 	omap_des_write_mask(dd, DES_REG_MASK(dd), 0, mask);  in omap_des_dma_stop()
307 	struct omap_des_dev *dd = NULL, *tmp;  in omap_des_find_dev()  local
310 	if (!ctx->dd) {  in omap_des_find_dev()
313 			dd = tmp;  in omap_des_find_dev()
316 		ctx->dd = dd;  in omap_des_find_dev()
319 		dd = ctx->dd;  in omap_des_find_dev()
323 	return dd;  in omap_des_find_dev()
328 	struct omap_des_dev *dd = data;  in omap_des_dma_out_callback()  local
331 	tasklet_schedule(&dd->done_task);  in omap_des_dma_out_callback()
334 static int omap_des_dma_init(struct omap_des_dev *dd)  in omap_des_dma_init()  argument
338 	dd->dma_lch_out = NULL;  in omap_des_dma_init()
339 	dd->dma_lch_in = NULL;  in omap_des_dma_init()
341 	dd->dma_lch_in = dma_request_chan(dd->dev, "rx");  in omap_des_dma_init()
342 	if (IS_ERR(dd->dma_lch_in)) {  in omap_des_dma_init()
343 		dev_err(dd->dev, "Unable to request in DMA channel\n");  in omap_des_dma_init()
344 		return PTR_ERR(dd->dma_lch_in);  in omap_des_dma_init()
347 	dd->dma_lch_out = dma_request_chan(dd->dev, "tx");  in omap_des_dma_init()
348 	if (IS_ERR(dd->dma_lch_out)) {  in omap_des_dma_init()
349 		dev_err(dd->dev, "Unable to request out DMA channel\n");  in omap_des_dma_init()
350 		err = PTR_ERR(dd->dma_lch_out);  in omap_des_dma_init()
357 	dma_release_channel(dd->dma_lch_in);  in omap_des_dma_init()
362 static void omap_des_dma_cleanup(struct omap_des_dev *dd)  in omap_des_dma_cleanup()  argument
364 	if (dd->pio_only)  in omap_des_dma_cleanup()
367 	dma_release_channel(dd->dma_lch_out);  in omap_des_dma_cleanup()
368 	dma_release_channel(dd->dma_lch_in);  in omap_des_dma_cleanup()
376 	struct omap_des_dev *dd = ctx->dd;  in omap_des_crypt_dma()  local
381 	if (dd->pio_only) {  in omap_des_crypt_dma()
382 		scatterwalk_start(&dd->in_walk, dd->in_sg);  in omap_des_crypt_dma()
383 		scatterwalk_start(&dd->out_walk, dd->out_sg);  in omap_des_crypt_dma()
387 		omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2);  in omap_des_crypt_dma()
391 	dma_sync_sg_for_device(dd->dev, dd->in_sg, in_sg_len, DMA_TO_DEVICE);  in omap_des_crypt_dma()
395 	cfg.src_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);  in omap_des_crypt_dma()
396 	cfg.dst_addr = dd->phys_base + DES_REG_DATA_N(dd, 0);  in omap_des_crypt_dma()
403 	ret = dmaengine_slave_config(dd->dma_lch_in, &cfg);  in omap_des_crypt_dma()
405 		dev_err(dd->dev, "can't configure IN dmaengine slave: %d\n",  in omap_des_crypt_dma()
410 	tx_in = dmaengine_prep_slave_sg(dd->dma_lch_in, in_sg, in_sg_len,  in omap_des_crypt_dma()
414 		dev_err(dd->dev, "IN prep_slave_sg() failed\n");  in omap_des_crypt_dma()
419 	tx_in->callback_param = dd;  in omap_des_crypt_dma()
422 	ret = dmaengine_slave_config(dd->dma_lch_out, &cfg);  in omap_des_crypt_dma()
424 		dev_err(dd->dev, "can't configure OUT dmaengine slave: %d\n",  in omap_des_crypt_dma()
429 	tx_out = dmaengine_prep_slave_sg(dd->dma_lch_out, out_sg, out_sg_len,  in omap_des_crypt_dma()
433 		dev_err(dd->dev, "OUT prep_slave_sg() failed\n");  in omap_des_crypt_dma()
438 	tx_out->callback_param = dd;  in omap_des_crypt_dma()
443 	dma_async_issue_pending(dd->dma_lch_in);  in omap_des_crypt_dma()
444 	dma_async_issue_pending(dd->dma_lch_out);  in omap_des_crypt_dma()
447 	dd->pdata->trigger(dd, dd->total);  in omap_des_crypt_dma()
452 static int omap_des_crypt_dma_start(struct omap_des_dev *dd)  in omap_des_crypt_dma_start()  argument
455 					crypto_skcipher_reqtfm(dd->req));  in omap_des_crypt_dma_start()
458 	pr_debug("total: %zd\n", dd->total);  in omap_des_crypt_dma_start()
460 	if (!dd->pio_only) {  in omap_des_crypt_dma_start()
461 		err = dma_map_sg(dd->dev, dd->in_sg, dd->in_sg_len,  in omap_des_crypt_dma_start()
464 			dev_err(dd->dev, "dma_map_sg() error\n");  in omap_des_crypt_dma_start()
468 		err = dma_map_sg(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_des_crypt_dma_start()
471 			dev_err(dd->dev, "dma_map_sg() error\n");  in omap_des_crypt_dma_start()
476 	err = omap_des_crypt_dma(tfm, dd->in_sg, dd->out_sg, dd->in_sg_len,  in omap_des_crypt_dma_start()
477 				 dd->out_sg_len);  in omap_des_crypt_dma_start()
478 	if (err && !dd->pio_only) {  in omap_des_crypt_dma_start()
479 		dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);  in omap_des_crypt_dma_start()
480 		dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_des_crypt_dma_start()
487 static void omap_des_finish_req(struct omap_des_dev *dd, int err)  in omap_des_finish_req()  argument
489 	struct skcipher_request *req = dd->req;  in omap_des_finish_req()
493 	crypto_finalize_skcipher_request(dd->engine, req, err);  in omap_des_finish_req()
495 	pm_runtime_mark_last_busy(dd->dev);  in omap_des_finish_req()
496 	pm_runtime_put_autosuspend(dd->dev);  in omap_des_finish_req()
499 static int omap_des_crypt_dma_stop(struct omap_des_dev *dd)  in omap_des_crypt_dma_stop()  argument
501 	pr_debug("total: %zd\n", dd->total);  in omap_des_crypt_dma_stop()
503 	omap_des_dma_stop(dd);  in omap_des_crypt_dma_stop()
505 	dmaengine_terminate_all(dd->dma_lch_in);  in omap_des_crypt_dma_stop()
506 	dmaengine_terminate_all(dd->dma_lch_out);  in omap_des_crypt_dma_stop()
511 static int omap_des_handle_queue(struct omap_des_dev *dd,  in omap_des_handle_queue()  argument
515 		return crypto_transfer_skcipher_request_to_engine(dd->engine, req);  in omap_des_handle_queue()
521 				struct omap_des_dev *dd)  in omap_des_prepare_req()  argument
530 	dd->req = req;  in omap_des_prepare_req()
531 	dd->total = req->cryptlen;  in omap_des_prepare_req()
532 	dd->total_save = req->cryptlen;  in omap_des_prepare_req()
533 	dd->in_sg = req->src;  in omap_des_prepare_req()
534 	dd->out_sg = req->dst;  in omap_des_prepare_req()
535 	dd->orig_out = req->dst;  in omap_des_prepare_req()
541 	ret = omap_crypto_align_sg(&dd->in_sg, dd->total, DES_BLOCK_SIZE,  in omap_des_prepare_req()
542 				   &dd->in_sgl, flags,  in omap_des_prepare_req()
543 				   FLAGS_IN_DATA_ST_SHIFT, &dd->flags);  in omap_des_prepare_req()
547 	ret = omap_crypto_align_sg(&dd->out_sg, dd->total, DES_BLOCK_SIZE,  in omap_des_prepare_req()
548 				   &dd->out_sgl, 0,  in omap_des_prepare_req()
549 				   FLAGS_OUT_DATA_ST_SHIFT, &dd->flags);  in omap_des_prepare_req()
553 	dd->in_sg_len = sg_nents_for_len(dd->in_sg, dd->total);  in omap_des_prepare_req()
554 	if (dd->in_sg_len < 0)  in omap_des_prepare_req()
555 		return dd->in_sg_len;  in omap_des_prepare_req()
557 	dd->out_sg_len = sg_nents_for_len(dd->out_sg, dd->total);  in omap_des_prepare_req()
558 	if (dd->out_sg_len < 0)  in omap_des_prepare_req()
559 		return dd->out_sg_len;  in omap_des_prepare_req()
564 	dd->flags = (dd->flags & ~FLAGS_MODE_MASK) | rctx->mode;  in omap_des_prepare_req()
566 	dd->ctx = ctx;  in omap_des_prepare_req()
567 	ctx->dd = dd;  in omap_des_prepare_req()
569 	return omap_des_write_ctrl(dd);  in omap_des_prepare_req()
578 	struct omap_des_dev *dd = omap_des_find_dev(ctx);  in omap_des_crypt_req()  local
580 	if (!dd)  in omap_des_crypt_req()
583 	return omap_des_prepare_req(req, dd) ?:  in omap_des_crypt_req()
584 	       omap_des_crypt_dma_start(dd);  in omap_des_crypt_req()
589 	struct omap_des_dev *dd = (struct omap_des_dev *)data;  in omap_des_done_task()  local
594 	if (!dd->pio_only) {  in omap_des_done_task()
595 		dma_sync_sg_for_device(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_des_done_task()
597 		dma_unmap_sg(dd->dev, dd->in_sg, dd->in_sg_len, DMA_TO_DEVICE);  in omap_des_done_task()
598 		dma_unmap_sg(dd->dev, dd->out_sg, dd->out_sg_len,  in omap_des_done_task()
600 		omap_des_crypt_dma_stop(dd);  in omap_des_done_task()
603 	omap_crypto_cleanup(&dd->in_sgl, NULL, 0, dd->total_save,  in omap_des_done_task()
604 			    FLAGS_IN_DATA_ST_SHIFT, dd->flags);  in omap_des_done_task()
606 	omap_crypto_cleanup(&dd->out_sgl, dd->orig_out, 0, dd->total_save,  in omap_des_done_task()
607 			    FLAGS_OUT_DATA_ST_SHIFT, dd->flags);  in omap_des_done_task()
609 	if ((dd->flags & FLAGS_CBC) && dd->req->iv)  in omap_des_done_task()
611 			((u32 *)dd->req->iv)[i] =  in omap_des_done_task()
612 				omap_des_read(dd, DES_REG_IV(dd, i));  in omap_des_done_task()
614 	omap_des_finish_req(dd, 0);  in omap_des_done_task()
624 	struct omap_des_dev *dd;  in omap_des_crypt()  local
636 	dd = omap_des_find_dev(ctx);  in omap_des_crypt()
637 	if (!dd)  in omap_des_crypt()
642 	return omap_des_handle_queue(dd, req);  in omap_des_crypt()
829 	struct omap_des_dev *dd = dev_id;  in omap_des_irq()  local
833 	status = omap_des_read(dd, DES_REG_IRQ_STATUS(dd));  in omap_des_irq()
835 		omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0);  in omap_des_irq()
837 		BUG_ON(!dd->in_sg);  in omap_des_irq()
839 		BUG_ON(_calc_walked(in) > dd->in_sg->length);  in omap_des_irq()
841 		src = sg_virt(dd->in_sg) + _calc_walked(in);  in omap_des_irq()
844 			omap_des_write(dd, DES_REG_DATA_N(dd, i), *src);  in omap_des_irq()
846 			scatterwalk_advance(&dd->in_walk, 4);  in omap_des_irq()
847 			if (dd->in_sg->length == _calc_walked(in)) {  in omap_des_irq()
848 				dd->in_sg = sg_next(dd->in_sg);  in omap_des_irq()
849 				if (dd->in_sg) {  in omap_des_irq()
850 					scatterwalk_start(&dd->in_walk,  in omap_des_irq()
851 							  dd->in_sg);  in omap_des_irq()
852 					src = sg_virt(dd->in_sg) +  in omap_des_irq()
862 		omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status);  in omap_des_irq()
865 		omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x4);  in omap_des_irq()
868 		omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x0);  in omap_des_irq()
870 		BUG_ON(!dd->out_sg);  in omap_des_irq()
872 		BUG_ON(_calc_walked(out) > dd->out_sg->length);  in omap_des_irq()
874 		dst = sg_virt(dd->out_sg) + _calc_walked(out);  in omap_des_irq()
877 			*dst = omap_des_read(dd, DES_REG_DATA_N(dd, i));  in omap_des_irq()
878 			scatterwalk_advance(&dd->out_walk, 4);  in omap_des_irq()
879 			if (dd->out_sg->length == _calc_walked(out)) {  in omap_des_irq()
880 				dd->out_sg = sg_next(dd->out_sg);  in omap_des_irq()
881 				if (dd->out_sg) {  in omap_des_irq()
882 					scatterwalk_start(&dd->out_walk,  in omap_des_irq()
883 							  dd->out_sg);  in omap_des_irq()
884 					dst = sg_virt(dd->out_sg) +  in omap_des_irq()
892 		BUG_ON(dd->total < DES_BLOCK_SIZE);  in omap_des_irq()
894 		dd->total -= DES_BLOCK_SIZE;  in omap_des_irq()
898 		omap_des_write(dd, DES_REG_IRQ_STATUS(dd), status);  in omap_des_irq()
900 		if (!dd->total)  in omap_des_irq()
902 			tasklet_schedule(&dd->done_task);  in omap_des_irq()
905 			omap_des_write(dd, DES_REG_IRQ_ENABLE(dd), 0x2);  in omap_des_irq()
920 static int omap_des_get_of(struct omap_des_dev *dd,  in omap_des_get_of()  argument
924 	dd->pdata = of_device_get_match_data(&pdev->dev);  in omap_des_get_of()
925 	if (!dd->pdata) {  in omap_des_get_of()
933 static int omap_des_get_of(struct omap_des_dev *dd,  in omap_des_get_of()  argument
940 static int omap_des_get_pdev(struct omap_des_dev *dd,  in omap_des_get_pdev()  argument
944 	dd->pdata = pdev->dev.platform_data;  in omap_des_get_pdev()
952 	struct omap_des_dev *dd;  in omap_des_probe()  local
958 	dd = devm_kzalloc(dev, sizeof(struct omap_des_dev), GFP_KERNEL);  in omap_des_probe()
959 	if (dd == NULL) {  in omap_des_probe()
963 	dd->dev = dev;  in omap_des_probe()
964 	platform_set_drvdata(pdev, dd);  in omap_des_probe()
966 	err = (dev->of_node) ? omap_des_get_of(dd, pdev) :  in omap_des_probe()
967 			       omap_des_get_pdev(dd, pdev);  in omap_des_probe()
971 	dd->io_base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);  in omap_des_probe()
972 	if (IS_ERR(dd->io_base)) {  in omap_des_probe()
973 		err = PTR_ERR(dd->io_base);  in omap_des_probe()
976 	dd->phys_base = res->start;  in omap_des_probe()
984 		dev_err(dd->dev, "%s: failed to get_sync(%d)\n", __func__, err);  in omap_des_probe()
988 	omap_des_dma_stop(dd);  in omap_des_probe()
990 	reg = omap_des_read(dd, DES_REG_REV(dd));  in omap_des_probe()
995 		 (reg & dd->pdata->major_mask) >> dd->pdata->major_shift,  in omap_des_probe()
996 		 (reg & dd->pdata->minor_mask) >> dd->pdata->minor_shift);  in omap_des_probe()
998 	tasklet_init(&dd->done_task, omap_des_done_task, (unsigned long)dd);  in omap_des_probe()
1000 	err = omap_des_dma_init(dd);  in omap_des_probe()
1003 	} else if (err && DES_REG_IRQ_STATUS(dd) && DES_REG_IRQ_ENABLE(dd)) {  in omap_des_probe()
1004 		dd->pio_only = 1;  in omap_des_probe()
1013 				dev_name(dev), dd);  in omap_des_probe()
1021 	INIT_LIST_HEAD(&dd->list);  in omap_des_probe()
1023 	list_add_tail(&dd->list, &dev_list);  in omap_des_probe()
1027 	dd->engine = crypto_engine_alloc_init(dev, 1);  in omap_des_probe()
1028 	if (!dd->engine) {  in omap_des_probe()
1033 	err = crypto_engine_start(dd->engine);  in omap_des_probe()
1037 	for (i = 0; i < dd->pdata->algs_info_size; i++) {  in omap_des_probe()
1038 		for (j = 0; j < dd->pdata->algs_info[i].size; j++) {  in omap_des_probe()
1039 			algp = &dd->pdata->algs_info[i].algs_list[j];  in omap_des_probe()
1047 			dd->pdata->algs_info[i].registered++;  in omap_des_probe()
1054 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)  in omap_des_probe()
1055 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)  in omap_des_probe()
1057 					&dd->pdata->algs_info[i].algs_list[j]);  in omap_des_probe()
1060 	if (dd->engine)  in omap_des_probe()
1061 		crypto_engine_exit(dd->engine);  in omap_des_probe()
1063 	omap_des_dma_cleanup(dd);  in omap_des_probe()
1065 	tasklet_kill(&dd->done_task);  in omap_des_probe()
1069 	dd = NULL;  in omap_des_probe()
1077 	struct omap_des_dev *dd = platform_get_drvdata(pdev);  in omap_des_remove()  local
1081 	list_del(&dd->list);  in omap_des_remove()
1084 	for (i = dd->pdata->algs_info_size - 1; i >= 0; i--)  in omap_des_remove()
1085 		for (j = dd->pdata->algs_info[i].registered - 1; j >= 0; j--)  in omap_des_remove()
1087 					&dd->pdata->algs_info[i].algs_list[j]);  in omap_des_remove()
1089 	tasklet_kill(&dd->done_task);  in omap_des_remove()
1090 	omap_des_dma_cleanup(dd);  in omap_des_remove()
1091 	pm_runtime_disable(dd->dev);  in omap_des_remove()