75 	struct atmel_tdes_dev *dd;  member
181 static inline u32 atmel_tdes_read(struct atmel_tdes_dev *dd, u32 offset)  in atmel_tdes_read()  argument
183 	return readl_relaxed(dd->io_base + offset);  in atmel_tdes_read()
186 static inline void atmel_tdes_write(struct atmel_tdes_dev *dd,  in atmel_tdes_write()  argument
189 	writel_relaxed(value, dd->io_base + offset);  in atmel_tdes_write()
192 static void atmel_tdes_write_n(struct atmel_tdes_dev *dd, u32 offset,  in atmel_tdes_write_n()  argument
196 		atmel_tdes_write(dd, offset, *value);  in atmel_tdes_write_n()
211 static int atmel_tdes_hw_init(struct atmel_tdes_dev *dd)  in atmel_tdes_hw_init()  argument
215 	err = clk_prepare_enable(dd->iclk);  in atmel_tdes_hw_init()
219 	if (!(dd->flags & TDES_FLAGS_INIT)) {  in atmel_tdes_hw_init()
220 		atmel_tdes_write(dd, TDES_CR, TDES_CR_SWRST);  in atmel_tdes_hw_init()
221 		dd->flags |= TDES_FLAGS_INIT;  in atmel_tdes_hw_init()
227 static inline unsigned int atmel_tdes_get_version(struct atmel_tdes_dev *dd)  in atmel_tdes_get_version()  argument
229 	return atmel_tdes_read(dd, TDES_HW_VERSION) & 0x00000fff;  in atmel_tdes_get_version()
232 static int atmel_tdes_hw_version_init(struct atmel_tdes_dev *dd)  in atmel_tdes_hw_version_init()  argument
236 	err = atmel_tdes_hw_init(dd);  in atmel_tdes_hw_version_init()
240 	dd->hw_version = atmel_tdes_get_version(dd);  in atmel_tdes_hw_version_init()
242 	dev_info(dd->dev,  in atmel_tdes_hw_version_init()
243 			"version: 0x%x\n", dd->hw_version);  in atmel_tdes_hw_version_init()
245 	clk_disable_unprepare(dd->iclk);  in atmel_tdes_hw_version_init()
252 	struct atmel_tdes_dev *dd = data;  in atmel_tdes_dma_callback()  local
255 	tasklet_schedule(&dd->done_task);  in atmel_tdes_dma_callback()
258 static int atmel_tdes_write_ctrl(struct atmel_tdes_dev *dd)  in atmel_tdes_write_ctrl()  argument
263 	err = atmel_tdes_hw_init(dd);  in atmel_tdes_write_ctrl()
268 	if (!dd->caps.has_dma)  in atmel_tdes_write_ctrl()
269 		atmel_tdes_write(dd, TDES_PTCR,  in atmel_tdes_write_ctrl()
273 	if (dd->ctx->keylen > (DES_KEY_SIZE << 1)) {  in atmel_tdes_write_ctrl()
276 	} else if (dd->ctx->keylen > DES_KEY_SIZE) {  in atmel_tdes_write_ctrl()
283 	valmr |= dd->flags & TDES_FLAGS_MODE_MASK;  in atmel_tdes_write_ctrl()
285 	atmel_tdes_write(dd, TDES_MR, valmr);  in atmel_tdes_write_ctrl()
287 	atmel_tdes_write_n(dd, TDES_KEY1W1R, dd->ctx->key,  in atmel_tdes_write_ctrl()
288 						dd->ctx->keylen >> 2);  in atmel_tdes_write_ctrl()
290 	if (dd->req->iv && (valmr & TDES_MR_OPMOD_MASK) != TDES_MR_OPMOD_ECB)  in atmel_tdes_write_ctrl()
291 		atmel_tdes_write_n(dd, TDES_IV1R, (void *)dd->req->iv, 2);  in atmel_tdes_write_ctrl()
296 static int atmel_tdes_crypt_pdc_stop(struct atmel_tdes_dev *dd)  in atmel_tdes_crypt_pdc_stop()  argument
301 	atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);  in atmel_tdes_crypt_pdc_stop()
303 	if (dd->flags & TDES_FLAGS_FAST) {  in atmel_tdes_crypt_pdc_stop()
304 		dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);  in atmel_tdes_crypt_pdc_stop()
305 		dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_pdc_stop()
307 		dma_sync_single_for_device(dd->dev, dd->dma_addr_out,  in atmel_tdes_crypt_pdc_stop()
308 					   dd->dma_size, DMA_FROM_DEVICE);  in atmel_tdes_crypt_pdc_stop()
311 		count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,  in atmel_tdes_crypt_pdc_stop()
312 				dd->buf_out, dd->buflen, dd->dma_size, 1);  in atmel_tdes_crypt_pdc_stop()
313 		if (count != dd->dma_size) {  in atmel_tdes_crypt_pdc_stop()
315 			dev_dbg(dd->dev, "not all data converted: %zu\n", count);  in atmel_tdes_crypt_pdc_stop()
322 static int atmel_tdes_buff_init(struct atmel_tdes_dev *dd)  in atmel_tdes_buff_init()  argument
326 	dd->buf_in = (void *)__get_free_pages(GFP_KERNEL, 0);  in atmel_tdes_buff_init()
327 	dd->buf_out = (void *)__get_free_pages(GFP_KERNEL, 0);  in atmel_tdes_buff_init()
328 	dd->buflen = PAGE_SIZE;  in atmel_tdes_buff_init()
329 	dd->buflen &= ~(DES_BLOCK_SIZE - 1);  in atmel_tdes_buff_init()
331 	if (!dd->buf_in || !dd->buf_out) {  in atmel_tdes_buff_init()
332 		dev_dbg(dd->dev, "unable to alloc pages.\n");  in atmel_tdes_buff_init()
337 	dd->dma_addr_in = dma_map_single(dd->dev, dd->buf_in,  in atmel_tdes_buff_init()
338 					dd->buflen, DMA_TO_DEVICE);  in atmel_tdes_buff_init()
339 	err = dma_mapping_error(dd->dev, dd->dma_addr_in);  in atmel_tdes_buff_init()
341 		dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen);  in atmel_tdes_buff_init()
345 	dd->dma_addr_out = dma_map_single(dd->dev, dd->buf_out,  in atmel_tdes_buff_init()
346 					dd->buflen, DMA_FROM_DEVICE);  in atmel_tdes_buff_init()
347 	err = dma_mapping_error(dd->dev, dd->dma_addr_out);  in atmel_tdes_buff_init()
349 		dev_dbg(dd->dev, "dma %zd bytes error\n", dd->buflen);  in atmel_tdes_buff_init()
356 	dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,  in atmel_tdes_buff_init()
360 	free_page((unsigned long)dd->buf_out);  in atmel_tdes_buff_init()
361 	free_page((unsigned long)dd->buf_in);  in atmel_tdes_buff_init()
365 static void atmel_tdes_buff_cleanup(struct atmel_tdes_dev *dd)  in atmel_tdes_buff_cleanup()  argument
367 	dma_unmap_single(dd->dev, dd->dma_addr_out, dd->buflen,  in atmel_tdes_buff_cleanup()
369 	dma_unmap_single(dd->dev, dd->dma_addr_in, dd->buflen,  in atmel_tdes_buff_cleanup()
371 	free_page((unsigned long)dd->buf_out);  in atmel_tdes_buff_cleanup()
372 	free_page((unsigned long)dd->buf_in);  in atmel_tdes_buff_cleanup()
375 static int atmel_tdes_crypt_pdc(struct atmel_tdes_dev *dd,  in atmel_tdes_crypt_pdc()  argument
379 	struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(dd->req);  in atmel_tdes_crypt_pdc()
382 	dd->dma_size = length;  in atmel_tdes_crypt_pdc()
384 	if (!(dd->flags & TDES_FLAGS_FAST)) {  in atmel_tdes_crypt_pdc()
385 		dma_sync_single_for_device(dd->dev, dma_addr_in, length,  in atmel_tdes_crypt_pdc()
403 	atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTDIS|TDES_PTCR_RXTDIS);  in atmel_tdes_crypt_pdc()
404 	atmel_tdes_write(dd, TDES_TPR, dma_addr_in);  in atmel_tdes_crypt_pdc()
405 	atmel_tdes_write(dd, TDES_TCR, len32);  in atmel_tdes_crypt_pdc()
406 	atmel_tdes_write(dd, TDES_RPR, dma_addr_out);  in atmel_tdes_crypt_pdc()
407 	atmel_tdes_write(dd, TDES_RCR, len32);  in atmel_tdes_crypt_pdc()
410 	atmel_tdes_write(dd, TDES_IER, TDES_INT_ENDRX);  in atmel_tdes_crypt_pdc()
413 	atmel_tdes_write(dd, TDES_PTCR, TDES_PTCR_TXTEN | TDES_PTCR_RXTEN);  in atmel_tdes_crypt_pdc()
418 static int atmel_tdes_crypt_dma(struct atmel_tdes_dev *dd,  in atmel_tdes_crypt_dma()  argument
422 	struct atmel_tdes_reqctx *rctx = skcipher_request_ctx(dd->req);  in atmel_tdes_crypt_dma()
427 	dd->dma_size = length;  in atmel_tdes_crypt_dma()
429 	if (!(dd->flags & TDES_FLAGS_FAST)) {  in atmel_tdes_crypt_dma()
430 		dma_sync_single_for_device(dd->dev, dma_addr_in, length,  in atmel_tdes_crypt_dma()
448 	dd->dma_lch_in.dma_conf.dst_addr_width = addr_width;  in atmel_tdes_crypt_dma()
449 	dd->dma_lch_out.dma_conf.src_addr_width = addr_width;  in atmel_tdes_crypt_dma()
451 	dmaengine_slave_config(dd->dma_lch_in.chan, &dd->dma_lch_in.dma_conf);  in atmel_tdes_crypt_dma()
452 	dmaengine_slave_config(dd->dma_lch_out.chan, &dd->dma_lch_out.dma_conf);  in atmel_tdes_crypt_dma()
454 	dd->flags |= TDES_FLAGS_DMA;  in atmel_tdes_crypt_dma()
464 	in_desc = dmaengine_prep_slave_sg(dd->dma_lch_in.chan, &sg[0],  in atmel_tdes_crypt_dma()
470 	out_desc = dmaengine_prep_slave_sg(dd->dma_lch_out.chan, &sg[1],  in atmel_tdes_crypt_dma()
477 	out_desc->callback_param = dd;  in atmel_tdes_crypt_dma()
480 	dma_async_issue_pending(dd->dma_lch_out.chan);  in atmel_tdes_crypt_dma()
483 	dma_async_issue_pending(dd->dma_lch_in.chan);  in atmel_tdes_crypt_dma()
488 static int atmel_tdes_crypt_start(struct atmel_tdes_dev *dd)  in atmel_tdes_crypt_start()  argument
494 	if ((!dd->in_offset) && (!dd->out_offset)) {  in atmel_tdes_crypt_start()
496 		in = IS_ALIGNED((u32)dd->in_sg->offset, sizeof(u32)) &&  in atmel_tdes_crypt_start()
497 			IS_ALIGNED(dd->in_sg->length, dd->ctx->block_size);  in atmel_tdes_crypt_start()
498 		out = IS_ALIGNED((u32)dd->out_sg->offset, sizeof(u32)) &&  in atmel_tdes_crypt_start()
499 			IS_ALIGNED(dd->out_sg->length, dd->ctx->block_size);  in atmel_tdes_crypt_start()
502 		if (sg_dma_len(dd->in_sg) != sg_dma_len(dd->out_sg))  in atmel_tdes_crypt_start()
508 		count = min_t(size_t, dd->total, sg_dma_len(dd->in_sg));  in atmel_tdes_crypt_start()
509 		count = min_t(size_t, count, sg_dma_len(dd->out_sg));  in atmel_tdes_crypt_start()
511 		err = dma_map_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_start()
513 			dev_dbg(dd->dev, "dma_map_sg() error\n");  in atmel_tdes_crypt_start()
517 		err = dma_map_sg(dd->dev, dd->out_sg, 1,  in atmel_tdes_crypt_start()
520 			dev_dbg(dd->dev, "dma_map_sg() error\n");  in atmel_tdes_crypt_start()
521 			dma_unmap_sg(dd->dev, dd->in_sg, 1,  in atmel_tdes_crypt_start()
526 		addr_in = sg_dma_address(dd->in_sg);  in atmel_tdes_crypt_start()
527 		addr_out = sg_dma_address(dd->out_sg);  in atmel_tdes_crypt_start()
529 		dd->flags |= TDES_FLAGS_FAST;  in atmel_tdes_crypt_start()
533 		count = atmel_tdes_sg_copy(&dd->in_sg, &dd->in_offset,  in atmel_tdes_crypt_start()
534 				dd->buf_in, dd->buflen, dd->total, 0);  in atmel_tdes_crypt_start()
536 		addr_in = dd->dma_addr_in;  in atmel_tdes_crypt_start()
537 		addr_out = dd->dma_addr_out;  in atmel_tdes_crypt_start()
539 		dd->flags &= ~TDES_FLAGS_FAST;  in atmel_tdes_crypt_start()
542 	dd->total -= count;  in atmel_tdes_crypt_start()
544 	if (dd->caps.has_dma)  in atmel_tdes_crypt_start()
545 		err = atmel_tdes_crypt_dma(dd, addr_in, addr_out, count);  in atmel_tdes_crypt_start()
547 		err = atmel_tdes_crypt_pdc(dd, addr_in, addr_out, count);  in atmel_tdes_crypt_start()
549 	if (err && (dd->flags & TDES_FLAGS_FAST)) {  in atmel_tdes_crypt_start()
550 		dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_start()
551 		dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_start()
558 atmel_tdes_set_iv_as_last_ciphertext_block(struct atmel_tdes_dev *dd)  in atmel_tdes_set_iv_as_last_ciphertext_block()  argument
560 	struct skcipher_request *req = dd->req;  in atmel_tdes_set_iv_as_last_ciphertext_block()
576 static void atmel_tdes_finish_req(struct atmel_tdes_dev *dd, int err)  in atmel_tdes_finish_req()  argument
578 	struct skcipher_request *req = dd->req;  in atmel_tdes_finish_req()
581 	clk_disable_unprepare(dd->iclk);  in atmel_tdes_finish_req()
583 	dd->flags &= ~TDES_FLAGS_BUSY;  in atmel_tdes_finish_req()
586 		atmel_tdes_set_iv_as_last_ciphertext_block(dd);  in atmel_tdes_finish_req()
591 static int atmel_tdes_handle_queue(struct atmel_tdes_dev *dd,  in atmel_tdes_handle_queue()  argument
600 	spin_lock_irqsave(&dd->lock, flags);  in atmel_tdes_handle_queue()
602 		ret = crypto_enqueue_request(&dd->queue, &req->base);  in atmel_tdes_handle_queue()
603 	if (dd->flags & TDES_FLAGS_BUSY) {  in atmel_tdes_handle_queue()
604 		spin_unlock_irqrestore(&dd->lock, flags);  in atmel_tdes_handle_queue()
607 	backlog = crypto_get_backlog(&dd->queue);  in atmel_tdes_handle_queue()
608 	async_req = crypto_dequeue_request(&dd->queue);  in atmel_tdes_handle_queue()
610 		dd->flags |= TDES_FLAGS_BUSY;  in atmel_tdes_handle_queue()
611 	spin_unlock_irqrestore(&dd->lock, flags);  in atmel_tdes_handle_queue()
622 	dd->req = req;  in atmel_tdes_handle_queue()
623 	dd->total = req->cryptlen;  in atmel_tdes_handle_queue()
624 	dd->in_offset = 0;  in atmel_tdes_handle_queue()
625 	dd->in_sg = req->src;  in atmel_tdes_handle_queue()
626 	dd->out_offset = 0;  in atmel_tdes_handle_queue()
627 	dd->out_sg = req->dst;  in atmel_tdes_handle_queue()
632 	dd->flags = (dd->flags & ~TDES_FLAGS_MODE_MASK) | rctx->mode;  in atmel_tdes_handle_queue()
633 	dd->ctx = ctx;  in atmel_tdes_handle_queue()
635 	err = atmel_tdes_write_ctrl(dd);  in atmel_tdes_handle_queue()
637 		err = atmel_tdes_crypt_start(dd);  in atmel_tdes_handle_queue()
640 		atmel_tdes_finish_req(dd, err);  in atmel_tdes_handle_queue()
641 		tasklet_schedule(&dd->queue_task);  in atmel_tdes_handle_queue()
647 static int atmel_tdes_crypt_dma_stop(struct atmel_tdes_dev *dd)  in atmel_tdes_crypt_dma_stop()  argument
652 	if (dd->flags & TDES_FLAGS_DMA) {  in atmel_tdes_crypt_dma_stop()
654 		if  (dd->flags & TDES_FLAGS_FAST) {  in atmel_tdes_crypt_dma_stop()
655 			dma_unmap_sg(dd->dev, dd->out_sg, 1, DMA_FROM_DEVICE);  in atmel_tdes_crypt_dma_stop()
656 			dma_unmap_sg(dd->dev, dd->in_sg, 1, DMA_TO_DEVICE);  in atmel_tdes_crypt_dma_stop()
658 			dma_sync_single_for_device(dd->dev, dd->dma_addr_out,  in atmel_tdes_crypt_dma_stop()
659 				dd->dma_size, DMA_FROM_DEVICE);  in atmel_tdes_crypt_dma_stop()
662 			count = atmel_tdes_sg_copy(&dd->out_sg, &dd->out_offset,  in atmel_tdes_crypt_dma_stop()
663 				dd->buf_out, dd->buflen, dd->dma_size, 1);  in atmel_tdes_crypt_dma_stop()
664 			if (count != dd->dma_size) {  in atmel_tdes_crypt_dma_stop()
666 				dev_dbg(dd->dev, "not all data converted: %zu\n", count);  in atmel_tdes_crypt_dma_stop()
678 	struct device *dev = ctx->dd->dev;  in atmel_tdes_crypt()
729 	return atmel_tdes_handle_queue(ctx->dd, req);  in atmel_tdes_crypt()
732 static int atmel_tdes_dma_init(struct atmel_tdes_dev *dd)  in atmel_tdes_dma_init()  argument
737 	dd->dma_lch_in.chan = dma_request_chan(dd->dev, "tx");  in atmel_tdes_dma_init()
738 	if (IS_ERR(dd->dma_lch_in.chan)) {  in atmel_tdes_dma_init()
739 		ret = PTR_ERR(dd->dma_lch_in.chan);  in atmel_tdes_dma_init()
743 	dd->dma_lch_in.dma_conf.dst_addr = dd->phys_base +  in atmel_tdes_dma_init()
745 	dd->dma_lch_in.dma_conf.src_maxburst = 1;  in atmel_tdes_dma_init()
746 	dd->dma_lch_in.dma_conf.src_addr_width =  in atmel_tdes_dma_init()
748 	dd->dma_lch_in.dma_conf.dst_maxburst = 1;  in atmel_tdes_dma_init()
749 	dd->dma_lch_in.dma_conf.dst_addr_width =  in atmel_tdes_dma_init()
751 	dd->dma_lch_in.dma_conf.device_fc = false;  in atmel_tdes_dma_init()
753 	dd->dma_lch_out.chan = dma_request_chan(dd->dev, "rx");  in atmel_tdes_dma_init()
754 	if (IS_ERR(dd->dma_lch_out.chan)) {  in atmel_tdes_dma_init()
755 		ret = PTR_ERR(dd->dma_lch_out.chan);  in atmel_tdes_dma_init()
759 	dd->dma_lch_out.dma_conf.src_addr = dd->phys_base +  in atmel_tdes_dma_init()
761 	dd->dma_lch_out.dma_conf.src_maxburst = 1;  in atmel_tdes_dma_init()
762 	dd->dma_lch_out.dma_conf.src_addr_width =  in atmel_tdes_dma_init()
764 	dd->dma_lch_out.dma_conf.dst_maxburst = 1;  in atmel_tdes_dma_init()
765 	dd->dma_lch_out.dma_conf.dst_addr_width =  in atmel_tdes_dma_init()
767 	dd->dma_lch_out.dma_conf.device_fc = false;  in atmel_tdes_dma_init()
772 	dma_release_channel(dd->dma_lch_in.chan);  in atmel_tdes_dma_init()
774 	dev_err(dd->dev, "no DMA channel available\n");  in atmel_tdes_dma_init()
778 static void atmel_tdes_dma_cleanup(struct atmel_tdes_dev *dd)  in atmel_tdes_dma_cleanup()  argument
780 	dma_release_channel(dd->dma_lch_in.chan);  in atmel_tdes_dma_cleanup()
781 	dma_release_channel(dd->dma_lch_out.chan);  in atmel_tdes_dma_cleanup()
889 	ctx->dd = atmel_tdes_dev_alloc();  in atmel_tdes_init_tfm()
890 	if (!ctx->dd)  in atmel_tdes_init_tfm()
1041 	struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *)data;  in atmel_tdes_queue_task()  local
1043 	atmel_tdes_handle_queue(dd, NULL);  in atmel_tdes_queue_task()
1048 	struct atmel_tdes_dev *dd = (struct atmel_tdes_dev *) data;  in atmel_tdes_done_task()  local
1051 	if (!(dd->flags & TDES_FLAGS_DMA))  in atmel_tdes_done_task()
1052 		err = atmel_tdes_crypt_pdc_stop(dd);  in atmel_tdes_done_task()
1054 		err = atmel_tdes_crypt_dma_stop(dd);  in atmel_tdes_done_task()
1056 	if (dd->total && !err) {  in atmel_tdes_done_task()
1057 		if (dd->flags & TDES_FLAGS_FAST) {  in atmel_tdes_done_task()
1058 			dd->in_sg = sg_next(dd->in_sg);  in atmel_tdes_done_task()
1059 			dd->out_sg = sg_next(dd->out_sg);  in atmel_tdes_done_task()
1060 			if (!dd->in_sg || !dd->out_sg)  in atmel_tdes_done_task()
1064 			err = atmel_tdes_crypt_start(dd);  in atmel_tdes_done_task()
1069 	atmel_tdes_finish_req(dd, err);  in atmel_tdes_done_task()
1070 	atmel_tdes_handle_queue(dd, NULL);  in atmel_tdes_done_task()
1091 static void atmel_tdes_unregister_algs(struct atmel_tdes_dev *dd)  in atmel_tdes_unregister_algs()  argument
1099 static int atmel_tdes_register_algs(struct atmel_tdes_dev *dd)  in atmel_tdes_register_algs()  argument
1120 static void atmel_tdes_get_cap(struct atmel_tdes_dev *dd)  in atmel_tdes_get_cap()  argument
1123 	dd->caps.has_dma = 0;  in atmel_tdes_get_cap()
1124 	dd->caps.has_cfb_3keys = 0;  in atmel_tdes_get_cap()
1127 	switch (dd->hw_version & 0xf00) {  in atmel_tdes_get_cap()
1130 		dd->caps.has_dma = 1;  in atmel_tdes_get_cap()
1131 		dd->caps.has_cfb_3keys = 1;  in atmel_tdes_get_cap()
1136 		dev_warn(dd->dev,  in atmel_tdes_get_cap()