1 /*
2  * Driver for the Atmel USBA high speed USB device controller
3  *
4  * Copyright (C) 2005-2007 Atmel Corporation
5  *
6  * This program is free software; you can redistribute it and/or modify
7  * it under the terms of the GNU General Public License version 2 as
8  * published by the Free Software Foundation.
9  */
10 #include <linux/clk.h>
11 #include <linux/module.h>
12 #include <linux/init.h>
13 #include <linux/interrupt.h>
14 #include <linux/io.h>
15 #include <linux/slab.h>
16 #include <linux/device.h>
17 #include <linux/dma-mapping.h>
18 #include <linux/list.h>
19 #include <linux/platform_device.h>
20 #include <linux/usb/ch9.h>
21 #include <linux/usb/gadget.h>
22 #include <linux/usb/atmel_usba_udc.h>
23 #include <linux/delay.h>
24 #include <linux/platform_data/atmel.h>
25 #include <linux/of.h>
26 #include <linux/of_gpio.h>
27 
28 #include <asm/gpio.h>
29 
30 #include "atmel_usba_udc.h"
31 
32 #ifdef CONFIG_USB_GADGET_DEBUG_FS
33 #include <linux/debugfs.h>
34 #include <linux/uaccess.h>
35 
36 static int queue_dbg_open(struct inode *inode, struct file *file)
37 {
38 	struct usba_ep *ep = inode->i_private;
39 	struct usba_request *req, *req_copy;
40 	struct list_head *queue_data;
41 
42 	queue_data = kmalloc(sizeof(*queue_data), GFP_KERNEL);
43 	if (!queue_data)
44 		return -ENOMEM;
45 	INIT_LIST_HEAD(queue_data);
46 
47 	spin_lock_irq(&ep->udc->lock);
48 	list_for_each_entry(req, &ep->queue, queue) {
49 		req_copy = kmemdup(req, sizeof(*req_copy), GFP_ATOMIC);
50 		if (!req_copy)
51 			goto fail;
52 		list_add_tail(&req_copy->queue, queue_data);
53 	}
54 	spin_unlock_irq(&ep->udc->lock);
55 
56 	file->private_data = queue_data;
57 	return 0;
58 
59 fail:
60 	spin_unlock_irq(&ep->udc->lock);
61 	list_for_each_entry_safe(req, req_copy, queue_data, queue) {
62 		list_del(&req->queue);
63 		kfree(req);
64 	}
65 	kfree(queue_data);
66 	return -ENOMEM;
67 }
68 
69 /*
70  * bbbbbbbb llllllll IZS sssss nnnn FDL\n\0
71  *
72  * b: buffer address
73  * l: buffer length
74  * I/i: interrupt/no interrupt
75  * Z/z: zero/no zero
76  * S/s: short ok/short not ok
77  * s: status
78  * n: nr_packets
79  * F/f: submitted/not submitted to FIFO
80  * D/d: using/not using DMA
81  * L/l: last transaction/not last transaction
82  */
83 static ssize_t queue_dbg_read(struct file *file, char __user *buf,
84 		size_t nbytes, loff_t *ppos)
85 {
86 	struct list_head *queue = file->private_data;
87 	struct usba_request *req, *tmp_req;
88 	size_t len, remaining, actual = 0;
89 	char tmpbuf[38];
90 
91 	if (!access_ok(VERIFY_WRITE, buf, nbytes))
92 		return -EFAULT;
93 
94 	mutex_lock(&file_inode(file)->i_mutex);
95 	list_for_each_entry_safe(req, tmp_req, queue, queue) {
96 		len = snprintf(tmpbuf, sizeof(tmpbuf),
97 				"%8p %08x %c%c%c %5d %c%c%c\n",
98 				req->req.buf, req->req.length,
99 				req->req.no_interrupt ? 'i' : 'I',
100 				req->req.zero ? 'Z' : 'z',
101 				req->req.short_not_ok ? 's' : 'S',
102 				req->req.status,
103 				req->submitted ? 'F' : 'f',
104 				req->using_dma ? 'D' : 'd',
105 				req->last_transaction ? 'L' : 'l');
106 		len = min(len, sizeof(tmpbuf));
107 		if (len > nbytes)
108 			break;
109 
110 		list_del(&req->queue);
111 		kfree(req);
112 
113 		remaining = __copy_to_user(buf, tmpbuf, len);
114 		actual += len - remaining;
115 		if (remaining)
116 			break;
117 
118 		nbytes -= len;
119 		buf += len;
120 	}
121 	mutex_unlock(&file_inode(file)->i_mutex);
122 
123 	return actual;
124 }
125 
126 static int queue_dbg_release(struct inode *inode, struct file *file)
127 {
128 	struct list_head *queue_data = file->private_data;
129 	struct usba_request *req, *tmp_req;
130 
131 	list_for_each_entry_safe(req, tmp_req, queue_data, queue) {
132 		list_del(&req->queue);
133 		kfree(req);
134 	}
135 	kfree(queue_data);
136 	return 0;
137 }
138 
139 static int regs_dbg_open(struct inode *inode, struct file *file)
140 {
141 	struct usba_udc *udc;
142 	unsigned int i;
143 	u32 *data;
144 	int ret = -ENOMEM;
145 
146 	mutex_lock(&inode->i_mutex);
147 	udc = inode->i_private;
148 	data = kmalloc(inode->i_size, GFP_KERNEL);
149 	if (!data)
150 		goto out;
151 
152 	spin_lock_irq(&udc->lock);
153 	for (i = 0; i < inode->i_size / 4; i++)
154 		data[i] = __raw_readl(udc->regs + i * 4);
155 	spin_unlock_irq(&udc->lock);
156 
157 	file->private_data = data;
158 	ret = 0;
159 
160 out:
161 	mutex_unlock(&inode->i_mutex);
162 
163 	return ret;
164 }
165 
166 static ssize_t regs_dbg_read(struct file *file, char __user *buf,
167 		size_t nbytes, loff_t *ppos)
168 {
169 	struct inode *inode = file_inode(file);
170 	int ret;
171 
172 	mutex_lock(&inode->i_mutex);
173 	ret = simple_read_from_buffer(buf, nbytes, ppos,
174 			file->private_data,
175 			file_inode(file)->i_size);
176 	mutex_unlock(&inode->i_mutex);
177 
178 	return ret;
179 }
180 
181 static int regs_dbg_release(struct inode *inode, struct file *file)
182 {
183 	kfree(file->private_data);
184 	return 0;
185 }
186 
187 const struct file_operations queue_dbg_fops = {
188 	.owner		= THIS_MODULE,
189 	.open		= queue_dbg_open,
190 	.llseek		= no_llseek,
191 	.read		= queue_dbg_read,
192 	.release	= queue_dbg_release,
193 };
194 
195 const struct file_operations regs_dbg_fops = {
196 	.owner		= THIS_MODULE,
197 	.open		= regs_dbg_open,
198 	.llseek		= generic_file_llseek,
199 	.read		= regs_dbg_read,
200 	.release	= regs_dbg_release,
201 };
202 
203 static void usba_ep_init_debugfs(struct usba_udc *udc,
204 		struct usba_ep *ep)
205 {
206 	struct dentry *ep_root;
207 
208 	ep_root = debugfs_create_dir(ep->ep.name, udc->debugfs_root);
209 	if (!ep_root)
210 		goto err_root;
211 	ep->debugfs_dir = ep_root;
212 
213 	ep->debugfs_queue = debugfs_create_file("queue", 0400, ep_root,
214 						ep, &queue_dbg_fops);
215 	if (!ep->debugfs_queue)
216 		goto err_queue;
217 
218 	if (ep->can_dma) {
219 		ep->debugfs_dma_status
220 			= debugfs_create_u32("dma_status", 0400, ep_root,
221 					&ep->last_dma_status);
222 		if (!ep->debugfs_dma_status)
223 			goto err_dma_status;
224 	}
225 	if (ep_is_control(ep)) {
226 		ep->debugfs_state
227 			= debugfs_create_u32("state", 0400, ep_root,
228 					&ep->state);
229 		if (!ep->debugfs_state)
230 			goto err_state;
231 	}
232 
233 	return;
234 
235 err_state:
236 	if (ep->can_dma)
237 		debugfs_remove(ep->debugfs_dma_status);
238 err_dma_status:
239 	debugfs_remove(ep->debugfs_queue);
240 err_queue:
241 	debugfs_remove(ep_root);
242 err_root:
243 	dev_err(&ep->udc->pdev->dev,
244 		"failed to create debugfs directory for %s\n", ep->ep.name);
245 }
246 
247 static void usba_ep_cleanup_debugfs(struct usba_ep *ep)
248 {
249 	debugfs_remove(ep->debugfs_queue);
250 	debugfs_remove(ep->debugfs_dma_status);
251 	debugfs_remove(ep->debugfs_state);
252 	debugfs_remove(ep->debugfs_dir);
253 	ep->debugfs_dma_status = NULL;
254 	ep->debugfs_dir = NULL;
255 }
256 
257 static void usba_init_debugfs(struct usba_udc *udc)
258 {
259 	struct dentry *root, *regs;
260 	struct resource *regs_resource;
261 
262 	root = debugfs_create_dir(udc->gadget.name, NULL);
263 	if (IS_ERR(root) || !root)
264 		goto err_root;
265 	udc->debugfs_root = root;
266 
267 	regs = debugfs_create_file("regs", 0400, root, udc, &regs_dbg_fops);
268 	if (!regs)
269 		goto err_regs;
270 
271 	regs_resource = platform_get_resource(udc->pdev, IORESOURCE_MEM,
272 				CTRL_IOMEM_ID);
273 	regs->d_inode->i_size = resource_size(regs_resource);
274 	udc->debugfs_regs = regs;
275 
276 	usba_ep_init_debugfs(udc, to_usba_ep(udc->gadget.ep0));
277 
278 	return;
279 
280 err_regs:
281 	debugfs_remove(root);
282 err_root:
283 	udc->debugfs_root = NULL;
284 	dev_err(&udc->pdev->dev, "debugfs is not available\n");
285 }
286 
287 static void usba_cleanup_debugfs(struct usba_udc *udc)
288 {
289 	usba_ep_cleanup_debugfs(to_usba_ep(udc->gadget.ep0));
290 	debugfs_remove(udc->debugfs_regs);
291 	debugfs_remove(udc->debugfs_root);
292 	udc->debugfs_regs = NULL;
293 	udc->debugfs_root = NULL;
294 }
295 #else
296 static inline void usba_ep_init_debugfs(struct usba_udc *udc,
297 					 struct usba_ep *ep)
298 {
299 
300 }
301 
302 static inline void usba_ep_cleanup_debugfs(struct usba_ep *ep)
303 {
304 
305 }
306 
307 static inline void usba_init_debugfs(struct usba_udc *udc)
308 {
309 
310 }
311 
312 static inline void usba_cleanup_debugfs(struct usba_udc *udc)
313 {
314 
315 }
316 #endif
317 
318 static int vbus_is_present(struct usba_udc *udc)
319 {
320 	if (gpio_is_valid(udc->vbus_pin))
321 		return gpio_get_value(udc->vbus_pin) ^ udc->vbus_pin_inverted;
322 
323 	/* No Vbus detection: Assume always present */
324 	return 1;
325 }
326 
327 #if defined(CONFIG_ARCH_AT91SAM9RL)
328 
329 #include <linux/clk/at91_pmc.h>
330 
331 static void toggle_bias(int is_on)
332 {
333 	unsigned int uckr = at91_pmc_read(AT91_CKGR_UCKR);
334 
335 	if (is_on)
336 		at91_pmc_write(AT91_CKGR_UCKR, uckr | AT91_PMC_BIASEN);
337 	else
338 		at91_pmc_write(AT91_CKGR_UCKR, uckr & ~(AT91_PMC_BIASEN));
339 }
340 
341 #else
342 
343 static void toggle_bias(int is_on)
344 {
345 }
346 
347 #endif /* CONFIG_ARCH_AT91SAM9RL */
348 
349 static void next_fifo_transaction(struct usba_ep *ep, struct usba_request *req)
350 {
351 	unsigned int transaction_len;
352 
353 	transaction_len = req->req.length - req->req.actual;
354 	req->last_transaction = 1;
355 	if (transaction_len > ep->ep.maxpacket) {
356 		transaction_len = ep->ep.maxpacket;
357 		req->last_transaction = 0;
358 	} else if (transaction_len == ep->ep.maxpacket && req->req.zero)
359 		req->last_transaction = 0;
360 
361 	DBG(DBG_QUEUE, "%s: submit_transaction, req %p (length %d)%s\n",
362 		ep->ep.name, req, transaction_len,
363 		req->last_transaction ? ", done" : "");
364 
365 	memcpy_toio(ep->fifo, req->req.buf + req->req.actual, transaction_len);
366 	usba_ep_writel(ep, SET_STA, USBA_TX_PK_RDY);
367 	req->req.actual += transaction_len;
368 }
369 
370 static void submit_request(struct usba_ep *ep, struct usba_request *req)
371 {
372 	DBG(DBG_QUEUE, "%s: submit_request: req %p (length %d)\n",
373 		ep->ep.name, req, req->req.length);
374 
375 	req->req.actual = 0;
376 	req->submitted = 1;
377 
378 	if (req->using_dma) {
379 		if (req->req.length == 0) {
380 			usba_ep_writel(ep, CTL_ENB, USBA_TX_PK_RDY);
381 			return;
382 		}
383 
384 		if (req->req.zero)
385 			usba_ep_writel(ep, CTL_ENB, USBA_SHORT_PACKET);
386 		else
387 			usba_ep_writel(ep, CTL_DIS, USBA_SHORT_PACKET);
388 
389 		usba_dma_writel(ep, ADDRESS, req->req.dma);
390 		usba_dma_writel(ep, CONTROL, req->ctrl);
391 	} else {
392 		next_fifo_transaction(ep, req);
393 		if (req->last_transaction) {
394 			usba_ep_writel(ep, CTL_DIS, USBA_TX_PK_RDY);
395 			usba_ep_writel(ep, CTL_ENB, USBA_TX_COMPLETE);
396 		} else {
397 			usba_ep_writel(ep, CTL_DIS, USBA_TX_COMPLETE);
398 			usba_ep_writel(ep, CTL_ENB, USBA_TX_PK_RDY);
399 		}
400 	}
401 }
402 
403 static void submit_next_request(struct usba_ep *ep)
404 {
405 	struct usba_request *req;
406 
407 	if (list_empty(&ep->queue)) {
408 		usba_ep_writel(ep, CTL_DIS, USBA_TX_PK_RDY | USBA_RX_BK_RDY);
409 		return;
410 	}
411 
412 	req = list_entry(ep->queue.next, struct usba_request, queue);
413 	if (!req->submitted)
414 		submit_request(ep, req);
415 }
416 
417 static void send_status(struct usba_udc *udc, struct usba_ep *ep)
418 {
419 	ep->state = STATUS_STAGE_IN;
420 	usba_ep_writel(ep, SET_STA, USBA_TX_PK_RDY);
421 	usba_ep_writel(ep, CTL_ENB, USBA_TX_COMPLETE);
422 }
423 
424 static void receive_data(struct usba_ep *ep)
425 {
426 	struct usba_udc *udc = ep->udc;
427 	struct usba_request *req;
428 	unsigned long status;
429 	unsigned int bytecount, nr_busy;
430 	int is_complete = 0;
431 
432 	status = usba_ep_readl(ep, STA);
433 	nr_busy = USBA_BFEXT(BUSY_BANKS, status);
434 
435 	DBG(DBG_QUEUE, "receive data: nr_busy=%u\n", nr_busy);
436 
437 	while (nr_busy > 0) {
438 		if (list_empty(&ep->queue)) {
439 			usba_ep_writel(ep, CTL_DIS, USBA_RX_BK_RDY);
440 			break;
441 		}
442 		req = list_entry(ep->queue.next,
443 				 struct usba_request, queue);
444 
445 		bytecount = USBA_BFEXT(BYTE_COUNT, status);
446 
447 		if (status & (1 << 31))
448 			is_complete = 1;
449 		if (req->req.actual + bytecount >= req->req.length) {
450 			is_complete = 1;
451 			bytecount = req->req.length - req->req.actual;
452 		}
453 
454 		memcpy_fromio(req->req.buf + req->req.actual,
455 				ep->fifo, bytecount);
456 		req->req.actual += bytecount;
457 
458 		usba_ep_writel(ep, CLR_STA, USBA_RX_BK_RDY);
459 
460 		if (is_complete) {
461 			DBG(DBG_QUEUE, "%s: request done\n", ep->ep.name);
462 			req->req.status = 0;
463 			list_del_init(&req->queue);
464 			usba_ep_writel(ep, CTL_DIS, USBA_RX_BK_RDY);
465 			spin_unlock(&udc->lock);
466 			usb_gadget_giveback_request(&ep->ep, &req->req);
467 			spin_lock(&udc->lock);
468 		}
469 
470 		status = usba_ep_readl(ep, STA);
471 		nr_busy = USBA_BFEXT(BUSY_BANKS, status);
472 
473 		if (is_complete && ep_is_control(ep)) {
474 			send_status(udc, ep);
475 			break;
476 		}
477 	}
478 }
479 
480 static void
481 request_complete(struct usba_ep *ep, struct usba_request *req, int status)
482 {
483 	struct usba_udc *udc = ep->udc;
484 
485 	WARN_ON(!list_empty(&req->queue));
486 
487 	if (req->req.status == -EINPROGRESS)
488 		req->req.status = status;
489 
490 	if (req->using_dma)
491 		usb_gadget_unmap_request(&udc->gadget, &req->req, ep->is_in);
492 
493 	DBG(DBG_GADGET | DBG_REQ,
494 		"%s: req %p complete: status %d, actual %u\n",
495 		ep->ep.name, req, req->req.status, req->req.actual);
496 
497 	spin_unlock(&udc->lock);
498 	usb_gadget_giveback_request(&ep->ep, &req->req);
499 	spin_lock(&udc->lock);
500 }
501 
502 static void
503 request_complete_list(struct usba_ep *ep, struct list_head *list, int status)
504 {
505 	struct usba_request *req, *tmp_req;
506 
507 	list_for_each_entry_safe(req, tmp_req, list, queue) {
508 		list_del_init(&req->queue);
509 		request_complete(ep, req, status);
510 	}
511 }
512 
513 static int
514 usba_ep_enable(struct usb_ep *_ep, const struct usb_endpoint_descriptor *desc)
515 {
516 	struct usba_ep *ep = to_usba_ep(_ep);
517 	struct usba_udc *udc = ep->udc;
518 	unsigned long flags, ept_cfg, maxpacket;
519 	unsigned int nr_trans;
520 
521 	DBG(DBG_GADGET, "%s: ep_enable: desc=%p\n", ep->ep.name, desc);
522 
523 	maxpacket = usb_endpoint_maxp(desc) & 0x7ff;
524 
525 	if (((desc->bEndpointAddress & USB_ENDPOINT_NUMBER_MASK) != ep->index)
526 			|| ep->index == 0
527 			|| desc->bDescriptorType != USB_DT_ENDPOINT
528 			|| maxpacket == 0
529 			|| maxpacket > ep->fifo_size) {
530 		DBG(DBG_ERR, "ep_enable: Invalid argument");
531 		return -EINVAL;
532 	}
533 
534 	ep->is_isoc = 0;
535 	ep->is_in = 0;
536 
537 	if (maxpacket <= 8)
538 		ept_cfg = USBA_BF(EPT_SIZE, USBA_EPT_SIZE_8);
539 	else
540 		/* LSB is bit 1, not 0 */
541 		ept_cfg = USBA_BF(EPT_SIZE, fls(maxpacket - 1) - 3);
542 
543 	DBG(DBG_HW, "%s: EPT_SIZE = %lu (maxpacket = %lu)\n",
544 			ep->ep.name, ept_cfg, maxpacket);
545 
546 	if (usb_endpoint_dir_in(desc)) {
547 		ep->is_in = 1;
548 		ept_cfg |= USBA_EPT_DIR_IN;
549 	}
550 
551 	switch (usb_endpoint_type(desc)) {
552 	case USB_ENDPOINT_XFER_CONTROL:
553 		ept_cfg |= USBA_BF(EPT_TYPE, USBA_EPT_TYPE_CONTROL);
554 		ept_cfg |= USBA_BF(BK_NUMBER, USBA_BK_NUMBER_ONE);
555 		break;
556 	case USB_ENDPOINT_XFER_ISOC:
557 		if (!ep->can_isoc) {
558 			DBG(DBG_ERR, "ep_enable: %s is not isoc capable\n",
559 					ep->ep.name);
560 			return -EINVAL;
561 		}
562 
563 		/*
564 		 * Bits 11:12 specify number of _additional_
565 		 * transactions per microframe.
566 		 */
567 		nr_trans = ((usb_endpoint_maxp(desc) >> 11) & 3) + 1;
568 		if (nr_trans > 3)
569 			return -EINVAL;
570 
571 		ep->is_isoc = 1;
572 		ept_cfg |= USBA_BF(EPT_TYPE, USBA_EPT_TYPE_ISO);
573 
574 		/*
575 		 * Do triple-buffering on high-bandwidth iso endpoints.
576 		 */
577 		if (nr_trans > 1 && ep->nr_banks == 3)
578 			ept_cfg |= USBA_BF(BK_NUMBER, USBA_BK_NUMBER_TRIPLE);
579 		else
580 			ept_cfg |= USBA_BF(BK_NUMBER, USBA_BK_NUMBER_DOUBLE);
581 		ept_cfg |= USBA_BF(NB_TRANS, nr_trans);
582 		break;
583 	case USB_ENDPOINT_XFER_BULK:
584 		ept_cfg |= USBA_BF(EPT_TYPE, USBA_EPT_TYPE_BULK);
585 		ept_cfg |= USBA_BF(BK_NUMBER, USBA_BK_NUMBER_DOUBLE);
586 		break;
587 	case USB_ENDPOINT_XFER_INT:
588 		ept_cfg |= USBA_BF(EPT_TYPE, USBA_EPT_TYPE_INT);
589 		ept_cfg |= USBA_BF(BK_NUMBER, USBA_BK_NUMBER_DOUBLE);
590 		break;
591 	}
592 
593 	spin_lock_irqsave(&ep->udc->lock, flags);
594 
595 	ep->ep.desc = desc;
596 	ep->ep.maxpacket = maxpacket;
597 
598 	usba_ep_writel(ep, CFG, ept_cfg);
599 	usba_ep_writel(ep, CTL_ENB, USBA_EPT_ENABLE);
600 
601 	if (ep->can_dma) {
602 		u32 ctrl;
603 
604 		usba_writel(udc, INT_ENB,
605 				(usba_readl(udc, INT_ENB)
606 					| USBA_BF(EPT_INT, 1 << ep->index)
607 					| USBA_BF(DMA_INT, 1 << ep->index)));
608 		ctrl = USBA_AUTO_VALID | USBA_INTDIS_DMA;
609 		usba_ep_writel(ep, CTL_ENB, ctrl);
610 	} else {
611 		usba_writel(udc, INT_ENB,
612 				(usba_readl(udc, INT_ENB)
613 					| USBA_BF(EPT_INT, 1 << ep->index)));
614 	}
615 
616 	spin_unlock_irqrestore(&udc->lock, flags);
617 
618 	DBG(DBG_HW, "EPT_CFG%d after init: %#08lx\n", ep->index,
619 			(unsigned long)usba_ep_readl(ep, CFG));
620 	DBG(DBG_HW, "INT_ENB after init: %#08lx\n",
621 			(unsigned long)usba_readl(udc, INT_ENB));
622 
623 	return 0;
624 }
625 
626 static int usba_ep_disable(struct usb_ep *_ep)
627 {
628 	struct usba_ep *ep = to_usba_ep(_ep);
629 	struct usba_udc *udc = ep->udc;
630 	LIST_HEAD(req_list);
631 	unsigned long flags;
632 
633 	DBG(DBG_GADGET, "ep_disable: %s\n", ep->ep.name);
634 
635 	spin_lock_irqsave(&udc->lock, flags);
636 
637 	if (!ep->ep.desc) {
638 		spin_unlock_irqrestore(&udc->lock, flags);
639 		/* REVISIT because this driver disables endpoints in
640 		 * reset_all_endpoints() before calling disconnect(),
641 		 * most gadget drivers would trigger this non-error ...
642 		 */
643 		if (udc->gadget.speed != USB_SPEED_UNKNOWN)
644 			DBG(DBG_ERR, "ep_disable: %s not enabled\n",
645 					ep->ep.name);
646 		return -EINVAL;
647 	}
648 	ep->ep.desc = NULL;
649 
650 	list_splice_init(&ep->queue, &req_list);
651 	if (ep->can_dma) {
652 		usba_dma_writel(ep, CONTROL, 0);
653 		usba_dma_writel(ep, ADDRESS, 0);
654 		usba_dma_readl(ep, STATUS);
655 	}
656 	usba_ep_writel(ep, CTL_DIS, USBA_EPT_ENABLE);
657 	usba_writel(udc, INT_ENB,
658 			usba_readl(udc, INT_ENB)
659 			& ~USBA_BF(EPT_INT, 1 << ep->index));
660 
661 	request_complete_list(ep, &req_list, -ESHUTDOWN);
662 
663 	spin_unlock_irqrestore(&udc->lock, flags);
664 
665 	return 0;
666 }
667 
668 static struct usb_request *
669 usba_ep_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
670 {
671 	struct usba_request *req;
672 
673 	DBG(DBG_GADGET, "ep_alloc_request: %p, 0x%x\n", _ep, gfp_flags);
674 
675 	req = kzalloc(sizeof(*req), gfp_flags);
676 	if (!req)
677 		return NULL;
678 
679 	INIT_LIST_HEAD(&req->queue);
680 
681 	return &req->req;
682 }
683 
684 static void
685 usba_ep_free_request(struct usb_ep *_ep, struct usb_request *_req)
686 {
687 	struct usba_request *req = to_usba_req(_req);
688 
689 	DBG(DBG_GADGET, "ep_free_request: %p, %p\n", _ep, _req);
690 
691 	kfree(req);
692 }
693 
694 static int queue_dma(struct usba_udc *udc, struct usba_ep *ep,
695 		struct usba_request *req, gfp_t gfp_flags)
696 {
697 	unsigned long flags;
698 	int ret;
699 
700 	DBG(DBG_DMA, "%s: req l/%u d/%08x %c%c%c\n",
701 		ep->ep.name, req->req.length, req->req.dma,
702 		req->req.zero ? 'Z' : 'z',
703 		req->req.short_not_ok ? 'S' : 's',
704 		req->req.no_interrupt ? 'I' : 'i');
705 
706 	if (req->req.length > 0x10000) {
707 		/* Lengths from 0 to 65536 (inclusive) are supported */
708 		DBG(DBG_ERR, "invalid request length %u\n", req->req.length);
709 		return -EINVAL;
710 	}
711 
712 	ret = usb_gadget_map_request(&udc->gadget, &req->req, ep->is_in);
713 	if (ret)
714 		return ret;
715 
716 	req->using_dma = 1;
717 	req->ctrl = USBA_BF(DMA_BUF_LEN, req->req.length)
718 			| USBA_DMA_CH_EN | USBA_DMA_END_BUF_IE
719 			| USBA_DMA_END_BUF_EN;
720 
721 	if (!ep->is_in)
722 		req->ctrl |= USBA_DMA_END_TR_EN | USBA_DMA_END_TR_IE;
723 
724 	/*
725 	 * Add this request to the queue and submit for DMA if
726 	 * possible. Check if we're still alive first -- we may have
727 	 * received a reset since last time we checked.
728 	 */
729 	ret = -ESHUTDOWN;
730 	spin_lock_irqsave(&udc->lock, flags);
731 	if (ep->ep.desc) {
732 		if (list_empty(&ep->queue))
733 			submit_request(ep, req);
734 
735 		list_add_tail(&req->queue, &ep->queue);
736 		ret = 0;
737 	}
738 	spin_unlock_irqrestore(&udc->lock, flags);
739 
740 	return ret;
741 }
742 
743 static int
744 usba_ep_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
745 {
746 	struct usba_request *req = to_usba_req(_req);
747 	struct usba_ep *ep = to_usba_ep(_ep);
748 	struct usba_udc *udc = ep->udc;
749 	unsigned long flags;
750 	int ret;
751 
752 	DBG(DBG_GADGET | DBG_QUEUE | DBG_REQ, "%s: queue req %p, len %u\n",
753 			ep->ep.name, req, _req->length);
754 
755 	if (!udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN ||
756 	    !ep->ep.desc)
757 		return -ESHUTDOWN;
758 
759 	req->submitted = 0;
760 	req->using_dma = 0;
761 	req->last_transaction = 0;
762 
763 	_req->status = -EINPROGRESS;
764 	_req->actual = 0;
765 
766 	if (ep->can_dma)
767 		return queue_dma(udc, ep, req, gfp_flags);
768 
769 	/* May have received a reset since last time we checked */
770 	ret = -ESHUTDOWN;
771 	spin_lock_irqsave(&udc->lock, flags);
772 	if (ep->ep.desc) {
773 		list_add_tail(&req->queue, &ep->queue);
774 
775 		if ((!ep_is_control(ep) && ep->is_in) ||
776 			(ep_is_control(ep)
777 				&& (ep->state == DATA_STAGE_IN
778 					|| ep->state == STATUS_STAGE_IN)))
779 			usba_ep_writel(ep, CTL_ENB, USBA_TX_PK_RDY);
780 		else
781 			usba_ep_writel(ep, CTL_ENB, USBA_RX_BK_RDY);
782 		ret = 0;
783 	}
784 	spin_unlock_irqrestore(&udc->lock, flags);
785 
786 	return ret;
787 }
788 
789 static void
790 usba_update_req(struct usba_ep *ep, struct usba_request *req, u32 status)
791 {
792 	req->req.actual = req->req.length - USBA_BFEXT(DMA_BUF_LEN, status);
793 }
794 
795 static int stop_dma(struct usba_ep *ep, u32 *pstatus)
796 {
797 	unsigned int timeout;
798 	u32 status;
799 
800 	/*
801 	 * Stop the DMA controller. When writing both CH_EN
802 	 * and LINK to 0, the other bits are not affected.
803 	 */
804 	usba_dma_writel(ep, CONTROL, 0);
805 
806 	/* Wait for the FIFO to empty */
807 	for (timeout = 40; timeout; --timeout) {
808 		status = usba_dma_readl(ep, STATUS);
809 		if (!(status & USBA_DMA_CH_EN))
810 			break;
811 		udelay(1);
812 	}
813 
814 	if (pstatus)
815 		*pstatus = status;
816 
817 	if (timeout == 0) {
818 		dev_err(&ep->udc->pdev->dev,
819 			"%s: timed out waiting for DMA FIFO to empty\n",
820 			ep->ep.name);
821 		return -ETIMEDOUT;
822 	}
823 
824 	return 0;
825 }
826 
827 static int usba_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
828 {
829 	struct usba_ep *ep = to_usba_ep(_ep);
830 	struct usba_udc *udc = ep->udc;
831 	struct usba_request *req;
832 	unsigned long flags;
833 	u32 status;
834 
835 	DBG(DBG_GADGET | DBG_QUEUE, "ep_dequeue: %s, req %p\n",
836 			ep->ep.name, req);
837 
838 	spin_lock_irqsave(&udc->lock, flags);
839 
840 	list_for_each_entry(req, &ep->queue, queue) {
841 		if (&req->req == _req)
842 			break;
843 	}
844 
845 	if (&req->req != _req) {
846 		spin_unlock_irqrestore(&udc->lock, flags);
847 		return -EINVAL;
848 	}
849 
850 	if (req->using_dma) {
851 		/*
852 		 * If this request is currently being transferred,
853 		 * stop the DMA controller and reset the FIFO.
854 		 */
855 		if (ep->queue.next == &req->queue) {
856 			status = usba_dma_readl(ep, STATUS);
857 			if (status & USBA_DMA_CH_EN)
858 				stop_dma(ep, &status);
859 
860 #ifdef CONFIG_USB_GADGET_DEBUG_FS
861 			ep->last_dma_status = status;
862 #endif
863 
864 			usba_writel(udc, EPT_RST, 1 << ep->index);
865 
866 			usba_update_req(ep, req, status);
867 		}
868 	}
869 
870 	/*
871 	 * Errors should stop the queue from advancing until the
872 	 * completion function returns.
873 	 */
874 	list_del_init(&req->queue);
875 
876 	request_complete(ep, req, -ECONNRESET);
877 
878 	/* Process the next request if any */
879 	submit_next_request(ep);
880 	spin_unlock_irqrestore(&udc->lock, flags);
881 
882 	return 0;
883 }
884 
885 static int usba_ep_set_halt(struct usb_ep *_ep, int value)
886 {
887 	struct usba_ep *ep = to_usba_ep(_ep);
888 	struct usba_udc *udc = ep->udc;
889 	unsigned long flags;
890 	int ret = 0;
891 
892 	DBG(DBG_GADGET, "endpoint %s: %s HALT\n", ep->ep.name,
893 			value ? "set" : "clear");
894 
895 	if (!ep->ep.desc) {
896 		DBG(DBG_ERR, "Attempted to halt uninitialized ep %s\n",
897 				ep->ep.name);
898 		return -ENODEV;
899 	}
900 	if (ep->is_isoc) {
901 		DBG(DBG_ERR, "Attempted to halt isochronous ep %s\n",
902 				ep->ep.name);
903 		return -ENOTTY;
904 	}
905 
906 	spin_lock_irqsave(&udc->lock, flags);
907 
908 	/*
909 	 * We can't halt IN endpoints while there are still data to be
910 	 * transferred
911 	 */
912 	if (!list_empty(&ep->queue)
913 			|| ((value && ep->is_in && (usba_ep_readl(ep, STA)
914 					& USBA_BF(BUSY_BANKS, -1L))))) {
915 		ret = -EAGAIN;
916 	} else {
917 		if (value)
918 			usba_ep_writel(ep, SET_STA, USBA_FORCE_STALL);
919 		else
920 			usba_ep_writel(ep, CLR_STA,
921 					USBA_FORCE_STALL | USBA_TOGGLE_CLR);
922 		usba_ep_readl(ep, STA);
923 	}
924 
925 	spin_unlock_irqrestore(&udc->lock, flags);
926 
927 	return ret;
928 }
929 
930 static int usba_ep_fifo_status(struct usb_ep *_ep)
931 {
932 	struct usba_ep *ep = to_usba_ep(_ep);
933 
934 	return USBA_BFEXT(BYTE_COUNT, usba_ep_readl(ep, STA));
935 }
936 
937 static void usba_ep_fifo_flush(struct usb_ep *_ep)
938 {
939 	struct usba_ep *ep = to_usba_ep(_ep);
940 	struct usba_udc *udc = ep->udc;
941 
942 	usba_writel(udc, EPT_RST, 1 << ep->index);
943 }
944 
945 static const struct usb_ep_ops usba_ep_ops = {
946 	.enable		= usba_ep_enable,
947 	.disable	= usba_ep_disable,
948 	.alloc_request	= usba_ep_alloc_request,
949 	.free_request	= usba_ep_free_request,
950 	.queue		= usba_ep_queue,
951 	.dequeue	= usba_ep_dequeue,
952 	.set_halt	= usba_ep_set_halt,
953 	.fifo_status	= usba_ep_fifo_status,
954 	.fifo_flush	= usba_ep_fifo_flush,
955 };
956 
957 static int usba_udc_get_frame(struct usb_gadget *gadget)
958 {
959 	struct usba_udc *udc = to_usba_udc(gadget);
960 
961 	return USBA_BFEXT(FRAME_NUMBER, usba_readl(udc, FNUM));
962 }
963 
964 static int usba_udc_wakeup(struct usb_gadget *gadget)
965 {
966 	struct usba_udc *udc = to_usba_udc(gadget);
967 	unsigned long flags;
968 	u32 ctrl;
969 	int ret = -EINVAL;
970 
971 	spin_lock_irqsave(&udc->lock, flags);
972 	if (udc->devstatus & (1 << USB_DEVICE_REMOTE_WAKEUP)) {
973 		ctrl = usba_readl(udc, CTRL);
974 		usba_writel(udc, CTRL, ctrl | USBA_REMOTE_WAKE_UP);
975 		ret = 0;
976 	}
977 	spin_unlock_irqrestore(&udc->lock, flags);
978 
979 	return ret;
980 }
981 
982 static int
983 usba_udc_set_selfpowered(struct usb_gadget *gadget, int is_selfpowered)
984 {
985 	struct usba_udc *udc = to_usba_udc(gadget);
986 	unsigned long flags;
987 
988 	spin_lock_irqsave(&udc->lock, flags);
989 	if (is_selfpowered)
990 		udc->devstatus |= 1 << USB_DEVICE_SELF_POWERED;
991 	else
992 		udc->devstatus &= ~(1 << USB_DEVICE_SELF_POWERED);
993 	spin_unlock_irqrestore(&udc->lock, flags);
994 
995 	return 0;
996 }
997 
998 static int atmel_usba_start(struct usb_gadget *gadget,
999 		struct usb_gadget_driver *driver);
1000 static int atmel_usba_stop(struct usb_gadget *gadget);
1001 
1002 static const struct usb_gadget_ops usba_udc_ops = {
1003 	.get_frame		= usba_udc_get_frame,
1004 	.wakeup			= usba_udc_wakeup,
1005 	.set_selfpowered	= usba_udc_set_selfpowered,
1006 	.udc_start		= atmel_usba_start,
1007 	.udc_stop		= atmel_usba_stop,
1008 };
1009 
1010 static struct usb_endpoint_descriptor usba_ep0_desc = {
1011 	.bLength = USB_DT_ENDPOINT_SIZE,
1012 	.bDescriptorType = USB_DT_ENDPOINT,
1013 	.bEndpointAddress = 0,
1014 	.bmAttributes = USB_ENDPOINT_XFER_CONTROL,
1015 	.wMaxPacketSize = cpu_to_le16(64),
1016 	/* FIXME: I have no idea what to put here */
1017 	.bInterval = 1,
1018 };
1019 
1020 static struct usb_gadget usba_gadget_template = {
1021 	.ops		= &usba_udc_ops,
1022 	.max_speed	= USB_SPEED_HIGH,
1023 	.name		= "atmel_usba_udc",
1024 };
1025 
1026 /*
1027  * Called with interrupts disabled and udc->lock held.
1028  */
1029 static void reset_all_endpoints(struct usba_udc *udc)
1030 {
1031 	struct usba_ep *ep;
1032 	struct usba_request *req, *tmp_req;
1033 
1034 	usba_writel(udc, EPT_RST, ~0UL);
1035 
1036 	ep = to_usba_ep(udc->gadget.ep0);
1037 	list_for_each_entry_safe(req, tmp_req, &ep->queue, queue) {
1038 		list_del_init(&req->queue);
1039 		request_complete(ep, req, -ECONNRESET);
1040 	}
1041 
1042 	/* NOTE:  normally, the next call to the gadget driver is in
1043 	 * charge of disabling endpoints... usually disconnect().
1044 	 * The exception would be entering a high speed test mode.
1045 	 *
1046 	 * FIXME remove this code ... and retest thoroughly.
1047 	 */
1048 	list_for_each_entry(ep, &udc->gadget.ep_list, ep.ep_list) {
1049 		if (ep->ep.desc) {
1050 			spin_unlock(&udc->lock);
1051 			usba_ep_disable(&ep->ep);
1052 			spin_lock(&udc->lock);
1053 		}
1054 	}
1055 }
1056 
1057 static struct usba_ep *get_ep_by_addr(struct usba_udc *udc, u16 wIndex)
1058 {
1059 	struct usba_ep *ep;
1060 
1061 	if ((wIndex & USB_ENDPOINT_NUMBER_MASK) == 0)
1062 		return to_usba_ep(udc->gadget.ep0);
1063 
1064 	list_for_each_entry (ep, &udc->gadget.ep_list, ep.ep_list) {
1065 		u8 bEndpointAddress;
1066 
1067 		if (!ep->ep.desc)
1068 			continue;
1069 		bEndpointAddress = ep->ep.desc->bEndpointAddress;
1070 		if ((wIndex ^ bEndpointAddress) & USB_DIR_IN)
1071 			continue;
1072 		if ((bEndpointAddress & USB_ENDPOINT_NUMBER_MASK)
1073 				== (wIndex & USB_ENDPOINT_NUMBER_MASK))
1074 			return ep;
1075 	}
1076 
1077 	return NULL;
1078 }
1079 
1080 /* Called with interrupts disabled and udc->lock held */
1081 static inline void set_protocol_stall(struct usba_udc *udc, struct usba_ep *ep)
1082 {
1083 	usba_ep_writel(ep, SET_STA, USBA_FORCE_STALL);
1084 	ep->state = WAIT_FOR_SETUP;
1085 }
1086 
1087 static inline int is_stalled(struct usba_udc *udc, struct usba_ep *ep)
1088 {
1089 	if (usba_ep_readl(ep, STA) & USBA_FORCE_STALL)
1090 		return 1;
1091 	return 0;
1092 }
1093 
1094 static inline void set_address(struct usba_udc *udc, unsigned int addr)
1095 {
1096 	u32 regval;
1097 
1098 	DBG(DBG_BUS, "setting address %u...\n", addr);
1099 	regval = usba_readl(udc, CTRL);
1100 	regval = USBA_BFINS(DEV_ADDR, addr, regval);
1101 	usba_writel(udc, CTRL, regval);
1102 }
1103 
1104 static int do_test_mode(struct usba_udc *udc)
1105 {
1106 	static const char test_packet_buffer[] = {
1107 		/* JKJKJKJK * 9 */
1108 		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
1109 		/* JJKKJJKK * 8 */
1110 		0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA, 0xAA,
1111 		/* JJKKJJKK * 8 */
1112 		0xEE, 0xEE, 0xEE, 0xEE, 0xEE, 0xEE, 0xEE, 0xEE,
1113 		/* JJJJJJJKKKKKKK * 8 */
1114 		0xFE, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
1115 		0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF,
1116 		/* JJJJJJJK * 8 */
1117 		0x7F, 0xBF, 0xDF, 0xEF, 0xF7, 0xFB, 0xFD,
1118 		/* {JKKKKKKK * 10}, JK */
1119 		0xFC, 0x7E, 0xBF, 0xDF, 0xEF, 0xF7, 0xFB, 0xFD, 0x7E
1120 	};
1121 	struct usba_ep *ep;
1122 	struct device *dev = &udc->pdev->dev;
1123 	int test_mode;
1124 
1125 	test_mode = udc->test_mode;
1126 
1127 	/* Start from a clean slate */
1128 	reset_all_endpoints(udc);
1129 
1130 	switch (test_mode) {
1131 	case 0x0100:
1132 		/* Test_J */
1133 		usba_writel(udc, TST, USBA_TST_J_MODE);
1134 		dev_info(dev, "Entering Test_J mode...\n");
1135 		break;
1136 	case 0x0200:
1137 		/* Test_K */
1138 		usba_writel(udc, TST, USBA_TST_K_MODE);
1139 		dev_info(dev, "Entering Test_K mode...\n");
1140 		break;
1141 	case 0x0300:
1142 		/*
1143 		 * Test_SE0_NAK: Force high-speed mode and set up ep0
1144 		 * for Bulk IN transfers
1145 		 */
1146 		ep = &udc->usba_ep[0];
1147 		usba_writel(udc, TST,
1148 				USBA_BF(SPEED_CFG, USBA_SPEED_CFG_FORCE_HIGH));
1149 		usba_ep_writel(ep, CFG,
1150 				USBA_BF(EPT_SIZE, USBA_EPT_SIZE_64)
1151 				| USBA_EPT_DIR_IN
1152 				| USBA_BF(EPT_TYPE, USBA_EPT_TYPE_BULK)
1153 				| USBA_BF(BK_NUMBER, 1));
1154 		if (!(usba_ep_readl(ep, CFG) & USBA_EPT_MAPPED)) {
1155 			set_protocol_stall(udc, ep);
1156 			dev_err(dev, "Test_SE0_NAK: ep0 not mapped\n");
1157 		} else {
1158 			usba_ep_writel(ep, CTL_ENB, USBA_EPT_ENABLE);
1159 			dev_info(dev, "Entering Test_SE0_NAK mode...\n");
1160 		}
1161 		break;
1162 	case 0x0400:
1163 		/* Test_Packet */
1164 		ep = &udc->usba_ep[0];
1165 		usba_ep_writel(ep, CFG,
1166 				USBA_BF(EPT_SIZE, USBA_EPT_SIZE_64)
1167 				| USBA_EPT_DIR_IN
1168 				| USBA_BF(EPT_TYPE, USBA_EPT_TYPE_BULK)
1169 				| USBA_BF(BK_NUMBER, 1));
1170 		if (!(usba_ep_readl(ep, CFG) & USBA_EPT_MAPPED)) {
1171 			set_protocol_stall(udc, ep);
1172 			dev_err(dev, "Test_Packet: ep0 not mapped\n");
1173 		} else {
1174 			usba_ep_writel(ep, CTL_ENB, USBA_EPT_ENABLE);
1175 			usba_writel(udc, TST, USBA_TST_PKT_MODE);
1176 			memcpy_toio(ep->fifo, test_packet_buffer,
1177 					sizeof(test_packet_buffer));
1178 			usba_ep_writel(ep, SET_STA, USBA_TX_PK_RDY);
1179 			dev_info(dev, "Entering Test_Packet mode...\n");
1180 		}
1181 		break;
1182 	default:
1183 		dev_err(dev, "Invalid test mode: 0x%04x\n", test_mode);
1184 		return -EINVAL;
1185 	}
1186 
1187 	return 0;
1188 }
1189 
1190 /* Avoid overly long expressions */
1191 static inline bool feature_is_dev_remote_wakeup(struct usb_ctrlrequest *crq)
1192 {
1193 	if (crq->wValue == cpu_to_le16(USB_DEVICE_REMOTE_WAKEUP))
1194 		return true;
1195 	return false;
1196 }
1197 
1198 static inline bool feature_is_dev_test_mode(struct usb_ctrlrequest *crq)
1199 {
1200 	if (crq->wValue == cpu_to_le16(USB_DEVICE_TEST_MODE))
1201 		return true;
1202 	return false;
1203 }
1204 
1205 static inline bool feature_is_ep_halt(struct usb_ctrlrequest *crq)
1206 {
1207 	if (crq->wValue == cpu_to_le16(USB_ENDPOINT_HALT))
1208 		return true;
1209 	return false;
1210 }
1211 
1212 static int handle_ep0_setup(struct usba_udc *udc, struct usba_ep *ep,
1213 		struct usb_ctrlrequest *crq)
1214 {
1215 	int retval = 0;
1216 
1217 	switch (crq->bRequest) {
1218 	case USB_REQ_GET_STATUS: {
1219 		u16 status;
1220 
1221 		if (crq->bRequestType == (USB_DIR_IN | USB_RECIP_DEVICE)) {
1222 			status = cpu_to_le16(udc->devstatus);
1223 		} else if (crq->bRequestType
1224 				== (USB_DIR_IN | USB_RECIP_INTERFACE)) {
1225 			status = cpu_to_le16(0);
1226 		} else if (crq->bRequestType
1227 				== (USB_DIR_IN | USB_RECIP_ENDPOINT)) {
1228 			struct usba_ep *target;
1229 
1230 			target = get_ep_by_addr(udc, le16_to_cpu(crq->wIndex));
1231 			if (!target)
1232 				goto stall;
1233 
1234 			status = 0;
1235 			if (is_stalled(udc, target))
1236 				status |= cpu_to_le16(1);
1237 		} else
1238 			goto delegate;
1239 
1240 		/* Write directly to the FIFO. No queueing is done. */
1241 		if (crq->wLength != cpu_to_le16(sizeof(status)))
1242 			goto stall;
1243 		ep->state = DATA_STAGE_IN;
1244 		__raw_writew(status, ep->fifo);
1245 		usba_ep_writel(ep, SET_STA, USBA_TX_PK_RDY);
1246 		break;
1247 	}
1248 
1249 	case USB_REQ_CLEAR_FEATURE: {
1250 		if (crq->bRequestType == USB_RECIP_DEVICE) {
1251 			if (feature_is_dev_remote_wakeup(crq))
1252 				udc->devstatus
1253 					&= ~(1 << USB_DEVICE_REMOTE_WAKEUP);
1254 			else
1255 				/* Can't CLEAR_FEATURE TEST_MODE */
1256 				goto stall;
1257 		} else if (crq->bRequestType == USB_RECIP_ENDPOINT) {
1258 			struct usba_ep *target;
1259 
1260 			if (crq->wLength != cpu_to_le16(0)
1261 					|| !feature_is_ep_halt(crq))
1262 				goto stall;
1263 			target = get_ep_by_addr(udc, le16_to_cpu(crq->wIndex));
1264 			if (!target)
1265 				goto stall;
1266 
1267 			usba_ep_writel(target, CLR_STA, USBA_FORCE_STALL);
1268 			if (target->index != 0)
1269 				usba_ep_writel(target, CLR_STA,
1270 						USBA_TOGGLE_CLR);
1271 		} else {
1272 			goto delegate;
1273 		}
1274 
1275 		send_status(udc, ep);
1276 		break;
1277 	}
1278 
1279 	case USB_REQ_SET_FEATURE: {
1280 		if (crq->bRequestType == USB_RECIP_DEVICE) {
1281 			if (feature_is_dev_test_mode(crq)) {
1282 				send_status(udc, ep);
1283 				ep->state = STATUS_STAGE_TEST;
1284 				udc->test_mode = le16_to_cpu(crq->wIndex);
1285 				return 0;
1286 			} else if (feature_is_dev_remote_wakeup(crq)) {
1287 				udc->devstatus |= 1 << USB_DEVICE_REMOTE_WAKEUP;
1288 			} else {
1289 				goto stall;
1290 			}
1291 		} else if (crq->bRequestType == USB_RECIP_ENDPOINT) {
1292 			struct usba_ep *target;
1293 
1294 			if (crq->wLength != cpu_to_le16(0)
1295 					|| !feature_is_ep_halt(crq))
1296 				goto stall;
1297 
1298 			target = get_ep_by_addr(udc, le16_to_cpu(crq->wIndex));
1299 			if (!target)
1300 				goto stall;
1301 
1302 			usba_ep_writel(target, SET_STA, USBA_FORCE_STALL);
1303 		} else
1304 			goto delegate;
1305 
1306 		send_status(udc, ep);
1307 		break;
1308 	}
1309 
1310 	case USB_REQ_SET_ADDRESS:
1311 		if (crq->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE))
1312 			goto delegate;
1313 
1314 		set_address(udc, le16_to_cpu(crq->wValue));
1315 		send_status(udc, ep);
1316 		ep->state = STATUS_STAGE_ADDR;
1317 		break;
1318 
1319 	default:
1320 delegate:
1321 		spin_unlock(&udc->lock);
1322 		retval = udc->driver->setup(&udc->gadget, crq);
1323 		spin_lock(&udc->lock);
1324 	}
1325 
1326 	return retval;
1327 
1328 stall:
1329 	pr_err("udc: %s: Invalid setup request: %02x.%02x v%04x i%04x l%d, "
1330 		"halting endpoint...\n",
1331 		ep->ep.name, crq->bRequestType, crq->bRequest,
1332 		le16_to_cpu(crq->wValue), le16_to_cpu(crq->wIndex),
1333 		le16_to_cpu(crq->wLength));
1334 	set_protocol_stall(udc, ep);
1335 	return -1;
1336 }
1337 
1338 static void usba_control_irq(struct usba_udc *udc, struct usba_ep *ep)
1339 {
1340 	struct usba_request *req;
1341 	u32 epstatus;
1342 	u32 epctrl;
1343 
1344 restart:
1345 	epstatus = usba_ep_readl(ep, STA);
1346 	epctrl = usba_ep_readl(ep, CTL);
1347 
1348 	DBG(DBG_INT, "%s [%d]: s/%08x c/%08x\n",
1349 			ep->ep.name, ep->state, epstatus, epctrl);
1350 
1351 	req = NULL;
1352 	if (!list_empty(&ep->queue))
1353 		req = list_entry(ep->queue.next,
1354 				 struct usba_request, queue);
1355 
1356 	if ((epctrl & USBA_TX_PK_RDY) && !(epstatus & USBA_TX_PK_RDY)) {
1357 		if (req->submitted)
1358 			next_fifo_transaction(ep, req);
1359 		else
1360 			submit_request(ep, req);
1361 
1362 		if (req->last_transaction) {
1363 			usba_ep_writel(ep, CTL_DIS, USBA_TX_PK_RDY);
1364 			usba_ep_writel(ep, CTL_ENB, USBA_TX_COMPLETE);
1365 		}
1366 		goto restart;
1367 	}
1368 	if ((epstatus & epctrl) & USBA_TX_COMPLETE) {
1369 		usba_ep_writel(ep, CLR_STA, USBA_TX_COMPLETE);
1370 
1371 		switch (ep->state) {
1372 		case DATA_STAGE_IN:
1373 			usba_ep_writel(ep, CTL_ENB, USBA_RX_BK_RDY);
1374 			usba_ep_writel(ep, CTL_DIS, USBA_TX_COMPLETE);
1375 			ep->state = STATUS_STAGE_OUT;
1376 			break;
1377 		case STATUS_STAGE_ADDR:
1378 			/* Activate our new address */
1379 			usba_writel(udc, CTRL, (usba_readl(udc, CTRL)
1380 						| USBA_FADDR_EN));
1381 			usba_ep_writel(ep, CTL_DIS, USBA_TX_COMPLETE);
1382 			ep->state = WAIT_FOR_SETUP;
1383 			break;
1384 		case STATUS_STAGE_IN:
1385 			if (req) {
1386 				list_del_init(&req->queue);
1387 				request_complete(ep, req, 0);
1388 				submit_next_request(ep);
1389 			}
1390 			usba_ep_writel(ep, CTL_DIS, USBA_TX_COMPLETE);
1391 			ep->state = WAIT_FOR_SETUP;
1392 			break;
1393 		case STATUS_STAGE_TEST:
1394 			usba_ep_writel(ep, CTL_DIS, USBA_TX_COMPLETE);
1395 			ep->state = WAIT_FOR_SETUP;
1396 			if (do_test_mode(udc))
1397 				set_protocol_stall(udc, ep);
1398 			break;
1399 		default:
1400 			pr_err("udc: %s: TXCOMP: Invalid endpoint state %d, "
1401 				"halting endpoint...\n",
1402 				ep->ep.name, ep->state);
1403 			set_protocol_stall(udc, ep);
1404 			break;
1405 		}
1406 
1407 		goto restart;
1408 	}
1409 	if ((epstatus & epctrl) & USBA_RX_BK_RDY) {
1410 		switch (ep->state) {
1411 		case STATUS_STAGE_OUT:
1412 			usba_ep_writel(ep, CLR_STA, USBA_RX_BK_RDY);
1413 			usba_ep_writel(ep, CTL_DIS, USBA_RX_BK_RDY);
1414 
1415 			if (req) {
1416 				list_del_init(&req->queue);
1417 				request_complete(ep, req, 0);
1418 			}
1419 			ep->state = WAIT_FOR_SETUP;
1420 			break;
1421 
1422 		case DATA_STAGE_OUT:
1423 			receive_data(ep);
1424 			break;
1425 
1426 		default:
1427 			usba_ep_writel(ep, CLR_STA, USBA_RX_BK_RDY);
1428 			usba_ep_writel(ep, CTL_DIS, USBA_RX_BK_RDY);
1429 			pr_err("udc: %s: RXRDY: Invalid endpoint state %d, "
1430 				"halting endpoint...\n",
1431 				ep->ep.name, ep->state);
1432 			set_protocol_stall(udc, ep);
1433 			break;
1434 		}
1435 
1436 		goto restart;
1437 	}
1438 	if (epstatus & USBA_RX_SETUP) {
1439 		union {
1440 			struct usb_ctrlrequest crq;
1441 			unsigned long data[2];
1442 		} crq;
1443 		unsigned int pkt_len;
1444 		int ret;
1445 
1446 		if (ep->state != WAIT_FOR_SETUP) {
1447 			/*
1448 			 * Didn't expect a SETUP packet at this
1449 			 * point. Clean up any pending requests (which
1450 			 * may be successful).
1451 			 */
1452 			int status = -EPROTO;
1453 
1454 			/*
1455 			 * RXRDY and TXCOMP are dropped when SETUP
1456 			 * packets arrive.  Just pretend we received
1457 			 * the status packet.
1458 			 */
1459 			if (ep->state == STATUS_STAGE_OUT
1460 					|| ep->state == STATUS_STAGE_IN) {
1461 				usba_ep_writel(ep, CTL_DIS, USBA_RX_BK_RDY);
1462 				status = 0;
1463 			}
1464 
1465 			if (req) {
1466 				list_del_init(&req->queue);
1467 				request_complete(ep, req, status);
1468 			}
1469 		}
1470 
1471 		pkt_len = USBA_BFEXT(BYTE_COUNT, usba_ep_readl(ep, STA));
1472 		DBG(DBG_HW, "Packet length: %u\n", pkt_len);
1473 		if (pkt_len != sizeof(crq)) {
1474 			pr_warning("udc: Invalid packet length %u "
1475 				"(expected %zu)\n", pkt_len, sizeof(crq));
1476 			set_protocol_stall(udc, ep);
1477 			return;
1478 		}
1479 
1480 		DBG(DBG_FIFO, "Copying ctrl request from 0x%p:\n", ep->fifo);
1481 		memcpy_fromio(crq.data, ep->fifo, sizeof(crq));
1482 
1483 		/* Free up one bank in the FIFO so that we can
1484 		 * generate or receive a reply right away. */
1485 		usba_ep_writel(ep, CLR_STA, USBA_RX_SETUP);
1486 
1487 		/* printk(KERN_DEBUG "setup: %d: %02x.%02x\n",
1488 			ep->state, crq.crq.bRequestType,
1489 			crq.crq.bRequest); */
1490 
1491 		if (crq.crq.bRequestType & USB_DIR_IN) {
1492 			/*
1493 			 * The USB 2.0 spec states that "if wLength is
1494 			 * zero, there is no data transfer phase."
1495 			 * However, testusb #14 seems to actually
1496 			 * expect a data phase even if wLength = 0...
1497 			 */
1498 			ep->state = DATA_STAGE_IN;
1499 		} else {
1500 			if (crq.crq.wLength != cpu_to_le16(0))
1501 				ep->state = DATA_STAGE_OUT;
1502 			else
1503 				ep->state = STATUS_STAGE_IN;
1504 		}
1505 
1506 		ret = -1;
1507 		if (ep->index == 0)
1508 			ret = handle_ep0_setup(udc, ep, &crq.crq);
1509 		else {
1510 			spin_unlock(&udc->lock);
1511 			ret = udc->driver->setup(&udc->gadget, &crq.crq);
1512 			spin_lock(&udc->lock);
1513 		}
1514 
1515 		DBG(DBG_BUS, "req %02x.%02x, length %d, state %d, ret %d\n",
1516 			crq.crq.bRequestType, crq.crq.bRequest,
1517 			le16_to_cpu(crq.crq.wLength), ep->state, ret);
1518 
1519 		if (ret < 0) {
1520 			/* Let the host know that we failed */
1521 			set_protocol_stall(udc, ep);
1522 		}
1523 	}
1524 }
1525 
1526 static void usba_ep_irq(struct usba_udc *udc, struct usba_ep *ep)
1527 {
1528 	struct usba_request *req;
1529 	u32 epstatus;
1530 	u32 epctrl;
1531 
1532 	epstatus = usba_ep_readl(ep, STA);
1533 	epctrl = usba_ep_readl(ep, CTL);
1534 
1535 	DBG(DBG_INT, "%s: interrupt, status: 0x%08x\n", ep->ep.name, epstatus);
1536 
1537 	while ((epctrl & USBA_TX_PK_RDY) && !(epstatus & USBA_TX_PK_RDY)) {
1538 		DBG(DBG_BUS, "%s: TX PK ready\n", ep->ep.name);
1539 
1540 		if (list_empty(&ep->queue)) {
1541 			dev_warn(&udc->pdev->dev, "ep_irq: queue empty\n");
1542 			usba_ep_writel(ep, CTL_DIS, USBA_TX_PK_RDY);
1543 			return;
1544 		}
1545 
1546 		req = list_entry(ep->queue.next, struct usba_request, queue);
1547 
1548 		if (req->using_dma) {
1549 			/* Send a zero-length packet */
1550 			usba_ep_writel(ep, SET_STA,
1551 					USBA_TX_PK_RDY);
1552 			usba_ep_writel(ep, CTL_DIS,
1553 					USBA_TX_PK_RDY);
1554 			list_del_init(&req->queue);
1555 			submit_next_request(ep);
1556 			request_complete(ep, req, 0);
1557 		} else {
1558 			if (req->submitted)
1559 				next_fifo_transaction(ep, req);
1560 			else
1561 				submit_request(ep, req);
1562 
1563 			if (req->last_transaction) {
1564 				list_del_init(&req->queue);
1565 				submit_next_request(ep);
1566 				request_complete(ep, req, 0);
1567 			}
1568 		}
1569 
1570 		epstatus = usba_ep_readl(ep, STA);
1571 		epctrl = usba_ep_readl(ep, CTL);
1572 	}
1573 	if ((epstatus & epctrl) & USBA_RX_BK_RDY) {
1574 		DBG(DBG_BUS, "%s: RX data ready\n", ep->ep.name);
1575 		receive_data(ep);
1576 	}
1577 }
1578 
1579 static void usba_dma_irq(struct usba_udc *udc, struct usba_ep *ep)
1580 {
1581 	struct usba_request *req;
1582 	u32 status, control, pending;
1583 
1584 	status = usba_dma_readl(ep, STATUS);
1585 	control = usba_dma_readl(ep, CONTROL);
1586 #ifdef CONFIG_USB_GADGET_DEBUG_FS
1587 	ep->last_dma_status = status;
1588 #endif
1589 	pending = status & control;
1590 	DBG(DBG_INT | DBG_DMA, "dma irq, s/%#08x, c/%#08x\n", status, control);
1591 
1592 	if (status & USBA_DMA_CH_EN) {
1593 		dev_err(&udc->pdev->dev,
1594 			"DMA_CH_EN is set after transfer is finished!\n");
1595 		dev_err(&udc->pdev->dev,
1596 			"status=%#08x, pending=%#08x, control=%#08x\n",
1597 			status, pending, control);
1598 
1599 		/*
1600 		 * try to pretend nothing happened. We might have to
1601 		 * do something here...
1602 		 */
1603 	}
1604 
1605 	if (list_empty(&ep->queue))
1606 		/* Might happen if a reset comes along at the right moment */
1607 		return;
1608 
1609 	if (pending & (USBA_DMA_END_TR_ST | USBA_DMA_END_BUF_ST)) {
1610 		req = list_entry(ep->queue.next, struct usba_request, queue);
1611 		usba_update_req(ep, req, status);
1612 
1613 		list_del_init(&req->queue);
1614 		submit_next_request(ep);
1615 		request_complete(ep, req, 0);
1616 	}
1617 }
1618 
1619 static irqreturn_t usba_udc_irq(int irq, void *devid)
1620 {
1621 	struct usba_udc *udc = devid;
1622 	u32 status;
1623 	u32 dma_status;
1624 	u32 ep_status;
1625 
1626 	spin_lock(&udc->lock);
1627 
1628 	status = usba_readl(udc, INT_STA);
1629 	DBG(DBG_INT, "irq, status=%#08x\n", status);
1630 
1631 	if (status & USBA_DET_SUSPEND) {
1632 		toggle_bias(0);
1633 		usba_writel(udc, INT_CLR, USBA_DET_SUSPEND);
1634 		DBG(DBG_BUS, "Suspend detected\n");
1635 		if (udc->gadget.speed != USB_SPEED_UNKNOWN
1636 				&& udc->driver && udc->driver->suspend) {
1637 			spin_unlock(&udc->lock);
1638 			udc->driver->suspend(&udc->gadget);
1639 			spin_lock(&udc->lock);
1640 		}
1641 	}
1642 
1643 	if (status & USBA_WAKE_UP) {
1644 		toggle_bias(1);
1645 		usba_writel(udc, INT_CLR, USBA_WAKE_UP);
1646 		DBG(DBG_BUS, "Wake Up CPU detected\n");
1647 	}
1648 
1649 	if (status & USBA_END_OF_RESUME) {
1650 		usba_writel(udc, INT_CLR, USBA_END_OF_RESUME);
1651 		DBG(DBG_BUS, "Resume detected\n");
1652 		if (udc->gadget.speed != USB_SPEED_UNKNOWN
1653 				&& udc->driver && udc->driver->resume) {
1654 			spin_unlock(&udc->lock);
1655 			udc->driver->resume(&udc->gadget);
1656 			spin_lock(&udc->lock);
1657 		}
1658 	}
1659 
1660 	dma_status = USBA_BFEXT(DMA_INT, status);
1661 	if (dma_status) {
1662 		int i;
1663 
1664 		for (i = 1; i <= USBA_NR_DMAS; i++)
1665 			if (dma_status & (1 << i))
1666 				usba_dma_irq(udc, &udc->usba_ep[i]);
1667 	}
1668 
1669 	ep_status = USBA_BFEXT(EPT_INT, status);
1670 	if (ep_status) {
1671 		int i;
1672 
1673 		for (i = 0; i < udc->num_ep; i++)
1674 			if (ep_status & (1 << i)) {
1675 				if (ep_is_control(&udc->usba_ep[i]))
1676 					usba_control_irq(udc, &udc->usba_ep[i]);
1677 				else
1678 					usba_ep_irq(udc, &udc->usba_ep[i]);
1679 			}
1680 	}
1681 
1682 	if (status & USBA_END_OF_RESET) {
1683 		struct usba_ep *ep0;
1684 
1685 		usba_writel(udc, INT_CLR, USBA_END_OF_RESET);
1686 		reset_all_endpoints(udc);
1687 
1688 		if (udc->gadget.speed != USB_SPEED_UNKNOWN && udc->driver) {
1689 			udc->gadget.speed = USB_SPEED_UNKNOWN;
1690 			spin_unlock(&udc->lock);
1691 			usb_gadget_udc_reset(&udc->gadget, udc->driver);
1692 			spin_lock(&udc->lock);
1693 		}
1694 
1695 		if (status & USBA_HIGH_SPEED)
1696 			udc->gadget.speed = USB_SPEED_HIGH;
1697 		else
1698 			udc->gadget.speed = USB_SPEED_FULL;
1699 		DBG(DBG_BUS, "%s bus reset detected\n",
1700 		    usb_speed_string(udc->gadget.speed));
1701 
1702 		ep0 = &udc->usba_ep[0];
1703 		ep0->ep.desc = &usba_ep0_desc;
1704 		ep0->state = WAIT_FOR_SETUP;
1705 		usba_ep_writel(ep0, CFG,
1706 				(USBA_BF(EPT_SIZE, EP0_EPT_SIZE)
1707 				| USBA_BF(EPT_TYPE, USBA_EPT_TYPE_CONTROL)
1708 				| USBA_BF(BK_NUMBER, USBA_BK_NUMBER_ONE)));
1709 		usba_ep_writel(ep0, CTL_ENB,
1710 				USBA_EPT_ENABLE | USBA_RX_SETUP);
1711 		usba_writel(udc, INT_ENB,
1712 				(usba_readl(udc, INT_ENB)
1713 				| USBA_BF(EPT_INT, 1)
1714 				| USBA_DET_SUSPEND
1715 				| USBA_END_OF_RESUME));
1716 
1717 		/*
1718 		 * Unclear why we hit this irregularly, e.g. in usbtest,
1719 		 * but it's clearly harmless...
1720 		 */
1721 		if (!(usba_ep_readl(ep0, CFG) & USBA_EPT_MAPPED))
1722 			dev_dbg(&udc->pdev->dev,
1723 				 "ODD: EP0 configuration is invalid!\n");
1724 	}
1725 
1726 	spin_unlock(&udc->lock);
1727 
1728 	return IRQ_HANDLED;
1729 }
1730 
1731 static irqreturn_t usba_vbus_irq(int irq, void *devid)
1732 {
1733 	struct usba_udc *udc = devid;
1734 	int vbus;
1735 
1736 	/* debounce */
1737 	udelay(10);
1738 
1739 	spin_lock(&udc->lock);
1740 
1741 	/* May happen if Vbus pin toggles during probe() */
1742 	if (!udc->driver)
1743 		goto out;
1744 
1745 	vbus = vbus_is_present(udc);
1746 	if (vbus != udc->vbus_prev) {
1747 		if (vbus) {
1748 			toggle_bias(1);
1749 			usba_writel(udc, CTRL, USBA_ENABLE_MASK);
1750 			usba_writel(udc, INT_ENB, USBA_END_OF_RESET);
1751 		} else {
1752 			udc->gadget.speed = USB_SPEED_UNKNOWN;
1753 			reset_all_endpoints(udc);
1754 			toggle_bias(0);
1755 			usba_writel(udc, CTRL, USBA_DISABLE_MASK);
1756 			if (udc->driver->disconnect) {
1757 				spin_unlock(&udc->lock);
1758 				udc->driver->disconnect(&udc->gadget);
1759 				spin_lock(&udc->lock);
1760 			}
1761 		}
1762 		udc->vbus_prev = vbus;
1763 	}
1764 
1765 out:
1766 	spin_unlock(&udc->lock);
1767 
1768 	return IRQ_HANDLED;
1769 }
1770 
1771 static int atmel_usba_start(struct usb_gadget *gadget,
1772 		struct usb_gadget_driver *driver)
1773 {
1774 	int ret;
1775 	struct usba_udc *udc = container_of(gadget, struct usba_udc, gadget);
1776 	unsigned long flags;
1777 
1778 	spin_lock_irqsave(&udc->lock, flags);
1779 
1780 	udc->devstatus = 1 << USB_DEVICE_SELF_POWERED;
1781 	udc->driver = driver;
1782 	spin_unlock_irqrestore(&udc->lock, flags);
1783 
1784 	ret = clk_prepare_enable(udc->pclk);
1785 	if (ret)
1786 		return ret;
1787 	ret = clk_prepare_enable(udc->hclk);
1788 	if (ret) {
1789 		clk_disable_unprepare(udc->pclk);
1790 		return ret;
1791 	}
1792 
1793 	udc->vbus_prev = 0;
1794 	if (gpio_is_valid(udc->vbus_pin))
1795 		enable_irq(gpio_to_irq(udc->vbus_pin));
1796 
1797 	/* If Vbus is present, enable the controller and wait for reset */
1798 	spin_lock_irqsave(&udc->lock, flags);
1799 	if (vbus_is_present(udc) && udc->vbus_prev == 0) {
1800 		toggle_bias(1);
1801 		usba_writel(udc, CTRL, USBA_ENABLE_MASK);
1802 		usba_writel(udc, INT_ENB, USBA_END_OF_RESET);
1803 	}
1804 	spin_unlock_irqrestore(&udc->lock, flags);
1805 
1806 	return 0;
1807 }
1808 
1809 static int atmel_usba_stop(struct usb_gadget *gadget)
1810 {
1811 	struct usba_udc *udc = container_of(gadget, struct usba_udc, gadget);
1812 	unsigned long flags;
1813 
1814 	if (gpio_is_valid(udc->vbus_pin))
1815 		disable_irq(gpio_to_irq(udc->vbus_pin));
1816 
1817 	spin_lock_irqsave(&udc->lock, flags);
1818 	udc->gadget.speed = USB_SPEED_UNKNOWN;
1819 	reset_all_endpoints(udc);
1820 	spin_unlock_irqrestore(&udc->lock, flags);
1821 
1822 	/* This will also disable the DP pullup */
1823 	toggle_bias(0);
1824 	usba_writel(udc, CTRL, USBA_DISABLE_MASK);
1825 
1826 	clk_disable_unprepare(udc->hclk);
1827 	clk_disable_unprepare(udc->pclk);
1828 
1829 	udc->driver = NULL;
1830 
1831 	return 0;
1832 }
1833 
1834 #ifdef CONFIG_OF
1835 static struct usba_ep * atmel_udc_of_init(struct platform_device *pdev,
1836 						    struct usba_udc *udc)
1837 {
1838 	u32 val;
1839 	const char *name;
1840 	enum of_gpio_flags flags;
1841 	struct device_node *np = pdev->dev.of_node;
1842 	struct device_node *pp;
1843 	int i, ret;
1844 	struct usba_ep *eps, *ep;
1845 
1846 	udc->num_ep = 0;
1847 
1848 	udc->vbus_pin = of_get_named_gpio_flags(np, "atmel,vbus-gpio", 0,
1849 						&flags);
1850 	udc->vbus_pin_inverted = (flags & OF_GPIO_ACTIVE_LOW) ? 1 : 0;
1851 
1852 	pp = NULL;
1853 	while ((pp = of_get_next_child(np, pp)))
1854 		udc->num_ep++;
1855 
1856 	eps = devm_kzalloc(&pdev->dev, sizeof(struct usba_ep) * udc->num_ep,
1857 			   GFP_KERNEL);
1858 	if (!eps)
1859 		return ERR_PTR(-ENOMEM);
1860 
1861 	udc->gadget.ep0 = &eps[0].ep;
1862 
1863 	INIT_LIST_HEAD(&eps[0].ep.ep_list);
1864 
1865 	pp = NULL;
1866 	i = 0;
1867 	while ((pp = of_get_next_child(np, pp))) {
1868 		ep = &eps[i];
1869 
1870 		ret = of_property_read_u32(pp, "reg", &val);
1871 		if (ret) {
1872 			dev_err(&pdev->dev, "of_probe: reg error(%d)\n", ret);
1873 			goto err;
1874 		}
1875 		ep->index = val;
1876 
1877 		ret = of_property_read_u32(pp, "atmel,fifo-size", &val);
1878 		if (ret) {
1879 			dev_err(&pdev->dev, "of_probe: fifo-size error(%d)\n", ret);
1880 			goto err;
1881 		}
1882 		ep->fifo_size = val;
1883 
1884 		ret = of_property_read_u32(pp, "atmel,nb-banks", &val);
1885 		if (ret) {
1886 			dev_err(&pdev->dev, "of_probe: nb-banks error(%d)\n", ret);
1887 			goto err;
1888 		}
1889 		ep->nr_banks = val;
1890 
1891 		ep->can_dma = of_property_read_bool(pp, "atmel,can-dma");
1892 		ep->can_isoc = of_property_read_bool(pp, "atmel,can-isoc");
1893 
1894 		ret = of_property_read_string(pp, "name", &name);
1895 		ep->ep.name = name;
1896 
1897 		ep->ep_regs = udc->regs + USBA_EPT_BASE(i);
1898 		ep->dma_regs = udc->regs + USBA_DMA_BASE(i);
1899 		ep->fifo = udc->fifo + USBA_FIFO_BASE(i);
1900 		ep->ep.ops = &usba_ep_ops;
1901 		usb_ep_set_maxpacket_limit(&ep->ep, ep->fifo_size);
1902 		ep->udc = udc;
1903 		INIT_LIST_HEAD(&ep->queue);
1904 
1905 		if (i)
1906 			list_add_tail(&ep->ep.ep_list, &udc->gadget.ep_list);
1907 
1908 		i++;
1909 	}
1910 
1911 	if (i == 0) {
1912 		dev_err(&pdev->dev, "of_probe: no endpoint specified\n");
1913 		ret = -EINVAL;
1914 		goto err;
1915 	}
1916 
1917 	return eps;
1918 err:
1919 	return ERR_PTR(ret);
1920 }
1921 #else
1922 static struct usba_ep * atmel_udc_of_init(struct platform_device *pdev,
1923 						    struct usba_udc *udc)
1924 {
1925 	return ERR_PTR(-ENOSYS);
1926 }
1927 #endif
1928 
1929 static struct usba_ep * usba_udc_pdata(struct platform_device *pdev,
1930 						 struct usba_udc *udc)
1931 {
1932 	struct usba_platform_data *pdata = dev_get_platdata(&pdev->dev);
1933 	struct usba_ep *eps;
1934 	int i;
1935 
1936 	if (!pdata)
1937 		return ERR_PTR(-ENXIO);
1938 
1939 	eps = devm_kzalloc(&pdev->dev, sizeof(struct usba_ep) * pdata->num_ep,
1940 			   GFP_KERNEL);
1941 	if (!eps)
1942 		return ERR_PTR(-ENOMEM);
1943 
1944 	udc->gadget.ep0 = &eps[0].ep;
1945 
1946 	udc->vbus_pin = pdata->vbus_pin;
1947 	udc->vbus_pin_inverted = pdata->vbus_pin_inverted;
1948 	udc->num_ep = pdata->num_ep;
1949 
1950 	INIT_LIST_HEAD(&eps[0].ep.ep_list);
1951 
1952 	for (i = 0; i < pdata->num_ep; i++) {
1953 		struct usba_ep *ep = &eps[i];
1954 
1955 		ep->ep_regs = udc->regs + USBA_EPT_BASE(i);
1956 		ep->dma_regs = udc->regs + USBA_DMA_BASE(i);
1957 		ep->fifo = udc->fifo + USBA_FIFO_BASE(i);
1958 		ep->ep.ops = &usba_ep_ops;
1959 		ep->ep.name = pdata->ep[i].name;
1960 		ep->fifo_size = pdata->ep[i].fifo_size;
1961 		usb_ep_set_maxpacket_limit(&ep->ep, ep->fifo_size);
1962 		ep->udc = udc;
1963 		INIT_LIST_HEAD(&ep->queue);
1964 		ep->nr_banks = pdata->ep[i].nr_banks;
1965 		ep->index = pdata->ep[i].index;
1966 		ep->can_dma = pdata->ep[i].can_dma;
1967 		ep->can_isoc = pdata->ep[i].can_isoc;
1968 
1969 		if (i)
1970 			list_add_tail(&ep->ep.ep_list, &udc->gadget.ep_list);
1971 	}
1972 
1973 	return eps;
1974 }
1975 
1976 static int usba_udc_probe(struct platform_device *pdev)
1977 {
1978 	struct resource *regs, *fifo;
1979 	struct clk *pclk, *hclk;
1980 	struct usba_udc *udc;
1981 	int irq, ret, i;
1982 
1983 	udc = devm_kzalloc(&pdev->dev, sizeof(*udc), GFP_KERNEL);
1984 	if (!udc)
1985 		return -ENOMEM;
1986 
1987 	udc->gadget = usba_gadget_template;
1988 	INIT_LIST_HEAD(&udc->gadget.ep_list);
1989 
1990 	regs = platform_get_resource(pdev, IORESOURCE_MEM, CTRL_IOMEM_ID);
1991 	fifo = platform_get_resource(pdev, IORESOURCE_MEM, FIFO_IOMEM_ID);
1992 	if (!regs || !fifo)
1993 		return -ENXIO;
1994 
1995 	irq = platform_get_irq(pdev, 0);
1996 	if (irq < 0)
1997 		return irq;
1998 
1999 	pclk = devm_clk_get(&pdev->dev, "pclk");
2000 	if (IS_ERR(pclk))
2001 		return PTR_ERR(pclk);
2002 	hclk = devm_clk_get(&pdev->dev, "hclk");
2003 	if (IS_ERR(hclk))
2004 		return PTR_ERR(hclk);
2005 
2006 	spin_lock_init(&udc->lock);
2007 	udc->pdev = pdev;
2008 	udc->pclk = pclk;
2009 	udc->hclk = hclk;
2010 	udc->vbus_pin = -ENODEV;
2011 
2012 	ret = -ENOMEM;
2013 	udc->regs = devm_ioremap(&pdev->dev, regs->start, resource_size(regs));
2014 	if (!udc->regs) {
2015 		dev_err(&pdev->dev, "Unable to map I/O memory, aborting.\n");
2016 		return ret;
2017 	}
2018 	dev_info(&pdev->dev, "MMIO registers at 0x%08lx mapped at %p\n",
2019 		 (unsigned long)regs->start, udc->regs);
2020 	udc->fifo = devm_ioremap(&pdev->dev, fifo->start, resource_size(fifo));
2021 	if (!udc->fifo) {
2022 		dev_err(&pdev->dev, "Unable to map FIFO, aborting.\n");
2023 		return ret;
2024 	}
2025 	dev_info(&pdev->dev, "FIFO at 0x%08lx mapped at %p\n",
2026 		 (unsigned long)fifo->start, udc->fifo);
2027 
2028 	platform_set_drvdata(pdev, udc);
2029 
2030 	/* Make sure we start from a clean slate */
2031 	ret = clk_prepare_enable(pclk);
2032 	if (ret) {
2033 		dev_err(&pdev->dev, "Unable to enable pclk, aborting.\n");
2034 		return ret;
2035 	}
2036 	toggle_bias(0);
2037 	usba_writel(udc, CTRL, USBA_DISABLE_MASK);
2038 	clk_disable_unprepare(pclk);
2039 
2040 	if (pdev->dev.of_node)
2041 		udc->usba_ep = atmel_udc_of_init(pdev, udc);
2042 	else
2043 		udc->usba_ep = usba_udc_pdata(pdev, udc);
2044 
2045 	if (IS_ERR(udc->usba_ep))
2046 		return PTR_ERR(udc->usba_ep);
2047 
2048 	ret = devm_request_irq(&pdev->dev, irq, usba_udc_irq, 0,
2049 				"atmel_usba_udc", udc);
2050 	if (ret) {
2051 		dev_err(&pdev->dev, "Cannot request irq %d (error %d)\n",
2052 			irq, ret);
2053 		return ret;
2054 	}
2055 	udc->irq = irq;
2056 
2057 	if (gpio_is_valid(udc->vbus_pin)) {
2058 		if (!devm_gpio_request(&pdev->dev, udc->vbus_pin, "atmel_usba_udc")) {
2059 			ret = devm_request_irq(&pdev->dev,
2060 					gpio_to_irq(udc->vbus_pin),
2061 					usba_vbus_irq, 0,
2062 					"atmel_usba_udc", udc);
2063 			if (ret) {
2064 				udc->vbus_pin = -ENODEV;
2065 				dev_warn(&udc->pdev->dev,
2066 					 "failed to request vbus irq; "
2067 					 "assuming always on\n");
2068 			} else {
2069 				disable_irq(gpio_to_irq(udc->vbus_pin));
2070 			}
2071 		} else {
2072 			/* gpio_request fail so use -EINVAL for gpio_is_valid */
2073 			udc->vbus_pin = -EINVAL;
2074 		}
2075 	}
2076 
2077 	ret = usb_add_gadget_udc(&pdev->dev, &udc->gadget);
2078 	if (ret)
2079 		return ret;
2080 
2081 	usba_init_debugfs(udc);
2082 	for (i = 1; i < udc->num_ep; i++)
2083 		usba_ep_init_debugfs(udc, &udc->usba_ep[i]);
2084 
2085 	return 0;
2086 }
2087 
2088 static int __exit usba_udc_remove(struct platform_device *pdev)
2089 {
2090 	struct usba_udc *udc;
2091 	int i;
2092 
2093 	udc = platform_get_drvdata(pdev);
2094 
2095 	usb_del_gadget_udc(&udc->gadget);
2096 
2097 	for (i = 1; i < udc->num_ep; i++)
2098 		usba_ep_cleanup_debugfs(&udc->usba_ep[i]);
2099 	usba_cleanup_debugfs(udc);
2100 
2101 	return 0;
2102 }
2103 
2104 #if defined(CONFIG_OF)
2105 static const struct of_device_id atmel_udc_dt_ids[] = {
2106 	{ .compatible = "atmel,at91sam9rl-udc" },
2107 	{ /* sentinel */ }
2108 };
2109 
2110 MODULE_DEVICE_TABLE(of, atmel_udc_dt_ids);
2111 #endif
2112 
2113 static struct platform_driver udc_driver = {
2114 	.remove		= __exit_p(usba_udc_remove),
2115 	.driver		= {
2116 		.name		= "atmel_usba_udc",
2117 		.of_match_table	= of_match_ptr(atmel_udc_dt_ids),
2118 	},
2119 };
2120 
2121 module_platform_driver_probe(udc_driver, usba_udc_probe);
2122 
2123 MODULE_DESCRIPTION("Atmel USBA UDC driver");
2124 MODULE_AUTHOR("Haavard Skinnemoen (Atmel)");
2125 MODULE_LICENSE("GPL");
2126 MODULE_ALIAS("platform:atmel_usba_udc");
2127