xref: /openbmc/linux/drivers/usb/gadget/udc/gr_udc.c (revision 6774def6)
1 /*
2  * USB Peripheral Controller driver for Aeroflex Gaisler GRUSBDC.
3  *
4  * 2013 (c) Aeroflex Gaisler AB
5  *
6  * This driver supports GRUSBDC USB Device Controller cores available in the
7  * GRLIB VHDL IP core library.
8  *
9  * Full documentation of the GRUSBDC core can be found here:
10  * http://www.gaisler.com/products/grlib/grip.pdf
11  *
12  * This program is free software; you can redistribute it and/or modify it
13  * under the terms of the GNU General Public License as published by the
14  * Free Software Foundation; either version 2 of the License, or (at your
15  * option) any later version.
16  *
17  * Contributors:
18  * - Andreas Larsson <andreas@gaisler.com>
19  * - Marko Isomaki
20  */
21 
22 /*
23  * A GRUSBDC core can have up to 16 IN endpoints and 16 OUT endpoints each
24  * individually configurable to any of the four USB transfer types. This driver
25  * only supports cores in DMA mode.
26  */
27 
28 #include <linux/kernel.h>
29 #include <linux/module.h>
30 #include <linux/slab.h>
31 #include <linux/spinlock.h>
32 #include <linux/errno.h>
33 #include <linux/list.h>
34 #include <linux/interrupt.h>
35 #include <linux/device.h>
36 #include <linux/usb/ch9.h>
37 #include <linux/usb/gadget.h>
38 #include <linux/dma-mapping.h>
39 #include <linux/dmapool.h>
40 #include <linux/debugfs.h>
41 #include <linux/seq_file.h>
42 #include <linux/of_platform.h>
43 #include <linux/of_irq.h>
44 #include <linux/of_address.h>
45 
46 #include <asm/byteorder.h>
47 
48 #include "gr_udc.h"
49 
50 #define	DRIVER_NAME	"gr_udc"
51 #define	DRIVER_DESC	"Aeroflex Gaisler GRUSBDC USB Peripheral Controller"
52 
53 static const char driver_name[] = DRIVER_NAME;
54 static const char driver_desc[] = DRIVER_DESC;
55 
56 #define gr_read32(x) (ioread32be((x)))
57 #define gr_write32(x, v) (iowrite32be((v), (x)))
58 
59 /* USB speed and corresponding string calculated from status register value */
60 #define GR_SPEED(status) \
61 	((status & GR_STATUS_SP) ? USB_SPEED_FULL : USB_SPEED_HIGH)
62 #define GR_SPEED_STR(status) usb_speed_string(GR_SPEED(status))
63 
64 /* Size of hardware buffer calculated from epctrl register value */
65 #define GR_BUFFER_SIZE(epctrl)					      \
66 	((((epctrl) & GR_EPCTRL_BUFSZ_MASK) >> GR_EPCTRL_BUFSZ_POS) * \
67 	 GR_EPCTRL_BUFSZ_SCALER)
68 
69 /* ---------------------------------------------------------------------- */
70 /* Debug printout functionality */
71 
72 static const char * const gr_modestring[] = {"control", "iso", "bulk", "int"};
73 
74 static const char *gr_ep0state_string(enum gr_ep0state state)
75 {
76 	static const char *const names[] = {
77 		[GR_EP0_DISCONNECT] = "disconnect",
78 		[GR_EP0_SETUP] = "setup",
79 		[GR_EP0_IDATA] = "idata",
80 		[GR_EP0_ODATA] = "odata",
81 		[GR_EP0_ISTATUS] = "istatus",
82 		[GR_EP0_OSTATUS] = "ostatus",
83 		[GR_EP0_STALL] = "stall",
84 		[GR_EP0_SUSPEND] = "suspend",
85 	};
86 
87 	if (state < 0 || state >= ARRAY_SIZE(names))
88 		return "UNKNOWN";
89 
90 	return names[state];
91 }
92 
93 #ifdef VERBOSE_DEBUG
94 
95 static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
96 				struct gr_request *req)
97 {
98 	int buflen = ep->is_in ? req->req.length : req->req.actual;
99 	int rowlen = 32;
100 	int plen = min(rowlen, buflen);
101 
102 	dev_dbg(ep->dev->dev, "%s: 0x%p, %d bytes data%s:\n", str, req, buflen,
103 		(buflen > plen ? " (truncated)" : ""));
104 	print_hex_dump_debug("   ", DUMP_PREFIX_NONE,
105 			     rowlen, 4, req->req.buf, plen, false);
106 }
107 
108 static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
109 			       u16 value, u16 index, u16 length)
110 {
111 	dev_vdbg(dev->dev, "REQ: %02x.%02x v%04x i%04x l%04x\n",
112 		 type, request, value, index, length);
113 }
114 #else /* !VERBOSE_DEBUG */
115 
116 static void gr_dbgprint_request(const char *str, struct gr_ep *ep,
117 				struct gr_request *req) {}
118 
119 static void gr_dbgprint_devreq(struct gr_udc *dev, u8 type, u8 request,
120 			       u16 value, u16 index, u16 length) {}
121 
122 #endif /* VERBOSE_DEBUG */
123 
124 /* ---------------------------------------------------------------------- */
125 /* Debugfs functionality */
126 
127 #ifdef CONFIG_USB_GADGET_DEBUG_FS
128 
129 static void gr_seq_ep_show(struct seq_file *seq, struct gr_ep *ep)
130 {
131 	u32 epctrl = gr_read32(&ep->regs->epctrl);
132 	u32 epstat = gr_read32(&ep->regs->epstat);
133 	int mode = (epctrl & GR_EPCTRL_TT_MASK) >> GR_EPCTRL_TT_POS;
134 	struct gr_request *req;
135 
136 	seq_printf(seq, "%s:\n", ep->ep.name);
137 	seq_printf(seq, "  mode = %s\n", gr_modestring[mode]);
138 	seq_printf(seq, "  halted: %d\n", !!(epctrl & GR_EPCTRL_EH));
139 	seq_printf(seq, "  disabled: %d\n", !!(epctrl & GR_EPCTRL_ED));
140 	seq_printf(seq, "  valid: %d\n", !!(epctrl & GR_EPCTRL_EV));
141 	seq_printf(seq, "  dma_start = %d\n", ep->dma_start);
142 	seq_printf(seq, "  stopped = %d\n", ep->stopped);
143 	seq_printf(seq, "  wedged = %d\n", ep->wedged);
144 	seq_printf(seq, "  callback = %d\n", ep->callback);
145 	seq_printf(seq, "  maxpacket = %d\n", ep->ep.maxpacket);
146 	seq_printf(seq, "  maxpacket_limit = %d\n", ep->ep.maxpacket_limit);
147 	seq_printf(seq, "  bytes_per_buffer = %d\n", ep->bytes_per_buffer);
148 	if (mode == 1 || mode == 3)
149 		seq_printf(seq, "  nt = %d\n",
150 			   (epctrl & GR_EPCTRL_NT_MASK) >> GR_EPCTRL_NT_POS);
151 
152 	seq_printf(seq, "  Buffer 0: %s %s%d\n",
153 		   epstat & GR_EPSTAT_B0 ? "valid" : "invalid",
154 		   epstat & GR_EPSTAT_BS ? " " : "selected ",
155 		   (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS);
156 	seq_printf(seq, "  Buffer 1: %s %s%d\n",
157 		   epstat & GR_EPSTAT_B1 ? "valid" : "invalid",
158 		   epstat & GR_EPSTAT_BS ? "selected " : " ",
159 		   (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS);
160 
161 	if (list_empty(&ep->queue)) {
162 		seq_puts(seq, "  Queue: empty\n\n");
163 		return;
164 	}
165 
166 	seq_puts(seq, "  Queue:\n");
167 	list_for_each_entry(req, &ep->queue, queue) {
168 		struct gr_dma_desc *desc;
169 		struct gr_dma_desc *next;
170 
171 		seq_printf(seq, "    0x%p: 0x%p %d %d\n", req,
172 			   &req->req.buf, req->req.actual, req->req.length);
173 
174 		next = req->first_desc;
175 		do {
176 			desc = next;
177 			next = desc->next_desc;
178 			seq_printf(seq, "    %c 0x%p (0x%08x): 0x%05x 0x%08x\n",
179 				   desc == req->curr_desc ? 'c' : ' ',
180 				   desc, desc->paddr, desc->ctrl, desc->data);
181 		} while (desc != req->last_desc);
182 	}
183 	seq_puts(seq, "\n");
184 }
185 
186 
187 static int gr_seq_show(struct seq_file *seq, void *v)
188 {
189 	struct gr_udc *dev = seq->private;
190 	u32 control = gr_read32(&dev->regs->control);
191 	u32 status = gr_read32(&dev->regs->status);
192 	struct gr_ep *ep;
193 
194 	seq_printf(seq, "usb state = %s\n",
195 		   usb_state_string(dev->gadget.state));
196 	seq_printf(seq, "address = %d\n",
197 		   (control & GR_CONTROL_UA_MASK) >> GR_CONTROL_UA_POS);
198 	seq_printf(seq, "speed = %s\n", GR_SPEED_STR(status));
199 	seq_printf(seq, "ep0state = %s\n", gr_ep0state_string(dev->ep0state));
200 	seq_printf(seq, "irq_enabled = %d\n", dev->irq_enabled);
201 	seq_printf(seq, "remote_wakeup = %d\n", dev->remote_wakeup);
202 	seq_printf(seq, "test_mode = %d\n", dev->test_mode);
203 	seq_puts(seq, "\n");
204 
205 	list_for_each_entry(ep, &dev->ep_list, ep_list)
206 		gr_seq_ep_show(seq, ep);
207 
208 	return 0;
209 }
210 
211 static int gr_dfs_open(struct inode *inode, struct file *file)
212 {
213 	return single_open(file, gr_seq_show, inode->i_private);
214 }
215 
216 static const struct file_operations gr_dfs_fops = {
217 	.owner		= THIS_MODULE,
218 	.open		= gr_dfs_open,
219 	.read		= seq_read,
220 	.llseek		= seq_lseek,
221 	.release	= single_release,
222 };
223 
224 static void gr_dfs_create(struct gr_udc *dev)
225 {
226 	const char *name = "gr_udc_state";
227 
228 	dev->dfs_root = debugfs_create_dir(dev_name(dev->dev), NULL);
229 	dev->dfs_state = debugfs_create_file(name, 0444, dev->dfs_root, dev,
230 					     &gr_dfs_fops);
231 }
232 
233 static void gr_dfs_delete(struct gr_udc *dev)
234 {
235 	/* Handles NULL and ERR pointers internally */
236 	debugfs_remove(dev->dfs_state);
237 	debugfs_remove(dev->dfs_root);
238 }
239 
240 #else /* !CONFIG_USB_GADGET_DEBUG_FS */
241 
242 static void gr_dfs_create(struct gr_udc *dev) {}
243 static void gr_dfs_delete(struct gr_udc *dev) {}
244 
245 #endif /* CONFIG_USB_GADGET_DEBUG_FS */
246 
247 /* ---------------------------------------------------------------------- */
248 /* DMA and request handling */
249 
250 /* Allocates a new struct gr_dma_desc, sets paddr and zeroes the rest */
251 static struct gr_dma_desc *gr_alloc_dma_desc(struct gr_ep *ep, gfp_t gfp_flags)
252 {
253 	dma_addr_t paddr;
254 	struct gr_dma_desc *dma_desc;
255 
256 	dma_desc = dma_pool_alloc(ep->dev->desc_pool, gfp_flags, &paddr);
257 	if (!dma_desc) {
258 		dev_err(ep->dev->dev, "Could not allocate from DMA pool\n");
259 		return NULL;
260 	}
261 
262 	memset(dma_desc, 0, sizeof(*dma_desc));
263 	dma_desc->paddr = paddr;
264 
265 	return dma_desc;
266 }
267 
268 static inline void gr_free_dma_desc(struct gr_udc *dev,
269 				    struct gr_dma_desc *desc)
270 {
271 	dma_pool_free(dev->desc_pool, desc, (dma_addr_t)desc->paddr);
272 }
273 
274 /* Frees the chain of struct gr_dma_desc for the given request */
275 static void gr_free_dma_desc_chain(struct gr_udc *dev, struct gr_request *req)
276 {
277 	struct gr_dma_desc *desc;
278 	struct gr_dma_desc *next;
279 
280 	next = req->first_desc;
281 	if (!next)
282 		return;
283 
284 	do {
285 		desc = next;
286 		next = desc->next_desc;
287 		gr_free_dma_desc(dev, desc);
288 	} while (desc != req->last_desc);
289 
290 	req->first_desc = NULL;
291 	req->curr_desc = NULL;
292 	req->last_desc = NULL;
293 }
294 
295 static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req);
296 
297 /*
298  * Frees allocated resources and calls the appropriate completion function/setup
299  * package handler for a finished request.
300  *
301  * Must be called with dev->lock held and irqs disabled.
302  */
303 static void gr_finish_request(struct gr_ep *ep, struct gr_request *req,
304 			      int status)
305 	__releases(&dev->lock)
306 	__acquires(&dev->lock)
307 {
308 	struct gr_udc *dev;
309 
310 	list_del_init(&req->queue);
311 
312 	if (likely(req->req.status == -EINPROGRESS))
313 		req->req.status = status;
314 	else
315 		status = req->req.status;
316 
317 	dev = ep->dev;
318 	usb_gadget_unmap_request(&dev->gadget, &req->req, ep->is_in);
319 	gr_free_dma_desc_chain(dev, req);
320 
321 	if (ep->is_in) { /* For OUT, req->req.actual gets updated bit by bit */
322 		req->req.actual = req->req.length;
323 	} else if (req->oddlen && req->req.actual > req->evenlen) {
324 		/*
325 		 * Copy to user buffer in this case where length was not evenly
326 		 * divisible by ep->ep.maxpacket and the last descriptor was
327 		 * actually used.
328 		 */
329 		char *buftail = ((char *)req->req.buf + req->evenlen);
330 
331 		memcpy(buftail, ep->tailbuf, req->oddlen);
332 
333 		if (req->req.actual > req->req.length) {
334 			/* We got more data than was requested */
335 			dev_dbg(ep->dev->dev, "Overflow for ep %s\n",
336 				ep->ep.name);
337 			gr_dbgprint_request("OVFL", ep, req);
338 			req->req.status = -EOVERFLOW;
339 		}
340 	}
341 
342 	if (!status) {
343 		if (ep->is_in)
344 			gr_dbgprint_request("SENT", ep, req);
345 		else
346 			gr_dbgprint_request("RECV", ep, req);
347 	}
348 
349 	/* Prevent changes to ep->queue during callback */
350 	ep->callback = 1;
351 	if (req == dev->ep0reqo && !status) {
352 		if (req->setup)
353 			gr_ep0_setup(dev, req);
354 		else
355 			dev_err(dev->dev,
356 				"Unexpected non setup packet on ep0in\n");
357 	} else if (req->req.complete) {
358 		spin_unlock(&dev->lock);
359 
360 		usb_gadget_giveback_request(&ep->ep, &req->req);
361 
362 		spin_lock(&dev->lock);
363 	}
364 	ep->callback = 0;
365 }
366 
367 static struct usb_request *gr_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
368 {
369 	struct gr_request *req;
370 
371 	req = kzalloc(sizeof(*req), gfp_flags);
372 	if (!req)
373 		return NULL;
374 
375 	INIT_LIST_HEAD(&req->queue);
376 
377 	return &req->req;
378 }
379 
380 /*
381  * Starts DMA for endpoint ep if there are requests in the queue.
382  *
383  * Must be called with dev->lock held and with !ep->stopped.
384  */
385 static void gr_start_dma(struct gr_ep *ep)
386 {
387 	struct gr_request *req;
388 	u32 dmactrl;
389 
390 	if (list_empty(&ep->queue)) {
391 		ep->dma_start = 0;
392 		return;
393 	}
394 
395 	req = list_first_entry(&ep->queue, struct gr_request, queue);
396 
397 	/* A descriptor should already have been allocated */
398 	BUG_ON(!req->curr_desc);
399 
400 	/*
401 	 * The DMA controller can not handle smaller OUT buffers than
402 	 * ep->ep.maxpacket. It could lead to buffer overruns if an unexpectedly
403 	 * long packet are received. Therefore an internal bounce buffer gets
404 	 * used when such a request gets enabled.
405 	 */
406 	if (!ep->is_in && req->oddlen)
407 		req->last_desc->data = ep->tailbuf_paddr;
408 
409 	wmb(); /* Make sure all is settled before handing it over to DMA */
410 
411 	/* Set the descriptor pointer in the hardware */
412 	gr_write32(&ep->regs->dmaaddr, req->curr_desc->paddr);
413 
414 	/* Announce available descriptors */
415 	dmactrl = gr_read32(&ep->regs->dmactrl);
416 	gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_DA);
417 
418 	ep->dma_start = 1;
419 }
420 
421 /*
422  * Finishes the first request in the ep's queue and, if available, starts the
423  * next request in queue.
424  *
425  * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
426  */
427 static void gr_dma_advance(struct gr_ep *ep, int status)
428 {
429 	struct gr_request *req;
430 
431 	req = list_first_entry(&ep->queue, struct gr_request, queue);
432 	gr_finish_request(ep, req, status);
433 	gr_start_dma(ep); /* Regardless of ep->dma_start */
434 }
435 
436 /*
437  * Abort DMA for an endpoint. Sets the abort DMA bit which causes an ongoing DMA
438  * transfer to be canceled and clears GR_DMACTRL_DA.
439  *
440  * Must be called with dev->lock held.
441  */
442 static void gr_abort_dma(struct gr_ep *ep)
443 {
444 	u32 dmactrl;
445 
446 	dmactrl = gr_read32(&ep->regs->dmactrl);
447 	gr_write32(&ep->regs->dmactrl, dmactrl | GR_DMACTRL_AD);
448 }
449 
450 /*
451  * Allocates and sets up a struct gr_dma_desc and putting it on the descriptor
452  * chain.
453  *
454  * Size is not used for OUT endpoints. Hardware can not be instructed to handle
455  * smaller buffer than MAXPL in the OUT direction.
456  */
457 static int gr_add_dma_desc(struct gr_ep *ep, struct gr_request *req,
458 			   dma_addr_t data, unsigned size, gfp_t gfp_flags)
459 {
460 	struct gr_dma_desc *desc;
461 
462 	desc = gr_alloc_dma_desc(ep, gfp_flags);
463 	if (!desc)
464 		return -ENOMEM;
465 
466 	desc->data = data;
467 	if (ep->is_in)
468 		desc->ctrl =
469 			(GR_DESC_IN_CTRL_LEN_MASK & size) | GR_DESC_IN_CTRL_EN;
470 	else
471 		desc->ctrl = GR_DESC_OUT_CTRL_IE;
472 
473 	if (!req->first_desc) {
474 		req->first_desc = desc;
475 		req->curr_desc = desc;
476 	} else {
477 		req->last_desc->next_desc = desc;
478 		req->last_desc->next = desc->paddr;
479 		req->last_desc->ctrl |= GR_DESC_OUT_CTRL_NX;
480 	}
481 	req->last_desc = desc;
482 
483 	return 0;
484 }
485 
486 /*
487  * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
488  * together covers req->req.length bytes of the buffer at DMA address
489  * req->req.dma for the OUT direction.
490  *
491  * The first descriptor in the chain is enabled, the rest disabled. The
492  * interrupt handler will later enable them one by one when needed so we can
493  * find out when the transfer is finished. For OUT endpoints, all descriptors
494  * therefore generate interrutps.
495  */
496 static int gr_setup_out_desc_list(struct gr_ep *ep, struct gr_request *req,
497 				  gfp_t gfp_flags)
498 {
499 	u16 bytes_left; /* Bytes left to provide descriptors for */
500 	u16 bytes_used; /* Bytes accommodated for */
501 	int ret = 0;
502 
503 	req->first_desc = NULL; /* Signals that no allocation is done yet */
504 	bytes_left = req->req.length;
505 	bytes_used = 0;
506 	while (bytes_left > 0) {
507 		dma_addr_t start = req->req.dma + bytes_used;
508 		u16 size = min(bytes_left, ep->bytes_per_buffer);
509 
510 		if (size < ep->bytes_per_buffer) {
511 			/* Prepare using bounce buffer */
512 			req->evenlen = req->req.length - bytes_left;
513 			req->oddlen = size;
514 		}
515 
516 		ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
517 		if (ret)
518 			goto alloc_err;
519 
520 		bytes_left -= size;
521 		bytes_used += size;
522 	}
523 
524 	req->first_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
525 
526 	return 0;
527 
528 alloc_err:
529 	gr_free_dma_desc_chain(ep->dev, req);
530 
531 	return ret;
532 }
533 
534 /*
535  * Sets up a chain of struct gr_dma_descriptors pointing to buffers that
536  * together covers req->req.length bytes of the buffer at DMA address
537  * req->req.dma for the IN direction.
538  *
539  * When more data is provided than the maximum payload size, the hardware splits
540  * this up into several payloads automatically. Moreover, ep->bytes_per_buffer
541  * is always set to a multiple of the maximum payload (restricted to the valid
542  * number of maximum payloads during high bandwidth isochronous or interrupt
543  * transfers)
544  *
545  * All descriptors are enabled from the beginning and we only generate an
546  * interrupt for the last one indicating that the entire request has been pushed
547  * to hardware.
548  */
549 static int gr_setup_in_desc_list(struct gr_ep *ep, struct gr_request *req,
550 				 gfp_t gfp_flags)
551 {
552 	u16 bytes_left; /* Bytes left in req to provide descriptors for */
553 	u16 bytes_used; /* Bytes in req accommodated for */
554 	int ret = 0;
555 
556 	req->first_desc = NULL; /* Signals that no allocation is done yet */
557 	bytes_left = req->req.length;
558 	bytes_used = 0;
559 	do { /* Allow for zero length packets */
560 		dma_addr_t start = req->req.dma + bytes_used;
561 		u16 size = min(bytes_left, ep->bytes_per_buffer);
562 
563 		ret = gr_add_dma_desc(ep, req, start, size, gfp_flags);
564 		if (ret)
565 			goto alloc_err;
566 
567 		bytes_left -= size;
568 		bytes_used += size;
569 	} while (bytes_left > 0);
570 
571 	/*
572 	 * Send an extra zero length packet to indicate that no more data is
573 	 * available when req->req.zero is set and the data length is even
574 	 * multiples of ep->ep.maxpacket.
575 	 */
576 	if (req->req.zero && (req->req.length % ep->ep.maxpacket == 0)) {
577 		ret = gr_add_dma_desc(ep, req, 0, 0, gfp_flags);
578 		if (ret)
579 			goto alloc_err;
580 	}
581 
582 	/*
583 	 * For IN packets we only want to know when the last packet has been
584 	 * transmitted (not just put into internal buffers).
585 	 */
586 	req->last_desc->ctrl |= GR_DESC_IN_CTRL_PI;
587 
588 	return 0;
589 
590 alloc_err:
591 	gr_free_dma_desc_chain(ep->dev, req);
592 
593 	return ret;
594 }
595 
596 /* Must be called with dev->lock held */
597 static int gr_queue(struct gr_ep *ep, struct gr_request *req, gfp_t gfp_flags)
598 {
599 	struct gr_udc *dev = ep->dev;
600 	int ret;
601 
602 	if (unlikely(!ep->ep.desc && ep->num != 0)) {
603 		dev_err(dev->dev, "No ep descriptor for %s\n", ep->ep.name);
604 		return -EINVAL;
605 	}
606 
607 	if (unlikely(!req->req.buf || !list_empty(&req->queue))) {
608 		dev_err(dev->dev,
609 			"Invalid request for %s: buf=%p list_empty=%d\n",
610 			ep->ep.name, req->req.buf, list_empty(&req->queue));
611 		return -EINVAL;
612 	}
613 
614 	if (unlikely(!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)) {
615 		dev_err(dev->dev, "-ESHUTDOWN");
616 		return -ESHUTDOWN;
617 	}
618 
619 	/* Can't touch registers when suspended */
620 	if (dev->ep0state == GR_EP0_SUSPEND) {
621 		dev_err(dev->dev, "-EBUSY");
622 		return -EBUSY;
623 	}
624 
625 	/* Set up DMA mapping in case the caller didn't */
626 	ret = usb_gadget_map_request(&dev->gadget, &req->req, ep->is_in);
627 	if (ret) {
628 		dev_err(dev->dev, "usb_gadget_map_request");
629 		return ret;
630 	}
631 
632 	if (ep->is_in)
633 		ret = gr_setup_in_desc_list(ep, req, gfp_flags);
634 	else
635 		ret = gr_setup_out_desc_list(ep, req, gfp_flags);
636 	if (ret)
637 		return ret;
638 
639 	req->req.status = -EINPROGRESS;
640 	req->req.actual = 0;
641 	list_add_tail(&req->queue, &ep->queue);
642 
643 	/* Start DMA if not started, otherwise interrupt handler handles it */
644 	if (!ep->dma_start && likely(!ep->stopped))
645 		gr_start_dma(ep);
646 
647 	return 0;
648 }
649 
650 /*
651  * Queue a request from within the driver.
652  *
653  * Must be called with dev->lock held.
654  */
655 static inline int gr_queue_int(struct gr_ep *ep, struct gr_request *req,
656 			       gfp_t gfp_flags)
657 {
658 	if (ep->is_in)
659 		gr_dbgprint_request("RESP", ep, req);
660 
661 	return gr_queue(ep, req, gfp_flags);
662 }
663 
664 /* ---------------------------------------------------------------------- */
665 /* General helper functions */
666 
667 /*
668  * Dequeue ALL requests.
669  *
670  * Must be called with dev->lock held and irqs disabled.
671  */
672 static void gr_ep_nuke(struct gr_ep *ep)
673 {
674 	struct gr_request *req;
675 
676 	ep->stopped = 1;
677 	ep->dma_start = 0;
678 	gr_abort_dma(ep);
679 
680 	while (!list_empty(&ep->queue)) {
681 		req = list_first_entry(&ep->queue, struct gr_request, queue);
682 		gr_finish_request(ep, req, -ESHUTDOWN);
683 	}
684 }
685 
686 /*
687  * Reset the hardware state of this endpoint.
688  *
689  * Must be called with dev->lock held.
690  */
691 static void gr_ep_reset(struct gr_ep *ep)
692 {
693 	gr_write32(&ep->regs->epctrl, 0);
694 	gr_write32(&ep->regs->dmactrl, 0);
695 
696 	ep->ep.maxpacket = MAX_CTRL_PL_SIZE;
697 	ep->ep.desc = NULL;
698 	ep->stopped = 1;
699 	ep->dma_start = 0;
700 }
701 
702 /*
703  * Generate STALL on ep0in/out.
704  *
705  * Must be called with dev->lock held.
706  */
707 static void gr_control_stall(struct gr_udc *dev)
708 {
709 	u32 epctrl;
710 
711 	epctrl = gr_read32(&dev->epo[0].regs->epctrl);
712 	gr_write32(&dev->epo[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
713 	epctrl = gr_read32(&dev->epi[0].regs->epctrl);
714 	gr_write32(&dev->epi[0].regs->epctrl, epctrl | GR_EPCTRL_CS);
715 
716 	dev->ep0state = GR_EP0_STALL;
717 }
718 
719 /*
720  * Halts, halts and wedges, or clears halt for an endpoint.
721  *
722  * Must be called with dev->lock held.
723  */
724 static int gr_ep_halt_wedge(struct gr_ep *ep, int halt, int wedge, int fromhost)
725 {
726 	u32 epctrl;
727 	int retval = 0;
728 
729 	if (ep->num && !ep->ep.desc)
730 		return -EINVAL;
731 
732 	if (ep->num && ep->ep.desc->bmAttributes == USB_ENDPOINT_XFER_ISOC)
733 		return -EOPNOTSUPP;
734 
735 	/* Never actually halt ep0, and therefore never clear halt for ep0 */
736 	if (!ep->num) {
737 		if (halt && !fromhost) {
738 			/* ep0 halt from gadget - generate protocol stall */
739 			gr_control_stall(ep->dev);
740 			dev_dbg(ep->dev->dev, "EP: stall ep0\n");
741 			return 0;
742 		}
743 		return -EINVAL;
744 	}
745 
746 	dev_dbg(ep->dev->dev, "EP: %s halt %s\n",
747 		(halt ? (wedge ? "wedge" : "set") : "clear"), ep->ep.name);
748 
749 	epctrl = gr_read32(&ep->regs->epctrl);
750 	if (halt) {
751 		/* Set HALT */
752 		gr_write32(&ep->regs->epctrl, epctrl | GR_EPCTRL_EH);
753 		ep->stopped = 1;
754 		if (wedge)
755 			ep->wedged = 1;
756 	} else {
757 		gr_write32(&ep->regs->epctrl, epctrl & ~GR_EPCTRL_EH);
758 		ep->stopped = 0;
759 		ep->wedged = 0;
760 
761 		/* Things might have been queued up in the meantime */
762 		if (!ep->dma_start)
763 			gr_start_dma(ep);
764 	}
765 
766 	return retval;
767 }
768 
769 /* Must be called with dev->lock held */
770 static inline void gr_set_ep0state(struct gr_udc *dev, enum gr_ep0state value)
771 {
772 	if (dev->ep0state != value)
773 		dev_vdbg(dev->dev, "STATE:  ep0state=%s\n",
774 			 gr_ep0state_string(value));
775 	dev->ep0state = value;
776 }
777 
778 /*
779  * Should only be called when endpoints can not generate interrupts.
780  *
781  * Must be called with dev->lock held.
782  */
783 static void gr_disable_interrupts_and_pullup(struct gr_udc *dev)
784 {
785 	gr_write32(&dev->regs->control, 0);
786 	wmb(); /* Make sure that we do not deny one of our interrupts */
787 	dev->irq_enabled = 0;
788 }
789 
790 /*
791  * Stop all device activity and disable data line pullup.
792  *
793  * Must be called with dev->lock held and irqs disabled.
794  */
795 static void gr_stop_activity(struct gr_udc *dev)
796 {
797 	struct gr_ep *ep;
798 
799 	list_for_each_entry(ep, &dev->ep_list, ep_list)
800 		gr_ep_nuke(ep);
801 
802 	gr_disable_interrupts_and_pullup(dev);
803 
804 	gr_set_ep0state(dev, GR_EP0_DISCONNECT);
805 	usb_gadget_set_state(&dev->gadget, USB_STATE_NOTATTACHED);
806 }
807 
808 /* ---------------------------------------------------------------------- */
809 /* ep0 setup packet handling */
810 
811 static void gr_ep0_testmode_complete(struct usb_ep *_ep,
812 				     struct usb_request *_req)
813 {
814 	struct gr_ep *ep;
815 	struct gr_udc *dev;
816 	u32 control;
817 
818 	ep = container_of(_ep, struct gr_ep, ep);
819 	dev = ep->dev;
820 
821 	spin_lock(&dev->lock);
822 
823 	control = gr_read32(&dev->regs->control);
824 	control |= GR_CONTROL_TM | (dev->test_mode << GR_CONTROL_TS_POS);
825 	gr_write32(&dev->regs->control, control);
826 
827 	spin_unlock(&dev->lock);
828 }
829 
830 static void gr_ep0_dummy_complete(struct usb_ep *_ep, struct usb_request *_req)
831 {
832 	/* Nothing needs to be done here */
833 }
834 
835 /*
836  * Queue a response on ep0in.
837  *
838  * Must be called with dev->lock held.
839  */
840 static int gr_ep0_respond(struct gr_udc *dev, u8 *buf, int length,
841 			  void (*complete)(struct usb_ep *ep,
842 					   struct usb_request *req))
843 {
844 	u8 *reqbuf = dev->ep0reqi->req.buf;
845 	int status;
846 	int i;
847 
848 	for (i = 0; i < length; i++)
849 		reqbuf[i] = buf[i];
850 	dev->ep0reqi->req.length = length;
851 	dev->ep0reqi->req.complete = complete;
852 
853 	status = gr_queue_int(&dev->epi[0], dev->ep0reqi, GFP_ATOMIC);
854 	if (status < 0)
855 		dev_err(dev->dev,
856 			"Could not queue ep0in setup response: %d\n", status);
857 
858 	return status;
859 }
860 
861 /*
862  * Queue a 2 byte response on ep0in.
863  *
864  * Must be called with dev->lock held.
865  */
866 static inline int gr_ep0_respond_u16(struct gr_udc *dev, u16 response)
867 {
868 	__le16 le_response = cpu_to_le16(response);
869 
870 	return gr_ep0_respond(dev, (u8 *)&le_response, 2,
871 			      gr_ep0_dummy_complete);
872 }
873 
874 /*
875  * Queue a ZLP response on ep0in.
876  *
877  * Must be called with dev->lock held.
878  */
879 static inline int gr_ep0_respond_empty(struct gr_udc *dev)
880 {
881 	return gr_ep0_respond(dev, NULL, 0, gr_ep0_dummy_complete);
882 }
883 
884 /*
885  * This is run when a SET_ADDRESS request is received. First writes
886  * the new address to the control register which is updated internally
887  * when the next IN packet is ACKED.
888  *
889  * Must be called with dev->lock held.
890  */
891 static void gr_set_address(struct gr_udc *dev, u8 address)
892 {
893 	u32 control;
894 
895 	control = gr_read32(&dev->regs->control) & ~GR_CONTROL_UA_MASK;
896 	control |= (address << GR_CONTROL_UA_POS) & GR_CONTROL_UA_MASK;
897 	control |= GR_CONTROL_SU;
898 	gr_write32(&dev->regs->control, control);
899 }
900 
901 /*
902  * Returns negative for STALL, 0 for successful handling and positive for
903  * delegation.
904  *
905  * Must be called with dev->lock held.
906  */
907 static int gr_device_request(struct gr_udc *dev, u8 type, u8 request,
908 			     u16 value, u16 index)
909 {
910 	u16 response;
911 	u8 test;
912 
913 	switch (request) {
914 	case USB_REQ_SET_ADDRESS:
915 		dev_dbg(dev->dev, "STATUS: address %d\n", value & 0xff);
916 		gr_set_address(dev, value & 0xff);
917 		if (value)
918 			usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
919 		else
920 			usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
921 		return gr_ep0_respond_empty(dev);
922 
923 	case USB_REQ_GET_STATUS:
924 		/* Self powered | remote wakeup */
925 		response = 0x0001 | (dev->remote_wakeup ? 0x0002 : 0);
926 		return gr_ep0_respond_u16(dev, response);
927 
928 	case USB_REQ_SET_FEATURE:
929 		switch (value) {
930 		case USB_DEVICE_REMOTE_WAKEUP:
931 			/* Allow remote wakeup */
932 			dev->remote_wakeup = 1;
933 			return gr_ep0_respond_empty(dev);
934 
935 		case USB_DEVICE_TEST_MODE:
936 			/* The hardware does not support TEST_FORCE_EN */
937 			test = index >> 8;
938 			if (test >= TEST_J && test <= TEST_PACKET) {
939 				dev->test_mode = test;
940 				return gr_ep0_respond(dev, NULL, 0,
941 						      gr_ep0_testmode_complete);
942 			}
943 		}
944 		break;
945 
946 	case USB_REQ_CLEAR_FEATURE:
947 		switch (value) {
948 		case USB_DEVICE_REMOTE_WAKEUP:
949 			/* Disallow remote wakeup */
950 			dev->remote_wakeup = 0;
951 			return gr_ep0_respond_empty(dev);
952 		}
953 		break;
954 	}
955 
956 	return 1; /* Delegate the rest */
957 }
958 
959 /*
960  * Returns negative for STALL, 0 for successful handling and positive for
961  * delegation.
962  *
963  * Must be called with dev->lock held.
964  */
965 static int gr_interface_request(struct gr_udc *dev, u8 type, u8 request,
966 				u16 value, u16 index)
967 {
968 	if (dev->gadget.state != USB_STATE_CONFIGURED)
969 		return -1;
970 
971 	/*
972 	 * Should return STALL for invalid interfaces, but udc driver does not
973 	 * know anything about that. However, many gadget drivers do not handle
974 	 * GET_STATUS so we need to take care of that.
975 	 */
976 
977 	switch (request) {
978 	case USB_REQ_GET_STATUS:
979 		return gr_ep0_respond_u16(dev, 0x0000);
980 
981 	case USB_REQ_SET_FEATURE:
982 	case USB_REQ_CLEAR_FEATURE:
983 		/*
984 		 * No possible valid standard requests. Still let gadget drivers
985 		 * have a go at it.
986 		 */
987 		break;
988 	}
989 
990 	return 1; /* Delegate the rest */
991 }
992 
993 /*
994  * Returns negative for STALL, 0 for successful handling and positive for
995  * delegation.
996  *
997  * Must be called with dev->lock held.
998  */
999 static int gr_endpoint_request(struct gr_udc *dev, u8 type, u8 request,
1000 			       u16 value, u16 index)
1001 {
1002 	struct gr_ep *ep;
1003 	int status;
1004 	int halted;
1005 	u8 epnum = index & USB_ENDPOINT_NUMBER_MASK;
1006 	u8 is_in = index & USB_ENDPOINT_DIR_MASK;
1007 
1008 	if ((is_in && epnum >= dev->nepi) || (!is_in && epnum >= dev->nepo))
1009 		return -1;
1010 
1011 	if (dev->gadget.state != USB_STATE_CONFIGURED && epnum != 0)
1012 		return -1;
1013 
1014 	ep = (is_in ? &dev->epi[epnum] : &dev->epo[epnum]);
1015 
1016 	switch (request) {
1017 	case USB_REQ_GET_STATUS:
1018 		halted = gr_read32(&ep->regs->epctrl) & GR_EPCTRL_EH;
1019 		return gr_ep0_respond_u16(dev, halted ? 0x0001 : 0);
1020 
1021 	case USB_REQ_SET_FEATURE:
1022 		switch (value) {
1023 		case USB_ENDPOINT_HALT:
1024 			status = gr_ep_halt_wedge(ep, 1, 0, 1);
1025 			if (status >= 0)
1026 				status = gr_ep0_respond_empty(dev);
1027 			return status;
1028 		}
1029 		break;
1030 
1031 	case USB_REQ_CLEAR_FEATURE:
1032 		switch (value) {
1033 		case USB_ENDPOINT_HALT:
1034 			if (ep->wedged)
1035 				return -1;
1036 			status = gr_ep_halt_wedge(ep, 0, 0, 1);
1037 			if (status >= 0)
1038 				status = gr_ep0_respond_empty(dev);
1039 			return status;
1040 		}
1041 		break;
1042 	}
1043 
1044 	return 1; /* Delegate the rest */
1045 }
1046 
1047 /* Must be called with dev->lock held */
1048 static void gr_ep0out_requeue(struct gr_udc *dev)
1049 {
1050 	int ret = gr_queue_int(&dev->epo[0], dev->ep0reqo, GFP_ATOMIC);
1051 
1052 	if (ret)
1053 		dev_err(dev->dev, "Could not queue ep0out setup request: %d\n",
1054 			ret);
1055 }
1056 
1057 /*
1058  * The main function dealing with setup requests on ep0.
1059  *
1060  * Must be called with dev->lock held and irqs disabled
1061  */
1062 static void gr_ep0_setup(struct gr_udc *dev, struct gr_request *req)
1063 	__releases(&dev->lock)
1064 	__acquires(&dev->lock)
1065 {
1066 	union {
1067 		struct usb_ctrlrequest ctrl;
1068 		u8 raw[8];
1069 		u32 word[2];
1070 	} u;
1071 	u8 type;
1072 	u8 request;
1073 	u16 value;
1074 	u16 index;
1075 	u16 length;
1076 	int i;
1077 	int status;
1078 
1079 	/* Restore from ep0 halt */
1080 	if (dev->ep0state == GR_EP0_STALL) {
1081 		gr_set_ep0state(dev, GR_EP0_SETUP);
1082 		if (!req->req.actual)
1083 			goto out;
1084 	}
1085 
1086 	if (dev->ep0state == GR_EP0_ISTATUS) {
1087 		gr_set_ep0state(dev, GR_EP0_SETUP);
1088 		if (req->req.actual > 0)
1089 			dev_dbg(dev->dev,
1090 				"Unexpected setup packet at state %s\n",
1091 				gr_ep0state_string(GR_EP0_ISTATUS));
1092 		else
1093 			goto out; /* Got expected ZLP */
1094 	} else if (dev->ep0state != GR_EP0_SETUP) {
1095 		dev_info(dev->dev,
1096 			 "Unexpected ep0out request at state %s - stalling\n",
1097 			 gr_ep0state_string(dev->ep0state));
1098 		gr_control_stall(dev);
1099 		gr_set_ep0state(dev, GR_EP0_SETUP);
1100 		goto out;
1101 	} else if (!req->req.actual) {
1102 		dev_dbg(dev->dev, "Unexpected ZLP at state %s\n",
1103 			gr_ep0state_string(dev->ep0state));
1104 		goto out;
1105 	}
1106 
1107 	/* Handle SETUP packet */
1108 	for (i = 0; i < req->req.actual; i++)
1109 		u.raw[i] = ((u8 *)req->req.buf)[i];
1110 
1111 	type = u.ctrl.bRequestType;
1112 	request = u.ctrl.bRequest;
1113 	value = le16_to_cpu(u.ctrl.wValue);
1114 	index = le16_to_cpu(u.ctrl.wIndex);
1115 	length = le16_to_cpu(u.ctrl.wLength);
1116 
1117 	gr_dbgprint_devreq(dev, type, request, value, index, length);
1118 
1119 	/* Check for data stage */
1120 	if (length) {
1121 		if (type & USB_DIR_IN)
1122 			gr_set_ep0state(dev, GR_EP0_IDATA);
1123 		else
1124 			gr_set_ep0state(dev, GR_EP0_ODATA);
1125 	}
1126 
1127 	status = 1; /* Positive status flags delegation */
1128 	if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1129 		switch (type & USB_RECIP_MASK) {
1130 		case USB_RECIP_DEVICE:
1131 			status = gr_device_request(dev, type, request,
1132 						   value, index);
1133 			break;
1134 		case USB_RECIP_ENDPOINT:
1135 			status =  gr_endpoint_request(dev, type, request,
1136 						      value, index);
1137 			break;
1138 		case USB_RECIP_INTERFACE:
1139 			status = gr_interface_request(dev, type, request,
1140 						      value, index);
1141 			break;
1142 		}
1143 	}
1144 
1145 	if (status > 0) {
1146 		spin_unlock(&dev->lock);
1147 
1148 		dev_vdbg(dev->dev, "DELEGATE\n");
1149 		status = dev->driver->setup(&dev->gadget, &u.ctrl);
1150 
1151 		spin_lock(&dev->lock);
1152 	}
1153 
1154 	/* Generate STALL on both ep0out and ep0in if requested */
1155 	if (unlikely(status < 0)) {
1156 		dev_vdbg(dev->dev, "STALL\n");
1157 		gr_control_stall(dev);
1158 	}
1159 
1160 	if ((type & USB_TYPE_MASK) == USB_TYPE_STANDARD &&
1161 	    request == USB_REQ_SET_CONFIGURATION) {
1162 		if (!value) {
1163 			dev_dbg(dev->dev, "STATUS: deconfigured\n");
1164 			usb_gadget_set_state(&dev->gadget, USB_STATE_ADDRESS);
1165 		} else if (status >= 0) {
1166 			/* Not configured unless gadget OK:s it */
1167 			dev_dbg(dev->dev, "STATUS: configured: %d\n", value);
1168 			usb_gadget_set_state(&dev->gadget,
1169 					     USB_STATE_CONFIGURED);
1170 		}
1171 	}
1172 
1173 	/* Get ready for next stage */
1174 	if (dev->ep0state == GR_EP0_ODATA)
1175 		gr_set_ep0state(dev, GR_EP0_OSTATUS);
1176 	else if (dev->ep0state == GR_EP0_IDATA)
1177 		gr_set_ep0state(dev, GR_EP0_ISTATUS);
1178 	else
1179 		gr_set_ep0state(dev, GR_EP0_SETUP);
1180 
1181 out:
1182 	gr_ep0out_requeue(dev);
1183 }
1184 
1185 /* ---------------------------------------------------------------------- */
1186 /* VBUS and USB reset handling */
1187 
1188 /* Must be called with dev->lock held and irqs disabled  */
1189 static void gr_vbus_connected(struct gr_udc *dev, u32 status)
1190 {
1191 	u32 control;
1192 
1193 	dev->gadget.speed = GR_SPEED(status);
1194 	usb_gadget_set_state(&dev->gadget, USB_STATE_POWERED);
1195 
1196 	/* Turn on full interrupts and pullup */
1197 	control = (GR_CONTROL_SI | GR_CONTROL_UI | GR_CONTROL_VI |
1198 		   GR_CONTROL_SP | GR_CONTROL_EP);
1199 	gr_write32(&dev->regs->control, control);
1200 }
1201 
1202 /* Must be called with dev->lock held */
1203 static void gr_enable_vbus_detect(struct gr_udc *dev)
1204 {
1205 	u32 status;
1206 
1207 	dev->irq_enabled = 1;
1208 	wmb(); /* Make sure we do not ignore an interrupt */
1209 	gr_write32(&dev->regs->control, GR_CONTROL_VI);
1210 
1211 	/* Take care of the case we are already plugged in at this point */
1212 	status = gr_read32(&dev->regs->status);
1213 	if (status & GR_STATUS_VB)
1214 		gr_vbus_connected(dev, status);
1215 }
1216 
1217 /* Must be called with dev->lock held and irqs disabled */
1218 static void gr_vbus_disconnected(struct gr_udc *dev)
1219 {
1220 	gr_stop_activity(dev);
1221 
1222 	/* Report disconnect */
1223 	if (dev->driver && dev->driver->disconnect) {
1224 		spin_unlock(&dev->lock);
1225 
1226 		dev->driver->disconnect(&dev->gadget);
1227 
1228 		spin_lock(&dev->lock);
1229 	}
1230 
1231 	gr_enable_vbus_detect(dev);
1232 }
1233 
1234 /* Must be called with dev->lock held and irqs disabled */
1235 static void gr_udc_usbreset(struct gr_udc *dev, u32 status)
1236 {
1237 	gr_set_address(dev, 0);
1238 	gr_set_ep0state(dev, GR_EP0_SETUP);
1239 	usb_gadget_set_state(&dev->gadget, USB_STATE_DEFAULT);
1240 	dev->gadget.speed = GR_SPEED(status);
1241 
1242 	gr_ep_nuke(&dev->epo[0]);
1243 	gr_ep_nuke(&dev->epi[0]);
1244 	dev->epo[0].stopped = 0;
1245 	dev->epi[0].stopped = 0;
1246 	gr_ep0out_requeue(dev);
1247 }
1248 
1249 /* ---------------------------------------------------------------------- */
1250 /* Irq handling */
1251 
1252 /*
1253  * Handles interrupts from in endpoints. Returns whether something was handled.
1254  *
1255  * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1256  */
1257 static int gr_handle_in_ep(struct gr_ep *ep)
1258 {
1259 	struct gr_request *req;
1260 
1261 	req = list_first_entry(&ep->queue, struct gr_request, queue);
1262 	if (!req->last_desc)
1263 		return 0;
1264 
1265 	if (ACCESS_ONCE(req->last_desc->ctrl) & GR_DESC_IN_CTRL_EN)
1266 		return 0; /* Not put in hardware buffers yet */
1267 
1268 	if (gr_read32(&ep->regs->epstat) & (GR_EPSTAT_B1 | GR_EPSTAT_B0))
1269 		return 0; /* Not transmitted yet, still in hardware buffers */
1270 
1271 	/* Write complete */
1272 	gr_dma_advance(ep, 0);
1273 
1274 	return 1;
1275 }
1276 
1277 /*
1278  * Handles interrupts from out endpoints. Returns whether something was handled.
1279  *
1280  * Must be called with dev->lock held, irqs disabled and with !ep->stopped.
1281  */
1282 static int gr_handle_out_ep(struct gr_ep *ep)
1283 {
1284 	u32 ep_dmactrl;
1285 	u32 ctrl;
1286 	u16 len;
1287 	struct gr_request *req;
1288 	struct gr_udc *dev = ep->dev;
1289 
1290 	req = list_first_entry(&ep->queue, struct gr_request, queue);
1291 	if (!req->curr_desc)
1292 		return 0;
1293 
1294 	ctrl = ACCESS_ONCE(req->curr_desc->ctrl);
1295 	if (ctrl & GR_DESC_OUT_CTRL_EN)
1296 		return 0; /* Not received yet */
1297 
1298 	/* Read complete */
1299 	len = ctrl & GR_DESC_OUT_CTRL_LEN_MASK;
1300 	req->req.actual += len;
1301 	if (ctrl & GR_DESC_OUT_CTRL_SE)
1302 		req->setup = 1;
1303 
1304 	if (len < ep->ep.maxpacket || req->req.actual >= req->req.length) {
1305 		/* Short packet or >= expected size - we are done */
1306 
1307 		if ((ep == &dev->epo[0]) && (dev->ep0state == GR_EP0_OSTATUS)) {
1308 			/*
1309 			 * Send a status stage ZLP to ack the DATA stage in the
1310 			 * OUT direction. This needs to be done before
1311 			 * gr_dma_advance as that can lead to a call to
1312 			 * ep0_setup that can change dev->ep0state.
1313 			 */
1314 			gr_ep0_respond_empty(dev);
1315 			gr_set_ep0state(dev, GR_EP0_SETUP);
1316 		}
1317 
1318 		gr_dma_advance(ep, 0);
1319 	} else {
1320 		/* Not done yet. Enable the next descriptor to receive more. */
1321 		req->curr_desc = req->curr_desc->next_desc;
1322 		req->curr_desc->ctrl |= GR_DESC_OUT_CTRL_EN;
1323 
1324 		ep_dmactrl = gr_read32(&ep->regs->dmactrl);
1325 		gr_write32(&ep->regs->dmactrl, ep_dmactrl | GR_DMACTRL_DA);
1326 	}
1327 
1328 	return 1;
1329 }
1330 
1331 /*
1332  * Handle state changes. Returns whether something was handled.
1333  *
1334  * Must be called with dev->lock held and irqs disabled.
1335  */
1336 static int gr_handle_state_changes(struct gr_udc *dev)
1337 {
1338 	u32 status = gr_read32(&dev->regs->status);
1339 	int handled = 0;
1340 	int powstate = !(dev->gadget.state == USB_STATE_NOTATTACHED ||
1341 			 dev->gadget.state == USB_STATE_ATTACHED);
1342 
1343 	/* VBUS valid detected */
1344 	if (!powstate && (status & GR_STATUS_VB)) {
1345 		dev_dbg(dev->dev, "STATUS: vbus valid detected\n");
1346 		gr_vbus_connected(dev, status);
1347 		handled = 1;
1348 	}
1349 
1350 	/* Disconnect */
1351 	if (powstate && !(status & GR_STATUS_VB)) {
1352 		dev_dbg(dev->dev, "STATUS: vbus invalid detected\n");
1353 		gr_vbus_disconnected(dev);
1354 		handled = 1;
1355 	}
1356 
1357 	/* USB reset detected */
1358 	if (status & GR_STATUS_UR) {
1359 		dev_dbg(dev->dev, "STATUS: USB reset - speed is %s\n",
1360 			GR_SPEED_STR(status));
1361 		gr_write32(&dev->regs->status, GR_STATUS_UR);
1362 		gr_udc_usbreset(dev, status);
1363 		handled = 1;
1364 	}
1365 
1366 	/* Speed change */
1367 	if (dev->gadget.speed != GR_SPEED(status)) {
1368 		dev_dbg(dev->dev, "STATUS: USB Speed change to %s\n",
1369 			GR_SPEED_STR(status));
1370 		dev->gadget.speed = GR_SPEED(status);
1371 		handled = 1;
1372 	}
1373 
1374 	/* Going into suspend */
1375 	if ((dev->ep0state != GR_EP0_SUSPEND) && !(status & GR_STATUS_SU)) {
1376 		dev_dbg(dev->dev, "STATUS: USB suspend\n");
1377 		gr_set_ep0state(dev, GR_EP0_SUSPEND);
1378 		dev->suspended_from = dev->gadget.state;
1379 		usb_gadget_set_state(&dev->gadget, USB_STATE_SUSPENDED);
1380 
1381 		if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1382 		    dev->driver && dev->driver->suspend) {
1383 			spin_unlock(&dev->lock);
1384 
1385 			dev->driver->suspend(&dev->gadget);
1386 
1387 			spin_lock(&dev->lock);
1388 		}
1389 		handled = 1;
1390 	}
1391 
1392 	/* Coming out of suspend */
1393 	if ((dev->ep0state == GR_EP0_SUSPEND) && (status & GR_STATUS_SU)) {
1394 		dev_dbg(dev->dev, "STATUS: USB resume\n");
1395 		if (dev->suspended_from == USB_STATE_POWERED)
1396 			gr_set_ep0state(dev, GR_EP0_DISCONNECT);
1397 		else
1398 			gr_set_ep0state(dev, GR_EP0_SETUP);
1399 		usb_gadget_set_state(&dev->gadget, dev->suspended_from);
1400 
1401 		if ((dev->gadget.speed != USB_SPEED_UNKNOWN) &&
1402 		    dev->driver && dev->driver->resume) {
1403 			spin_unlock(&dev->lock);
1404 
1405 			dev->driver->resume(&dev->gadget);
1406 
1407 			spin_lock(&dev->lock);
1408 		}
1409 		handled = 1;
1410 	}
1411 
1412 	return handled;
1413 }
1414 
1415 /* Non-interrupt context irq handler */
1416 static irqreturn_t gr_irq_handler(int irq, void *_dev)
1417 {
1418 	struct gr_udc *dev = _dev;
1419 	struct gr_ep *ep;
1420 	int handled = 0;
1421 	int i;
1422 	unsigned long flags;
1423 
1424 	spin_lock_irqsave(&dev->lock, flags);
1425 
1426 	if (!dev->irq_enabled)
1427 		goto out;
1428 
1429 	/*
1430 	 * Check IN ep interrupts. We check these before the OUT eps because
1431 	 * some gadgets reuse the request that might already be currently
1432 	 * outstanding and needs to be completed (mainly setup requests).
1433 	 */
1434 	for (i = 0; i < dev->nepi; i++) {
1435 		ep = &dev->epi[i];
1436 		if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1437 			handled = gr_handle_in_ep(ep) || handled;
1438 	}
1439 
1440 	/* Check OUT ep interrupts */
1441 	for (i = 0; i < dev->nepo; i++) {
1442 		ep = &dev->epo[i];
1443 		if (!ep->stopped && !ep->callback && !list_empty(&ep->queue))
1444 			handled = gr_handle_out_ep(ep) || handled;
1445 	}
1446 
1447 	/* Check status interrupts */
1448 	handled = gr_handle_state_changes(dev) || handled;
1449 
1450 	/*
1451 	 * Check AMBA DMA errors. Only check if we didn't find anything else to
1452 	 * handle because this shouldn't happen if we did everything right.
1453 	 */
1454 	if (!handled) {
1455 		list_for_each_entry(ep, &dev->ep_list, ep_list) {
1456 			if (gr_read32(&ep->regs->dmactrl) & GR_DMACTRL_AE) {
1457 				dev_err(dev->dev,
1458 					"AMBA Error occurred for %s\n",
1459 					ep->ep.name);
1460 				handled = 1;
1461 			}
1462 		}
1463 	}
1464 
1465 out:
1466 	spin_unlock_irqrestore(&dev->lock, flags);
1467 
1468 	return handled ? IRQ_HANDLED : IRQ_NONE;
1469 }
1470 
1471 /* Interrupt context irq handler */
1472 static irqreturn_t gr_irq(int irq, void *_dev)
1473 {
1474 	struct gr_udc *dev = _dev;
1475 
1476 	if (!dev->irq_enabled)
1477 		return IRQ_NONE;
1478 
1479 	return IRQ_WAKE_THREAD;
1480 }
1481 
1482 /* ---------------------------------------------------------------------- */
1483 /* USB ep ops */
1484 
1485 /* Enable endpoint. Not for ep0in and ep0out that are handled separately. */
1486 static int gr_ep_enable(struct usb_ep *_ep,
1487 			const struct usb_endpoint_descriptor *desc)
1488 {
1489 	struct gr_udc *dev;
1490 	struct gr_ep *ep;
1491 	u8 mode;
1492 	u8 nt;
1493 	u16 max;
1494 	u16 buffer_size = 0;
1495 	u32 epctrl;
1496 
1497 	ep = container_of(_ep, struct gr_ep, ep);
1498 	if (!_ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT)
1499 		return -EINVAL;
1500 
1501 	dev = ep->dev;
1502 
1503 	/* 'ep0' IN and OUT are reserved */
1504 	if (ep == &dev->epo[0] || ep == &dev->epi[0])
1505 		return -EINVAL;
1506 
1507 	if (!dev->driver || dev->gadget.speed == USB_SPEED_UNKNOWN)
1508 		return -ESHUTDOWN;
1509 
1510 	/* Make sure we are clear for enabling */
1511 	epctrl = gr_read32(&ep->regs->epctrl);
1512 	if (epctrl & GR_EPCTRL_EV)
1513 		return -EBUSY;
1514 
1515 	/* Check that directions match */
1516 	if (!ep->is_in != !usb_endpoint_dir_in(desc))
1517 		return -EINVAL;
1518 
1519 	/* Check ep num */
1520 	if ((!ep->is_in && ep->num >= dev->nepo) ||
1521 	    (ep->is_in && ep->num >= dev->nepi))
1522 		return -EINVAL;
1523 
1524 	if (usb_endpoint_xfer_control(desc)) {
1525 		mode = 0;
1526 	} else if (usb_endpoint_xfer_isoc(desc)) {
1527 		mode = 1;
1528 	} else if (usb_endpoint_xfer_bulk(desc)) {
1529 		mode = 2;
1530 	} else if (usb_endpoint_xfer_int(desc)) {
1531 		mode = 3;
1532 	} else {
1533 		dev_err(dev->dev, "Unknown transfer type for %s\n",
1534 			ep->ep.name);
1535 		return -EINVAL;
1536 	}
1537 
1538 	/*
1539 	 * Bits 10-0 set the max payload. 12-11 set the number of
1540 	 * additional transactions.
1541 	 */
1542 	max = 0x7ff & usb_endpoint_maxp(desc);
1543 	nt = 0x3 & (usb_endpoint_maxp(desc) >> 11);
1544 	buffer_size = GR_BUFFER_SIZE(epctrl);
1545 	if (nt && (mode == 0 || mode == 2)) {
1546 		dev_err(dev->dev,
1547 			"%s mode: multiple trans./microframe not valid\n",
1548 			(mode == 2 ? "Bulk" : "Control"));
1549 		return -EINVAL;
1550 	} else if (nt == 0x3) {
1551 		dev_err(dev->dev,
1552 			"Invalid value 0x3 for additional trans./microframe\n");
1553 		return -EINVAL;
1554 	} else if ((nt + 1) * max > buffer_size) {
1555 		dev_err(dev->dev, "Hw buffer size %d < max payload %d * %d\n",
1556 			buffer_size, (nt + 1), max);
1557 		return -EINVAL;
1558 	} else if (max == 0) {
1559 		dev_err(dev->dev, "Max payload cannot be set to 0\n");
1560 		return -EINVAL;
1561 	} else if (max > ep->ep.maxpacket_limit) {
1562 		dev_err(dev->dev, "Requested max payload %d > limit %d\n",
1563 			max, ep->ep.maxpacket_limit);
1564 		return -EINVAL;
1565 	}
1566 
1567 	spin_lock(&ep->dev->lock);
1568 
1569 	if (!ep->stopped) {
1570 		spin_unlock(&ep->dev->lock);
1571 		return -EBUSY;
1572 	}
1573 
1574 	ep->stopped = 0;
1575 	ep->wedged = 0;
1576 	ep->ep.desc = desc;
1577 	ep->ep.maxpacket = max;
1578 	ep->dma_start = 0;
1579 
1580 
1581 	if (nt) {
1582 		/*
1583 		 * Maximum possible size of all payloads in one microframe
1584 		 * regardless of direction when using high-bandwidth mode.
1585 		 */
1586 		ep->bytes_per_buffer = (nt + 1) * max;
1587 	} else if (ep->is_in) {
1588 		/*
1589 		 * The biggest multiple of maximum packet size that fits into
1590 		 * the buffer. The hardware will split up into many packets in
1591 		 * the IN direction.
1592 		 */
1593 		ep->bytes_per_buffer = (buffer_size / max) * max;
1594 	} else {
1595 		/*
1596 		 * Only single packets will be placed the buffers in the OUT
1597 		 * direction.
1598 		 */
1599 		ep->bytes_per_buffer = max;
1600 	}
1601 
1602 	epctrl = (max << GR_EPCTRL_MAXPL_POS)
1603 		| (nt << GR_EPCTRL_NT_POS)
1604 		| (mode << GR_EPCTRL_TT_POS)
1605 		| GR_EPCTRL_EV;
1606 	if (ep->is_in)
1607 		epctrl |= GR_EPCTRL_PI;
1608 	gr_write32(&ep->regs->epctrl, epctrl);
1609 
1610 	gr_write32(&ep->regs->dmactrl, GR_DMACTRL_IE | GR_DMACTRL_AI);
1611 
1612 	spin_unlock(&ep->dev->lock);
1613 
1614 	dev_dbg(ep->dev->dev, "EP: %s enabled - %s with %d bytes/buffer\n",
1615 		ep->ep.name, gr_modestring[mode], ep->bytes_per_buffer);
1616 	return 0;
1617 }
1618 
1619 /* Disable endpoint. Not for ep0in and ep0out that are handled separately. */
1620 static int gr_ep_disable(struct usb_ep *_ep)
1621 {
1622 	struct gr_ep *ep;
1623 	struct gr_udc *dev;
1624 	unsigned long flags;
1625 
1626 	ep = container_of(_ep, struct gr_ep, ep);
1627 	if (!_ep || !ep->ep.desc)
1628 		return -ENODEV;
1629 
1630 	dev = ep->dev;
1631 
1632 	/* 'ep0' IN and OUT are reserved */
1633 	if (ep == &dev->epo[0] || ep == &dev->epi[0])
1634 		return -EINVAL;
1635 
1636 	if (dev->ep0state == GR_EP0_SUSPEND)
1637 		return -EBUSY;
1638 
1639 	dev_dbg(ep->dev->dev, "EP: disable %s\n", ep->ep.name);
1640 
1641 	spin_lock_irqsave(&dev->lock, flags);
1642 
1643 	gr_ep_nuke(ep);
1644 	gr_ep_reset(ep);
1645 	ep->ep.desc = NULL;
1646 
1647 	spin_unlock_irqrestore(&dev->lock, flags);
1648 
1649 	return 0;
1650 }
1651 
1652 /*
1653  * Frees a request, but not any DMA buffers associated with it
1654  * (gr_finish_request should already have taken care of that).
1655  */
1656 static void gr_free_request(struct usb_ep *_ep, struct usb_request *_req)
1657 {
1658 	struct gr_request *req;
1659 
1660 	if (!_ep || !_req)
1661 		return;
1662 	req = container_of(_req, struct gr_request, req);
1663 
1664 	/* Leads to memory leak */
1665 	WARN(!list_empty(&req->queue),
1666 	     "request not dequeued properly before freeing\n");
1667 
1668 	kfree(req);
1669 }
1670 
1671 /* Queue a request from the gadget */
1672 static int gr_queue_ext(struct usb_ep *_ep, struct usb_request *_req,
1673 			gfp_t gfp_flags)
1674 {
1675 	struct gr_ep *ep;
1676 	struct gr_request *req;
1677 	struct gr_udc *dev;
1678 	int ret;
1679 
1680 	if (unlikely(!_ep || !_req))
1681 		return -EINVAL;
1682 
1683 	ep = container_of(_ep, struct gr_ep, ep);
1684 	req = container_of(_req, struct gr_request, req);
1685 	dev = ep->dev;
1686 
1687 	spin_lock(&ep->dev->lock);
1688 
1689 	/*
1690 	 * The ep0 pointer in the gadget struct is used both for ep0in and
1691 	 * ep0out. In a data stage in the out direction ep0out needs to be used
1692 	 * instead of the default ep0in. Completion functions might use
1693 	 * driver_data, so that needs to be copied as well.
1694 	 */
1695 	if ((ep == &dev->epi[0]) && (dev->ep0state == GR_EP0_ODATA)) {
1696 		ep = &dev->epo[0];
1697 		ep->ep.driver_data = dev->epi[0].ep.driver_data;
1698 	}
1699 
1700 	if (ep->is_in)
1701 		gr_dbgprint_request("EXTERN", ep, req);
1702 
1703 	ret = gr_queue(ep, req, GFP_ATOMIC);
1704 
1705 	spin_unlock(&ep->dev->lock);
1706 
1707 	return ret;
1708 }
1709 
1710 /* Dequeue JUST ONE request */
1711 static int gr_dequeue(struct usb_ep *_ep, struct usb_request *_req)
1712 {
1713 	struct gr_request *req;
1714 	struct gr_ep *ep;
1715 	struct gr_udc *dev;
1716 	int ret = 0;
1717 	unsigned long flags;
1718 
1719 	ep = container_of(_ep, struct gr_ep, ep);
1720 	if (!_ep || !_req || (!ep->ep.desc && ep->num != 0))
1721 		return -EINVAL;
1722 	dev = ep->dev;
1723 	if (!dev->driver)
1724 		return -ESHUTDOWN;
1725 
1726 	/* We can't touch (DMA) registers when suspended */
1727 	if (dev->ep0state == GR_EP0_SUSPEND)
1728 		return -EBUSY;
1729 
1730 	spin_lock_irqsave(&dev->lock, flags);
1731 
1732 	/* Make sure it's actually queued on this endpoint */
1733 	list_for_each_entry(req, &ep->queue, queue) {
1734 		if (&req->req == _req)
1735 			break;
1736 	}
1737 	if (&req->req != _req) {
1738 		ret = -EINVAL;
1739 		goto out;
1740 	}
1741 
1742 	if (list_first_entry(&ep->queue, struct gr_request, queue) == req) {
1743 		/* This request is currently being processed */
1744 		gr_abort_dma(ep);
1745 		if (ep->stopped)
1746 			gr_finish_request(ep, req, -ECONNRESET);
1747 		else
1748 			gr_dma_advance(ep, -ECONNRESET);
1749 	} else if (!list_empty(&req->queue)) {
1750 		/* Not being processed - gr_finish_request dequeues it */
1751 		gr_finish_request(ep, req, -ECONNRESET);
1752 	} else {
1753 		ret = -EOPNOTSUPP;
1754 	}
1755 
1756 out:
1757 	spin_unlock_irqrestore(&dev->lock, flags);
1758 
1759 	return ret;
1760 }
1761 
1762 /* Helper for gr_set_halt and gr_set_wedge */
1763 static int gr_set_halt_wedge(struct usb_ep *_ep, int halt, int wedge)
1764 {
1765 	int ret;
1766 	struct gr_ep *ep;
1767 
1768 	if (!_ep)
1769 		return -ENODEV;
1770 	ep = container_of(_ep, struct gr_ep, ep);
1771 
1772 	spin_lock(&ep->dev->lock);
1773 
1774 	/* Halting an IN endpoint should fail if queue is not empty */
1775 	if (halt && ep->is_in && !list_empty(&ep->queue)) {
1776 		ret = -EAGAIN;
1777 		goto out;
1778 	}
1779 
1780 	ret = gr_ep_halt_wedge(ep, halt, wedge, 0);
1781 
1782 out:
1783 	spin_unlock(&ep->dev->lock);
1784 
1785 	return ret;
1786 }
1787 
1788 /* Halt endpoint */
1789 static int gr_set_halt(struct usb_ep *_ep, int halt)
1790 {
1791 	return gr_set_halt_wedge(_ep, halt, 0);
1792 }
1793 
1794 /* Halt and wedge endpoint */
1795 static int gr_set_wedge(struct usb_ep *_ep)
1796 {
1797 	return gr_set_halt_wedge(_ep, 1, 1);
1798 }
1799 
1800 /*
1801  * Return the total number of bytes currently stored in the internal buffers of
1802  * the endpoint.
1803  */
1804 static int gr_fifo_status(struct usb_ep *_ep)
1805 {
1806 	struct gr_ep *ep;
1807 	u32 epstat;
1808 	u32 bytes = 0;
1809 
1810 	if (!_ep)
1811 		return -ENODEV;
1812 	ep = container_of(_ep, struct gr_ep, ep);
1813 
1814 	epstat = gr_read32(&ep->regs->epstat);
1815 
1816 	if (epstat & GR_EPSTAT_B0)
1817 		bytes += (epstat & GR_EPSTAT_B0CNT_MASK) >> GR_EPSTAT_B0CNT_POS;
1818 	if (epstat & GR_EPSTAT_B1)
1819 		bytes += (epstat & GR_EPSTAT_B1CNT_MASK) >> GR_EPSTAT_B1CNT_POS;
1820 
1821 	return bytes;
1822 }
1823 
1824 
1825 /* Empty data from internal buffers of an endpoint. */
1826 static void gr_fifo_flush(struct usb_ep *_ep)
1827 {
1828 	struct gr_ep *ep;
1829 	u32 epctrl;
1830 
1831 	if (!_ep)
1832 		return;
1833 	ep = container_of(_ep, struct gr_ep, ep);
1834 	dev_vdbg(ep->dev->dev, "EP: flush fifo %s\n", ep->ep.name);
1835 
1836 	spin_lock(&ep->dev->lock);
1837 
1838 	epctrl = gr_read32(&ep->regs->epctrl);
1839 	epctrl |= GR_EPCTRL_CB;
1840 	gr_write32(&ep->regs->epctrl, epctrl);
1841 
1842 	spin_unlock(&ep->dev->lock);
1843 }
1844 
1845 static struct usb_ep_ops gr_ep_ops = {
1846 	.enable		= gr_ep_enable,
1847 	.disable	= gr_ep_disable,
1848 
1849 	.alloc_request	= gr_alloc_request,
1850 	.free_request	= gr_free_request,
1851 
1852 	.queue		= gr_queue_ext,
1853 	.dequeue	= gr_dequeue,
1854 
1855 	.set_halt	= gr_set_halt,
1856 	.set_wedge	= gr_set_wedge,
1857 	.fifo_status	= gr_fifo_status,
1858 	.fifo_flush	= gr_fifo_flush,
1859 };
1860 
1861 /* ---------------------------------------------------------------------- */
1862 /* USB Gadget ops */
1863 
1864 static int gr_get_frame(struct usb_gadget *_gadget)
1865 {
1866 	struct gr_udc *dev;
1867 
1868 	if (!_gadget)
1869 		return -ENODEV;
1870 	dev = container_of(_gadget, struct gr_udc, gadget);
1871 	return gr_read32(&dev->regs->status) & GR_STATUS_FN_MASK;
1872 }
1873 
1874 static int gr_wakeup(struct usb_gadget *_gadget)
1875 {
1876 	struct gr_udc *dev;
1877 
1878 	if (!_gadget)
1879 		return -ENODEV;
1880 	dev = container_of(_gadget, struct gr_udc, gadget);
1881 
1882 	/* Remote wakeup feature not enabled by host*/
1883 	if (!dev->remote_wakeup)
1884 		return -EINVAL;
1885 
1886 	spin_lock(&dev->lock);
1887 
1888 	gr_write32(&dev->regs->control,
1889 		   gr_read32(&dev->regs->control) | GR_CONTROL_RW);
1890 
1891 	spin_unlock(&dev->lock);
1892 
1893 	return 0;
1894 }
1895 
1896 static int gr_pullup(struct usb_gadget *_gadget, int is_on)
1897 {
1898 	struct gr_udc *dev;
1899 	u32 control;
1900 
1901 	if (!_gadget)
1902 		return -ENODEV;
1903 	dev = container_of(_gadget, struct gr_udc, gadget);
1904 
1905 	spin_lock(&dev->lock);
1906 
1907 	control = gr_read32(&dev->regs->control);
1908 	if (is_on)
1909 		control |= GR_CONTROL_EP;
1910 	else
1911 		control &= ~GR_CONTROL_EP;
1912 	gr_write32(&dev->regs->control, control);
1913 
1914 	spin_unlock(&dev->lock);
1915 
1916 	return 0;
1917 }
1918 
1919 static int gr_udc_start(struct usb_gadget *gadget,
1920 			struct usb_gadget_driver *driver)
1921 {
1922 	struct gr_udc *dev = to_gr_udc(gadget);
1923 
1924 	spin_lock(&dev->lock);
1925 
1926 	/* Hook up the driver */
1927 	driver->driver.bus = NULL;
1928 	dev->driver = driver;
1929 
1930 	/* Get ready for host detection */
1931 	gr_enable_vbus_detect(dev);
1932 
1933 	spin_unlock(&dev->lock);
1934 
1935 	dev_info(dev->dev, "Started with gadget driver '%s'\n",
1936 		 driver->driver.name);
1937 
1938 	return 0;
1939 }
1940 
1941 static int gr_udc_stop(struct usb_gadget *gadget,
1942 		       struct usb_gadget_driver *driver)
1943 {
1944 	struct gr_udc *dev = to_gr_udc(gadget);
1945 	unsigned long flags;
1946 
1947 	spin_lock_irqsave(&dev->lock, flags);
1948 
1949 	dev->driver = NULL;
1950 	gr_stop_activity(dev);
1951 
1952 	spin_unlock_irqrestore(&dev->lock, flags);
1953 
1954 	dev_info(dev->dev, "Stopped\n");
1955 
1956 	return 0;
1957 }
1958 
1959 static const struct usb_gadget_ops gr_ops = {
1960 	.get_frame	= gr_get_frame,
1961 	.wakeup         = gr_wakeup,
1962 	.pullup         = gr_pullup,
1963 	.udc_start	= gr_udc_start,
1964 	.udc_stop	= gr_udc_stop,
1965 	/* Other operations not supported */
1966 };
1967 
1968 /* ---------------------------------------------------------------------- */
1969 /* Module probe, removal and of-matching */
1970 
1971 static const char * const onames[] = {
1972 	"ep0out", "ep1out", "ep2out", "ep3out", "ep4out", "ep5out",
1973 	"ep6out", "ep7out", "ep8out", "ep9out", "ep10out", "ep11out",
1974 	"ep12out", "ep13out", "ep14out", "ep15out"
1975 };
1976 
1977 static const char * const inames[] = {
1978 	"ep0in", "ep1in", "ep2in", "ep3in", "ep4in", "ep5in",
1979 	"ep6in", "ep7in", "ep8in", "ep9in", "ep10in", "ep11in",
1980 	"ep12in", "ep13in", "ep14in", "ep15in"
1981 };
1982 
1983 /* Must be called with dev->lock held */
1984 static int gr_ep_init(struct gr_udc *dev, int num, int is_in, u32 maxplimit)
1985 {
1986 	struct gr_ep *ep;
1987 	struct gr_request *req;
1988 	struct usb_request *_req;
1989 	void *buf;
1990 
1991 	if (is_in) {
1992 		ep = &dev->epi[num];
1993 		ep->ep.name = inames[num];
1994 		ep->regs = &dev->regs->epi[num];
1995 	} else {
1996 		ep = &dev->epo[num];
1997 		ep->ep.name = onames[num];
1998 		ep->regs = &dev->regs->epo[num];
1999 	}
2000 
2001 	gr_ep_reset(ep);
2002 	ep->num = num;
2003 	ep->is_in = is_in;
2004 	ep->dev = dev;
2005 	ep->ep.ops = &gr_ep_ops;
2006 	INIT_LIST_HEAD(&ep->queue);
2007 
2008 	if (num == 0) {
2009 		_req = gr_alloc_request(&ep->ep, GFP_ATOMIC);
2010 		buf = devm_kzalloc(dev->dev, PAGE_SIZE, GFP_DMA | GFP_ATOMIC);
2011 		if (!_req || !buf) {
2012 			/* possible _req freed by gr_probe via gr_remove */
2013 			return -ENOMEM;
2014 		}
2015 
2016 		req = container_of(_req, struct gr_request, req);
2017 		req->req.buf = buf;
2018 		req->req.length = MAX_CTRL_PL_SIZE;
2019 
2020 		if (is_in)
2021 			dev->ep0reqi = req; /* Complete gets set as used */
2022 		else
2023 			dev->ep0reqo = req; /* Completion treated separately */
2024 
2025 		usb_ep_set_maxpacket_limit(&ep->ep, MAX_CTRL_PL_SIZE);
2026 		ep->bytes_per_buffer = MAX_CTRL_PL_SIZE;
2027 	} else {
2028 		usb_ep_set_maxpacket_limit(&ep->ep, (u16)maxplimit);
2029 		list_add_tail(&ep->ep.ep_list, &dev->gadget.ep_list);
2030 	}
2031 	list_add_tail(&ep->ep_list, &dev->ep_list);
2032 
2033 	ep->tailbuf = dma_alloc_coherent(dev->dev, ep->ep.maxpacket_limit,
2034 					 &ep->tailbuf_paddr, GFP_ATOMIC);
2035 	if (!ep->tailbuf)
2036 		return -ENOMEM;
2037 
2038 	return 0;
2039 }
2040 
2041 /* Must be called with dev->lock held */
2042 static int gr_udc_init(struct gr_udc *dev)
2043 {
2044 	struct device_node *np = dev->dev->of_node;
2045 	u32 epctrl_val;
2046 	u32 dmactrl_val;
2047 	int i;
2048 	int ret = 0;
2049 	u32 bufsize;
2050 
2051 	gr_set_address(dev, 0);
2052 
2053 	INIT_LIST_HEAD(&dev->gadget.ep_list);
2054 	dev->gadget.speed = USB_SPEED_UNKNOWN;
2055 	dev->gadget.ep0 = &dev->epi[0].ep;
2056 
2057 	INIT_LIST_HEAD(&dev->ep_list);
2058 	gr_set_ep0state(dev, GR_EP0_DISCONNECT);
2059 
2060 	for (i = 0; i < dev->nepo; i++) {
2061 		if (of_property_read_u32_index(np, "epobufsizes", i, &bufsize))
2062 			bufsize = 1024;
2063 		ret = gr_ep_init(dev, i, 0, bufsize);
2064 		if (ret)
2065 			return ret;
2066 	}
2067 
2068 	for (i = 0; i < dev->nepi; i++) {
2069 		if (of_property_read_u32_index(np, "epibufsizes", i, &bufsize))
2070 			bufsize = 1024;
2071 		ret = gr_ep_init(dev, i, 1, bufsize);
2072 		if (ret)
2073 			return ret;
2074 	}
2075 
2076 	/* Must be disabled by default */
2077 	dev->remote_wakeup = 0;
2078 
2079 	/* Enable ep0out and ep0in */
2080 	epctrl_val = (MAX_CTRL_PL_SIZE << GR_EPCTRL_MAXPL_POS) | GR_EPCTRL_EV;
2081 	dmactrl_val = GR_DMACTRL_IE | GR_DMACTRL_AI;
2082 	gr_write32(&dev->epo[0].regs->epctrl, epctrl_val);
2083 	gr_write32(&dev->epi[0].regs->epctrl, epctrl_val | GR_EPCTRL_PI);
2084 	gr_write32(&dev->epo[0].regs->dmactrl, dmactrl_val);
2085 	gr_write32(&dev->epi[0].regs->dmactrl, dmactrl_val);
2086 
2087 	return 0;
2088 }
2089 
2090 static void gr_ep_remove(struct gr_udc *dev, int num, int is_in)
2091 {
2092 	struct gr_ep *ep;
2093 
2094 	if (is_in)
2095 		ep = &dev->epi[num];
2096 	else
2097 		ep = &dev->epo[num];
2098 
2099 	if (ep->tailbuf)
2100 		dma_free_coherent(dev->dev, ep->ep.maxpacket_limit,
2101 				  ep->tailbuf, ep->tailbuf_paddr);
2102 }
2103 
2104 static int gr_remove(struct platform_device *pdev)
2105 {
2106 	struct gr_udc *dev = platform_get_drvdata(pdev);
2107 	int i;
2108 
2109 	if (dev->added)
2110 		usb_del_gadget_udc(&dev->gadget); /* Shuts everything down */
2111 	if (dev->driver)
2112 		return -EBUSY;
2113 
2114 	gr_dfs_delete(dev);
2115 	if (dev->desc_pool)
2116 		dma_pool_destroy(dev->desc_pool);
2117 	platform_set_drvdata(pdev, NULL);
2118 
2119 	gr_free_request(&dev->epi[0].ep, &dev->ep0reqi->req);
2120 	gr_free_request(&dev->epo[0].ep, &dev->ep0reqo->req);
2121 
2122 	for (i = 0; i < dev->nepo; i++)
2123 		gr_ep_remove(dev, i, 0);
2124 	for (i = 0; i < dev->nepi; i++)
2125 		gr_ep_remove(dev, i, 1);
2126 
2127 	return 0;
2128 }
2129 static int gr_request_irq(struct gr_udc *dev, int irq)
2130 {
2131 	return devm_request_threaded_irq(dev->dev, irq, gr_irq, gr_irq_handler,
2132 					 IRQF_SHARED, driver_name, dev);
2133 }
2134 
2135 static int gr_probe(struct platform_device *pdev)
2136 {
2137 	struct gr_udc *dev;
2138 	struct resource *res;
2139 	struct gr_regs __iomem *regs;
2140 	int retval;
2141 	u32 status;
2142 
2143 	dev = devm_kzalloc(&pdev->dev, sizeof(*dev), GFP_KERNEL);
2144 	if (!dev)
2145 		return -ENOMEM;
2146 	dev->dev = &pdev->dev;
2147 
2148 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2149 	regs = devm_ioremap_resource(dev->dev, res);
2150 	if (IS_ERR(regs))
2151 		return PTR_ERR(regs);
2152 
2153 	dev->irq = platform_get_irq(pdev, 0);
2154 	if (dev->irq <= 0) {
2155 		dev_err(dev->dev, "No irq found\n");
2156 		return -ENODEV;
2157 	}
2158 
2159 	/* Some core configurations has separate irqs for IN and OUT events */
2160 	dev->irqi = platform_get_irq(pdev, 1);
2161 	if (dev->irqi > 0) {
2162 		dev->irqo = platform_get_irq(pdev, 2);
2163 		if (dev->irqo <= 0) {
2164 			dev_err(dev->dev, "Found irqi but not irqo\n");
2165 			return -ENODEV;
2166 		}
2167 	} else {
2168 		dev->irqi = 0;
2169 	}
2170 
2171 	dev->gadget.name = driver_name;
2172 	dev->gadget.max_speed = USB_SPEED_HIGH;
2173 	dev->gadget.ops = &gr_ops;
2174 
2175 	spin_lock_init(&dev->lock);
2176 	dev->regs = regs;
2177 
2178 	platform_set_drvdata(pdev, dev);
2179 
2180 	/* Determine number of endpoints and data interface mode */
2181 	status = gr_read32(&dev->regs->status);
2182 	dev->nepi = ((status & GR_STATUS_NEPI_MASK) >> GR_STATUS_NEPI_POS) + 1;
2183 	dev->nepo = ((status & GR_STATUS_NEPO_MASK) >> GR_STATUS_NEPO_POS) + 1;
2184 
2185 	if (!(status & GR_STATUS_DM)) {
2186 		dev_err(dev->dev, "Slave mode cores are not supported\n");
2187 		return -ENODEV;
2188 	}
2189 
2190 	/* --- Effects of the following calls might need explicit cleanup --- */
2191 
2192 	/* Create DMA pool for descriptors */
2193 	dev->desc_pool = dma_pool_create("desc_pool", dev->dev,
2194 					 sizeof(struct gr_dma_desc), 4, 0);
2195 	if (!dev->desc_pool) {
2196 		dev_err(dev->dev, "Could not allocate DMA pool");
2197 		return -ENOMEM;
2198 	}
2199 
2200 	spin_lock(&dev->lock);
2201 
2202 	/* Inside lock so that no gadget can use this udc until probe is done */
2203 	retval = usb_add_gadget_udc(dev->dev, &dev->gadget);
2204 	if (retval) {
2205 		dev_err(dev->dev, "Could not add gadget udc");
2206 		goto out;
2207 	}
2208 	dev->added = 1;
2209 
2210 	retval = gr_udc_init(dev);
2211 	if (retval)
2212 		goto out;
2213 
2214 	gr_dfs_create(dev);
2215 
2216 	/* Clear all interrupt enables that might be left on since last boot */
2217 	gr_disable_interrupts_and_pullup(dev);
2218 
2219 	retval = gr_request_irq(dev, dev->irq);
2220 	if (retval) {
2221 		dev_err(dev->dev, "Failed to request irq %d\n", dev->irq);
2222 		goto out;
2223 	}
2224 
2225 	if (dev->irqi) {
2226 		retval = gr_request_irq(dev, dev->irqi);
2227 		if (retval) {
2228 			dev_err(dev->dev, "Failed to request irqi %d\n",
2229 				dev->irqi);
2230 			goto out;
2231 		}
2232 		retval = gr_request_irq(dev, dev->irqo);
2233 		if (retval) {
2234 			dev_err(dev->dev, "Failed to request irqo %d\n",
2235 				dev->irqo);
2236 			goto out;
2237 		}
2238 	}
2239 
2240 	if (dev->irqi)
2241 		dev_info(dev->dev, "regs: %p, irqs %d, %d, %d\n", dev->regs,
2242 			 dev->irq, dev->irqi, dev->irqo);
2243 	else
2244 		dev_info(dev->dev, "regs: %p, irq %d\n", dev->regs, dev->irq);
2245 
2246 out:
2247 	spin_unlock(&dev->lock);
2248 
2249 	if (retval)
2250 		gr_remove(pdev);
2251 
2252 	return retval;
2253 }
2254 
2255 static const struct of_device_id gr_match[] = {
2256 	{.name = "GAISLER_USBDC"},
2257 	{.name = "01_021"},
2258 	{},
2259 };
2260 MODULE_DEVICE_TABLE(of, gr_match);
2261 
2262 static struct platform_driver gr_driver = {
2263 	.driver = {
2264 		.name = DRIVER_NAME,
2265 		.owner = THIS_MODULE,
2266 		.of_match_table = gr_match,
2267 	},
2268 	.probe = gr_probe,
2269 	.remove = gr_remove,
2270 };
2271 module_platform_driver(gr_driver);
2272 
2273 MODULE_AUTHOR("Aeroflex Gaisler AB.");
2274 MODULE_DESCRIPTION(DRIVER_DESC);
2275 MODULE_LICENSE("GPL");
2276