xref: /openbmc/linux/drivers/usb/dwc2/gadget.c (revision ae3473231e77a3f1909d48cd144cebe5e1d049b3)
1 /**
2  * Copyright (c) 2011 Samsung Electronics Co., Ltd.
3  *		http://www.samsung.com
4  *
5  * Copyright 2008 Openmoko, Inc.
6  * Copyright 2008 Simtec Electronics
7  *      Ben Dooks <ben@simtec.co.uk>
8  *      http://armlinux.simtec.co.uk/
9  *
10  * S3C USB2.0 High-speed / OtG driver
11  *
12  * This program is free software; you can redistribute it and/or modify
13  * it under the terms of the GNU General Public License version 2 as
14  * published by the Free Software Foundation.
15  */
16 
17 #include <linux/kernel.h>
18 #include <linux/module.h>
19 #include <linux/spinlock.h>
20 #include <linux/interrupt.h>
21 #include <linux/platform_device.h>
22 #include <linux/dma-mapping.h>
23 #include <linux/mutex.h>
24 #include <linux/seq_file.h>
25 #include <linux/delay.h>
26 #include <linux/io.h>
27 #include <linux/slab.h>
28 #include <linux/of_platform.h>
29 
30 #include <linux/usb/ch9.h>
31 #include <linux/usb/gadget.h>
32 #include <linux/usb/phy.h>
33 
34 #include "core.h"
35 #include "hw.h"
36 
37 /* conversion functions */
38 static inline struct dwc2_hsotg_req *our_req(struct usb_request *req)
39 {
40 	return container_of(req, struct dwc2_hsotg_req, req);
41 }
42 
43 static inline struct dwc2_hsotg_ep *our_ep(struct usb_ep *ep)
44 {
45 	return container_of(ep, struct dwc2_hsotg_ep, ep);
46 }
47 
48 static inline struct dwc2_hsotg *to_hsotg(struct usb_gadget *gadget)
49 {
50 	return container_of(gadget, struct dwc2_hsotg, gadget);
51 }
52 
53 static inline void __orr32(void __iomem *ptr, u32 val)
54 {
55 	dwc2_writel(dwc2_readl(ptr) | val, ptr);
56 }
57 
58 static inline void __bic32(void __iomem *ptr, u32 val)
59 {
60 	dwc2_writel(dwc2_readl(ptr) & ~val, ptr);
61 }
62 
63 static inline struct dwc2_hsotg_ep *index_to_ep(struct dwc2_hsotg *hsotg,
64 						u32 ep_index, u32 dir_in)
65 {
66 	if (dir_in)
67 		return hsotg->eps_in[ep_index];
68 	else
69 		return hsotg->eps_out[ep_index];
70 }
71 
72 /* forward declaration of functions */
73 static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg);
74 
75 /**
76  * using_dma - return the DMA status of the driver.
77  * @hsotg: The driver state.
78  *
79  * Return true if we're using DMA.
80  *
81  * Currently, we have the DMA support code worked into everywhere
82  * that needs it, but the AMBA DMA implementation in the hardware can
83  * only DMA from 32bit aligned addresses. This means that gadgets such
84  * as the CDC Ethernet cannot work as they often pass packets which are
85  * not 32bit aligned.
86  *
87  * Unfortunately the choice to use DMA or not is global to the controller
88  * and seems to be only settable when the controller is being put through
89  * a core reset. This means we either need to fix the gadgets to take
90  * account of DMA alignment, or add bounce buffers (yuerk).
91  *
92  * g_using_dma is set depending on dts flag.
93  */
94 static inline bool using_dma(struct dwc2_hsotg *hsotg)
95 {
96 	return hsotg->params.g_dma;
97 }
98 
99 /*
100  * using_desc_dma - return the descriptor DMA status of the driver.
101  * @hsotg: The driver state.
102  *
103  * Return true if we're using descriptor DMA.
104  */
105 static inline bool using_desc_dma(struct dwc2_hsotg *hsotg)
106 {
107 	return hsotg->params.g_dma_desc;
108 }
109 
110 /**
111  * dwc2_gadget_incr_frame_num - Increments the targeted frame number.
112  * @hs_ep: The endpoint
113  * @increment: The value to increment by
114  *
115  * This function will also check if the frame number overruns DSTS_SOFFN_LIMIT.
116  * If an overrun occurs it will wrap the value and set the frame_overrun flag.
117  */
118 static inline void dwc2_gadget_incr_frame_num(struct dwc2_hsotg_ep *hs_ep)
119 {
120 	hs_ep->target_frame += hs_ep->interval;
121 	if (hs_ep->target_frame > DSTS_SOFFN_LIMIT) {
122 		hs_ep->frame_overrun = 1;
123 		hs_ep->target_frame &= DSTS_SOFFN_LIMIT;
124 	} else {
125 		hs_ep->frame_overrun = 0;
126 	}
127 }
128 
129 /**
130  * dwc2_hsotg_en_gsint - enable one or more of the general interrupt
131  * @hsotg: The device state
132  * @ints: A bitmask of the interrupts to enable
133  */
134 static void dwc2_hsotg_en_gsint(struct dwc2_hsotg *hsotg, u32 ints)
135 {
136 	u32 gsintmsk = dwc2_readl(hsotg->regs + GINTMSK);
137 	u32 new_gsintmsk;
138 
139 	new_gsintmsk = gsintmsk | ints;
140 
141 	if (new_gsintmsk != gsintmsk) {
142 		dev_dbg(hsotg->dev, "gsintmsk now 0x%08x\n", new_gsintmsk);
143 		dwc2_writel(new_gsintmsk, hsotg->regs + GINTMSK);
144 	}
145 }
146 
147 /**
148  * dwc2_hsotg_disable_gsint - disable one or more of the general interrupt
149  * @hsotg: The device state
150  * @ints: A bitmask of the interrupts to enable
151  */
152 static void dwc2_hsotg_disable_gsint(struct dwc2_hsotg *hsotg, u32 ints)
153 {
154 	u32 gsintmsk = dwc2_readl(hsotg->regs + GINTMSK);
155 	u32 new_gsintmsk;
156 
157 	new_gsintmsk = gsintmsk & ~ints;
158 
159 	if (new_gsintmsk != gsintmsk)
160 		dwc2_writel(new_gsintmsk, hsotg->regs + GINTMSK);
161 }
162 
163 /**
164  * dwc2_hsotg_ctrl_epint - enable/disable an endpoint irq
165  * @hsotg: The device state
166  * @ep: The endpoint index
167  * @dir_in: True if direction is in.
168  * @en: The enable value, true to enable
169  *
170  * Set or clear the mask for an individual endpoint's interrupt
171  * request.
172  */
173 static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg,
174 				 unsigned int ep, unsigned int dir_in,
175 				 unsigned int en)
176 {
177 	unsigned long flags;
178 	u32 bit = 1 << ep;
179 	u32 daint;
180 
181 	if (!dir_in)
182 		bit <<= 16;
183 
184 	local_irq_save(flags);
185 	daint = dwc2_readl(hsotg->regs + DAINTMSK);
186 	if (en)
187 		daint |= bit;
188 	else
189 		daint &= ~bit;
190 	dwc2_writel(daint, hsotg->regs + DAINTMSK);
191 	local_irq_restore(flags);
192 }
193 
194 /**
195  * dwc2_hsotg_init_fifo - initialise non-periodic FIFOs
196  * @hsotg: The device instance.
197  */
198 static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg)
199 {
200 	unsigned int ep;
201 	unsigned int addr;
202 	int timeout;
203 	u32 val;
204 	u32 *txfsz = hsotg->params.g_tx_fifo_size;
205 
206 	/* Reset fifo map if not correctly cleared during previous session */
207 	WARN_ON(hsotg->fifo_map);
208 	hsotg->fifo_map = 0;
209 
210 	/* set RX/NPTX FIFO sizes */
211 	dwc2_writel(hsotg->params.g_rx_fifo_size, hsotg->regs + GRXFSIZ);
212 	dwc2_writel((hsotg->params.g_rx_fifo_size << FIFOSIZE_STARTADDR_SHIFT) |
213 		    (hsotg->params.g_np_tx_fifo_size << FIFOSIZE_DEPTH_SHIFT),
214 		    hsotg->regs + GNPTXFSIZ);
215 
216 	/*
217 	 * arange all the rest of the TX FIFOs, as some versions of this
218 	 * block have overlapping default addresses. This also ensures
219 	 * that if the settings have been changed, then they are set to
220 	 * known values.
221 	 */
222 
223 	/* start at the end of the GNPTXFSIZ, rounded up */
224 	addr = hsotg->params.g_rx_fifo_size + hsotg->params.g_np_tx_fifo_size;
225 
226 	/*
227 	 * Configure fifos sizes from provided configuration and assign
228 	 * them to endpoints dynamically according to maxpacket size value of
229 	 * given endpoint.
230 	 */
231 	for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) {
232 		if (!txfsz[ep])
233 			continue;
234 		val = addr;
235 		val |= txfsz[ep] << FIFOSIZE_DEPTH_SHIFT;
236 		WARN_ONCE(addr + txfsz[ep] > hsotg->fifo_mem,
237 			  "insufficient fifo memory");
238 		addr += txfsz[ep];
239 
240 		dwc2_writel(val, hsotg->regs + DPTXFSIZN(ep));
241 		val = dwc2_readl(hsotg->regs + DPTXFSIZN(ep));
242 	}
243 
244 	/*
245 	 * according to p428 of the design guide, we need to ensure that
246 	 * all fifos are flushed before continuing
247 	 */
248 
249 	dwc2_writel(GRSTCTL_TXFNUM(0x10) | GRSTCTL_TXFFLSH |
250 	       GRSTCTL_RXFFLSH, hsotg->regs + GRSTCTL);
251 
252 	/* wait until the fifos are both flushed */
253 	timeout = 100;
254 	while (1) {
255 		val = dwc2_readl(hsotg->regs + GRSTCTL);
256 
257 		if ((val & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)) == 0)
258 			break;
259 
260 		if (--timeout == 0) {
261 			dev_err(hsotg->dev,
262 				"%s: timeout flushing fifos (GRSTCTL=%08x)\n",
263 				__func__, val);
264 			break;
265 		}
266 
267 		udelay(1);
268 	}
269 
270 	dev_dbg(hsotg->dev, "FIFOs reset, timeout at %d\n", timeout);
271 }
272 
273 /**
274  * @ep: USB endpoint to allocate request for.
275  * @flags: Allocation flags
276  *
277  * Allocate a new USB request structure appropriate for the specified endpoint
278  */
279 static struct usb_request *dwc2_hsotg_ep_alloc_request(struct usb_ep *ep,
280 						      gfp_t flags)
281 {
282 	struct dwc2_hsotg_req *req;
283 
284 	req = kzalloc(sizeof(struct dwc2_hsotg_req), flags);
285 	if (!req)
286 		return NULL;
287 
288 	INIT_LIST_HEAD(&req->queue);
289 
290 	return &req->req;
291 }
292 
293 /**
294  * is_ep_periodic - return true if the endpoint is in periodic mode.
295  * @hs_ep: The endpoint to query.
296  *
297  * Returns true if the endpoint is in periodic mode, meaning it is being
298  * used for an Interrupt or ISO transfer.
299  */
300 static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep)
301 {
302 	return hs_ep->periodic;
303 }
304 
305 /**
306  * dwc2_hsotg_unmap_dma - unmap the DMA memory being used for the request
307  * @hsotg: The device state.
308  * @hs_ep: The endpoint for the request
309  * @hs_req: The request being processed.
310  *
311  * This is the reverse of dwc2_hsotg_map_dma(), called for the completion
312  * of a request to ensure the buffer is ready for access by the caller.
313  */
314 static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg,
315 				struct dwc2_hsotg_ep *hs_ep,
316 				struct dwc2_hsotg_req *hs_req)
317 {
318 	struct usb_request *req = &hs_req->req;
319 	usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in);
320 }
321 
322 /*
323  * dwc2_gadget_alloc_ctrl_desc_chains - allocate DMA descriptor chains
324  * for Control endpoint
325  * @hsotg: The device state.
326  *
327  * This function will allocate 4 descriptor chains for EP 0: 2 for
328  * Setup stage, per one for IN and OUT data/status transactions.
329  */
330 static int dwc2_gadget_alloc_ctrl_desc_chains(struct dwc2_hsotg *hsotg)
331 {
332 	hsotg->setup_desc[0] =
333 		dmam_alloc_coherent(hsotg->dev,
334 				    sizeof(struct dwc2_dma_desc),
335 				    &hsotg->setup_desc_dma[0],
336 				    GFP_KERNEL);
337 	if (!hsotg->setup_desc[0])
338 		goto fail;
339 
340 	hsotg->setup_desc[1] =
341 		dmam_alloc_coherent(hsotg->dev,
342 				    sizeof(struct dwc2_dma_desc),
343 				    &hsotg->setup_desc_dma[1],
344 				    GFP_KERNEL);
345 	if (!hsotg->setup_desc[1])
346 		goto fail;
347 
348 	hsotg->ctrl_in_desc =
349 		dmam_alloc_coherent(hsotg->dev,
350 				    sizeof(struct dwc2_dma_desc),
351 				    &hsotg->ctrl_in_desc_dma,
352 				    GFP_KERNEL);
353 	if (!hsotg->ctrl_in_desc)
354 		goto fail;
355 
356 	hsotg->ctrl_out_desc =
357 		dmam_alloc_coherent(hsotg->dev,
358 				    sizeof(struct dwc2_dma_desc),
359 				    &hsotg->ctrl_out_desc_dma,
360 				    GFP_KERNEL);
361 	if (!hsotg->ctrl_out_desc)
362 		goto fail;
363 
364 	return 0;
365 
366 fail:
367 	return -ENOMEM;
368 }
369 
370 /**
371  * dwc2_hsotg_write_fifo - write packet Data to the TxFIFO
372  * @hsotg: The controller state.
373  * @hs_ep: The endpoint we're going to write for.
374  * @hs_req: The request to write data for.
375  *
376  * This is called when the TxFIFO has some space in it to hold a new
377  * transmission and we have something to give it. The actual setup of
378  * the data size is done elsewhere, so all we have to do is to actually
379  * write the data.
380  *
381  * The return value is zero if there is more space (or nothing was done)
382  * otherwise -ENOSPC is returned if the FIFO space was used up.
383  *
384  * This routine is only needed for PIO
385  */
386 static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg,
387 				struct dwc2_hsotg_ep *hs_ep,
388 				struct dwc2_hsotg_req *hs_req)
389 {
390 	bool periodic = is_ep_periodic(hs_ep);
391 	u32 gnptxsts = dwc2_readl(hsotg->regs + GNPTXSTS);
392 	int buf_pos = hs_req->req.actual;
393 	int to_write = hs_ep->size_loaded;
394 	void *data;
395 	int can_write;
396 	int pkt_round;
397 	int max_transfer;
398 
399 	to_write -= (buf_pos - hs_ep->last_load);
400 
401 	/* if there's nothing to write, get out early */
402 	if (to_write == 0)
403 		return 0;
404 
405 	if (periodic && !hsotg->dedicated_fifos) {
406 		u32 epsize = dwc2_readl(hsotg->regs + DIEPTSIZ(hs_ep->index));
407 		int size_left;
408 		int size_done;
409 
410 		/*
411 		 * work out how much data was loaded so we can calculate
412 		 * how much data is left in the fifo.
413 		 */
414 
415 		size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
416 
417 		/*
418 		 * if shared fifo, we cannot write anything until the
419 		 * previous data has been completely sent.
420 		 */
421 		if (hs_ep->fifo_load != 0) {
422 			dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
423 			return -ENOSPC;
424 		}
425 
426 		dev_dbg(hsotg->dev, "%s: left=%d, load=%d, fifo=%d, size %d\n",
427 			__func__, size_left,
428 			hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size);
429 
430 		/* how much of the data has moved */
431 		size_done = hs_ep->size_loaded - size_left;
432 
433 		/* how much data is left in the fifo */
434 		can_write = hs_ep->fifo_load - size_done;
435 		dev_dbg(hsotg->dev, "%s: => can_write1=%d\n",
436 			__func__, can_write);
437 
438 		can_write = hs_ep->fifo_size - can_write;
439 		dev_dbg(hsotg->dev, "%s: => can_write2=%d\n",
440 			__func__, can_write);
441 
442 		if (can_write <= 0) {
443 			dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP);
444 			return -ENOSPC;
445 		}
446 	} else if (hsotg->dedicated_fifos && hs_ep->index != 0) {
447 		can_write = dwc2_readl(hsotg->regs +
448 				DTXFSTS(hs_ep->fifo_index));
449 
450 		can_write &= 0xffff;
451 		can_write *= 4;
452 	} else {
453 		if (GNPTXSTS_NP_TXQ_SPC_AVAIL_GET(gnptxsts) == 0) {
454 			dev_dbg(hsotg->dev,
455 				"%s: no queue slots available (0x%08x)\n",
456 				__func__, gnptxsts);
457 
458 			dwc2_hsotg_en_gsint(hsotg, GINTSTS_NPTXFEMP);
459 			return -ENOSPC;
460 		}
461 
462 		can_write = GNPTXSTS_NP_TXF_SPC_AVAIL_GET(gnptxsts);
463 		can_write *= 4;	/* fifo size is in 32bit quantities. */
464 	}
465 
466 	max_transfer = hs_ep->ep.maxpacket * hs_ep->mc;
467 
468 	dev_dbg(hsotg->dev, "%s: GNPTXSTS=%08x, can=%d, to=%d, max_transfer %d\n",
469 		 __func__, gnptxsts, can_write, to_write, max_transfer);
470 
471 	/*
472 	 * limit to 512 bytes of data, it seems at least on the non-periodic
473 	 * FIFO, requests of >512 cause the endpoint to get stuck with a
474 	 * fragment of the end of the transfer in it.
475 	 */
476 	if (can_write > 512 && !periodic)
477 		can_write = 512;
478 
479 	/*
480 	 * limit the write to one max-packet size worth of data, but allow
481 	 * the transfer to return that it did not run out of fifo space
482 	 * doing it.
483 	 */
484 	if (to_write > max_transfer) {
485 		to_write = max_transfer;
486 
487 		/* it's needed only when we do not use dedicated fifos */
488 		if (!hsotg->dedicated_fifos)
489 			dwc2_hsotg_en_gsint(hsotg,
490 					   periodic ? GINTSTS_PTXFEMP :
491 					   GINTSTS_NPTXFEMP);
492 	}
493 
494 	/* see if we can write data */
495 
496 	if (to_write > can_write) {
497 		to_write = can_write;
498 		pkt_round = to_write % max_transfer;
499 
500 		/*
501 		 * Round the write down to an
502 		 * exact number of packets.
503 		 *
504 		 * Note, we do not currently check to see if we can ever
505 		 * write a full packet or not to the FIFO.
506 		 */
507 
508 		if (pkt_round)
509 			to_write -= pkt_round;
510 
511 		/*
512 		 * enable correct FIFO interrupt to alert us when there
513 		 * is more room left.
514 		 */
515 
516 		/* it's needed only when we do not use dedicated fifos */
517 		if (!hsotg->dedicated_fifos)
518 			dwc2_hsotg_en_gsint(hsotg,
519 					   periodic ? GINTSTS_PTXFEMP :
520 					   GINTSTS_NPTXFEMP);
521 	}
522 
523 	dev_dbg(hsotg->dev, "write %d/%d, can_write %d, done %d\n",
524 		 to_write, hs_req->req.length, can_write, buf_pos);
525 
526 	if (to_write <= 0)
527 		return -ENOSPC;
528 
529 	hs_req->req.actual = buf_pos + to_write;
530 	hs_ep->total_data += to_write;
531 
532 	if (periodic)
533 		hs_ep->fifo_load += to_write;
534 
535 	to_write = DIV_ROUND_UP(to_write, 4);
536 	data = hs_req->req.buf + buf_pos;
537 
538 	iowrite32_rep(hsotg->regs + EPFIFO(hs_ep->index), data, to_write);
539 
540 	return (to_write >= can_write) ? -ENOSPC : 0;
541 }
542 
543 /**
544  * get_ep_limit - get the maximum data legnth for this endpoint
545  * @hs_ep: The endpoint
546  *
547  * Return the maximum data that can be queued in one go on a given endpoint
548  * so that transfers that are too long can be split.
549  */
550 static unsigned get_ep_limit(struct dwc2_hsotg_ep *hs_ep)
551 {
552 	int index = hs_ep->index;
553 	unsigned maxsize;
554 	unsigned maxpkt;
555 
556 	if (index != 0) {
557 		maxsize = DXEPTSIZ_XFERSIZE_LIMIT + 1;
558 		maxpkt = DXEPTSIZ_PKTCNT_LIMIT + 1;
559 	} else {
560 		maxsize = 64+64;
561 		if (hs_ep->dir_in)
562 			maxpkt = DIEPTSIZ0_PKTCNT_LIMIT + 1;
563 		else
564 			maxpkt = 2;
565 	}
566 
567 	/* we made the constant loading easier above by using +1 */
568 	maxpkt--;
569 	maxsize--;
570 
571 	/*
572 	 * constrain by packet count if maxpkts*pktsize is greater
573 	 * than the length register size.
574 	 */
575 
576 	if ((maxpkt * hs_ep->ep.maxpacket) < maxsize)
577 		maxsize = maxpkt * hs_ep->ep.maxpacket;
578 
579 	return maxsize;
580 }
581 
582 /**
583 * dwc2_hsotg_read_frameno - read current frame number
584 * @hsotg: The device instance
585 *
586 * Return the current frame number
587 */
588 static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg)
589 {
590 	u32 dsts;
591 
592 	dsts = dwc2_readl(hsotg->regs + DSTS);
593 	dsts &= DSTS_SOFFN_MASK;
594 	dsts >>= DSTS_SOFFN_SHIFT;
595 
596 	return dsts;
597 }
598 
599 /**
600  * dwc2_gadget_get_chain_limit - get the maximum data payload value of the
601  * DMA descriptor chain prepared for specific endpoint
602  * @hs_ep: The endpoint
603  *
604  * Return the maximum data that can be queued in one go on a given endpoint
605  * depending on its descriptor chain capacity so that transfers that
606  * are too long can be split.
607  */
608 static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep)
609 {
610 	int is_isoc = hs_ep->isochronous;
611 	unsigned int maxsize;
612 
613 	if (is_isoc)
614 		maxsize = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT :
615 					   DEV_DMA_ISOC_RX_NBYTES_LIMIT;
616 	else
617 		maxsize = DEV_DMA_NBYTES_LIMIT;
618 
619 	/* Above size of one descriptor was chosen, multiple it */
620 	maxsize *= MAX_DMA_DESC_NUM_GENERIC;
621 
622 	return maxsize;
623 }
624 
625 /*
626  * dwc2_gadget_get_desc_params - get DMA descriptor parameters.
627  * @hs_ep: The endpoint
628  * @mask: RX/TX bytes mask to be defined
629  *
630  * Returns maximum data payload for one descriptor after analyzing endpoint
631  * characteristics.
632  * DMA descriptor transfer bytes limit depends on EP type:
633  * Control out - MPS,
634  * Isochronous - descriptor rx/tx bytes bitfield limit,
635  * Control In/Bulk/Interrupt - multiple of mps. This will allow to not
636  * have concatenations from various descriptors within one packet.
637  *
638  * Selects corresponding mask for RX/TX bytes as well.
639  */
640 static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask)
641 {
642 	u32 mps = hs_ep->ep.maxpacket;
643 	int dir_in = hs_ep->dir_in;
644 	u32 desc_size = 0;
645 
646 	if (!hs_ep->index && !dir_in) {
647 		desc_size = mps;
648 		*mask = DEV_DMA_NBYTES_MASK;
649 	} else if (hs_ep->isochronous) {
650 		if (dir_in) {
651 			desc_size = DEV_DMA_ISOC_TX_NBYTES_LIMIT;
652 			*mask = DEV_DMA_ISOC_TX_NBYTES_MASK;
653 		} else {
654 			desc_size = DEV_DMA_ISOC_RX_NBYTES_LIMIT;
655 			*mask = DEV_DMA_ISOC_RX_NBYTES_MASK;
656 		}
657 	} else {
658 		desc_size = DEV_DMA_NBYTES_LIMIT;
659 		*mask = DEV_DMA_NBYTES_MASK;
660 
661 		/* Round down desc_size to be mps multiple */
662 		desc_size -= desc_size % mps;
663 	}
664 
665 	return desc_size;
666 }
667 
668 /*
669  * dwc2_gadget_config_nonisoc_xfer_ddma - prepare non ISOC DMA desc chain.
670  * @hs_ep: The endpoint
671  * @dma_buff: DMA address to use
672  * @len: Length of the transfer
673  *
674  * This function will iterate over descriptor chain and fill its entries
675  * with corresponding information based on transfer data.
676  */
677 static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep,
678 						 dma_addr_t dma_buff,
679 						 unsigned int len)
680 {
681 	struct dwc2_hsotg *hsotg = hs_ep->parent;
682 	int dir_in = hs_ep->dir_in;
683 	struct dwc2_dma_desc *desc = hs_ep->desc_list;
684 	u32 mps = hs_ep->ep.maxpacket;
685 	u32 maxsize = 0;
686 	u32 offset = 0;
687 	u32 mask = 0;
688 	int i;
689 
690 	maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
691 
692 	hs_ep->desc_count = (len / maxsize) +
693 				((len % maxsize) ? 1 : 0);
694 	if (len == 0)
695 		hs_ep->desc_count = 1;
696 
697 	for (i = 0; i < hs_ep->desc_count; ++i) {
698 		desc->status = 0;
699 		desc->status |= (DEV_DMA_BUFF_STS_HBUSY
700 				 << DEV_DMA_BUFF_STS_SHIFT);
701 
702 		if (len > maxsize) {
703 			if (!hs_ep->index && !dir_in)
704 				desc->status |= (DEV_DMA_L | DEV_DMA_IOC);
705 
706 			desc->status |= (maxsize <<
707 						DEV_DMA_NBYTES_SHIFT & mask);
708 			desc->buf = dma_buff + offset;
709 
710 			len -= maxsize;
711 			offset += maxsize;
712 		} else {
713 			desc->status |= (DEV_DMA_L | DEV_DMA_IOC);
714 
715 			if (dir_in)
716 				desc->status |= (len % mps) ? DEV_DMA_SHORT :
717 					((hs_ep->send_zlp) ? DEV_DMA_SHORT : 0);
718 			if (len > maxsize)
719 				dev_err(hsotg->dev, "wrong len %d\n", len);
720 
721 			desc->status |=
722 				len << DEV_DMA_NBYTES_SHIFT & mask;
723 			desc->buf = dma_buff + offset;
724 		}
725 
726 		desc->status &= ~DEV_DMA_BUFF_STS_MASK;
727 		desc->status |= (DEV_DMA_BUFF_STS_HREADY
728 				 << DEV_DMA_BUFF_STS_SHIFT);
729 		desc++;
730 	}
731 }
732 
733 /*
734  * dwc2_gadget_fill_isoc_desc - fills next isochronous descriptor in chain.
735  * @hs_ep: The isochronous endpoint.
736  * @dma_buff: usb requests dma buffer.
737  * @len: usb request transfer length.
738  *
739  * Finds out index of first free entry either in the bottom or up half of
740  * descriptor chain depend on which is under SW control and not processed
741  * by HW. Then fills that descriptor with the data of the arrived usb request,
742  * frame info, sets Last and IOC bits increments next_desc. If filled
743  * descriptor is not the first one, removes L bit from the previous descriptor
744  * status.
745  */
746 static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep,
747 				      dma_addr_t dma_buff, unsigned int len)
748 {
749 	struct dwc2_dma_desc *desc;
750 	struct dwc2_hsotg *hsotg = hs_ep->parent;
751 	u32 index;
752 	u32 maxsize = 0;
753 	u32 mask = 0;
754 
755 	maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask);
756 	if (len > maxsize) {
757 		dev_err(hsotg->dev, "wrong len %d\n", len);
758 		return -EINVAL;
759 	}
760 
761 	/*
762 	 * If SW has already filled half of chain, then return and wait for
763 	 * the other chain to be processed by HW.
764 	 */
765 	if (hs_ep->next_desc == MAX_DMA_DESC_NUM_GENERIC / 2)
766 		return -EBUSY;
767 
768 	/* Increment frame number by interval for IN */
769 	if (hs_ep->dir_in)
770 		dwc2_gadget_incr_frame_num(hs_ep);
771 
772 	index = (MAX_DMA_DESC_NUM_GENERIC / 2) * hs_ep->isoc_chain_num +
773 		 hs_ep->next_desc;
774 
775 	/* Sanity check of calculated index */
776 	if ((hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC) ||
777 	    (!hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC / 2)) {
778 		dev_err(hsotg->dev, "wrong index %d for iso chain\n", index);
779 		return -EINVAL;
780 	}
781 
782 	desc = &hs_ep->desc_list[index];
783 
784 	/* Clear L bit of previous desc if more than one entries in the chain */
785 	if (hs_ep->next_desc)
786 		hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L;
787 
788 	dev_dbg(hsotg->dev, "%s: Filling ep %d, dir %s isoc desc # %d\n",
789 		__func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index);
790 
791 	desc->status = 0;
792 	desc->status |= (DEV_DMA_BUFF_STS_HBUSY	<< DEV_DMA_BUFF_STS_SHIFT);
793 
794 	desc->buf = dma_buff;
795 	desc->status |= (DEV_DMA_L | DEV_DMA_IOC |
796 			 ((len << DEV_DMA_NBYTES_SHIFT) & mask));
797 
798 	if (hs_ep->dir_in) {
799 		desc->status |= ((hs_ep->mc << DEV_DMA_ISOC_PID_SHIFT) &
800 				 DEV_DMA_ISOC_PID_MASK) |
801 				((len % hs_ep->ep.maxpacket) ?
802 				 DEV_DMA_SHORT : 0) |
803 				((hs_ep->target_frame <<
804 				  DEV_DMA_ISOC_FRNUM_SHIFT) &
805 				 DEV_DMA_ISOC_FRNUM_MASK);
806 	}
807 
808 	desc->status &= ~DEV_DMA_BUFF_STS_MASK;
809 	desc->status |= (DEV_DMA_BUFF_STS_HREADY << DEV_DMA_BUFF_STS_SHIFT);
810 
811 	/* Update index of last configured entry in the chain */
812 	hs_ep->next_desc++;
813 
814 	return 0;
815 }
816 
817 /*
818  * dwc2_gadget_start_isoc_ddma - start isochronous transfer in DDMA
819  * @hs_ep: The isochronous endpoint.
820  *
821  * Prepare first descriptor chain for isochronous endpoints. Afterwards
822  * write DMA address to HW and enable the endpoint.
823  *
824  * Switch between descriptor chains via isoc_chain_num to give SW opportunity
825  * to prepare second descriptor chain while first one is being processed by HW.
826  */
827 static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
828 {
829 	struct dwc2_hsotg *hsotg = hs_ep->parent;
830 	struct dwc2_hsotg_req *hs_req, *treq;
831 	int index = hs_ep->index;
832 	int ret;
833 	u32 dma_reg;
834 	u32 depctl;
835 	u32 ctrl;
836 
837 	if (list_empty(&hs_ep->queue)) {
838 		dev_dbg(hsotg->dev, "%s: No requests in queue\n", __func__);
839 		return;
840 	}
841 
842 	list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) {
843 		ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma,
844 						 hs_req->req.length);
845 		if (ret) {
846 			dev_dbg(hsotg->dev, "%s: desc chain full\n", __func__);
847 			break;
848 		}
849 	}
850 
851 	depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
852 	dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index);
853 
854 	/* write descriptor chain address to control register */
855 	dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg);
856 
857 	ctrl = dwc2_readl(hsotg->regs + depctl);
858 	ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK;
859 	dwc2_writel(ctrl, hsotg->regs + depctl);
860 
861 	/* Switch ISOC descriptor chain number being processed by SW*/
862 	hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1;
863 	hs_ep->next_desc = 0;
864 }
865 
866 /**
867  * dwc2_hsotg_start_req - start a USB request from an endpoint's queue
868  * @hsotg: The controller state.
869  * @hs_ep: The endpoint to process a request for
870  * @hs_req: The request to start.
871  * @continuing: True if we are doing more for the current request.
872  *
873  * Start the given request running by setting the endpoint registers
874  * appropriately, and writing any data to the FIFOs.
875  */
876 static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg,
877 				struct dwc2_hsotg_ep *hs_ep,
878 				struct dwc2_hsotg_req *hs_req,
879 				bool continuing)
880 {
881 	struct usb_request *ureq = &hs_req->req;
882 	int index = hs_ep->index;
883 	int dir_in = hs_ep->dir_in;
884 	u32 epctrl_reg;
885 	u32 epsize_reg;
886 	u32 epsize;
887 	u32 ctrl;
888 	unsigned length;
889 	unsigned packets;
890 	unsigned maxreq;
891 	unsigned int dma_reg;
892 
893 	if (index != 0) {
894 		if (hs_ep->req && !continuing) {
895 			dev_err(hsotg->dev, "%s: active request\n", __func__);
896 			WARN_ON(1);
897 			return;
898 		} else if (hs_ep->req != hs_req && continuing) {
899 			dev_err(hsotg->dev,
900 				"%s: continue different req\n", __func__);
901 			WARN_ON(1);
902 			return;
903 		}
904 	}
905 
906 	dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index);
907 	epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
908 	epsize_reg = dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
909 
910 	dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x, ep %d, dir %s\n",
911 		__func__, dwc2_readl(hsotg->regs + epctrl_reg), index,
912 		hs_ep->dir_in ? "in" : "out");
913 
914 	/* If endpoint is stalled, we will restart request later */
915 	ctrl = dwc2_readl(hsotg->regs + epctrl_reg);
916 
917 	if (index && ctrl & DXEPCTL_STALL) {
918 		dev_warn(hsotg->dev, "%s: ep%d is stalled\n", __func__, index);
919 		return;
920 	}
921 
922 	length = ureq->length - ureq->actual;
923 	dev_dbg(hsotg->dev, "ureq->length:%d ureq->actual:%d\n",
924 		ureq->length, ureq->actual);
925 
926 	if (!using_desc_dma(hsotg))
927 		maxreq = get_ep_limit(hs_ep);
928 	else
929 		maxreq = dwc2_gadget_get_chain_limit(hs_ep);
930 
931 	if (length > maxreq) {
932 		int round = maxreq % hs_ep->ep.maxpacket;
933 
934 		dev_dbg(hsotg->dev, "%s: length %d, max-req %d, r %d\n",
935 			__func__, length, maxreq, round);
936 
937 		/* round down to multiple of packets */
938 		if (round)
939 			maxreq -= round;
940 
941 		length = maxreq;
942 	}
943 
944 	if (length)
945 		packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket);
946 	else
947 		packets = 1;	/* send one packet if length is zero. */
948 
949 	if (hs_ep->isochronous && length > (hs_ep->mc * hs_ep->ep.maxpacket)) {
950 		dev_err(hsotg->dev, "req length > maxpacket*mc\n");
951 		return;
952 	}
953 
954 	if (dir_in && index != 0)
955 		if (hs_ep->isochronous)
956 			epsize = DXEPTSIZ_MC(packets);
957 		else
958 			epsize = DXEPTSIZ_MC(1);
959 	else
960 		epsize = 0;
961 
962 	/*
963 	 * zero length packet should be programmed on its own and should not
964 	 * be counted in DIEPTSIZ.PktCnt with other packets.
965 	 */
966 	if (dir_in && ureq->zero && !continuing) {
967 		/* Test if zlp is actually required. */
968 		if ((ureq->length >= hs_ep->ep.maxpacket) &&
969 					!(ureq->length % hs_ep->ep.maxpacket))
970 			hs_ep->send_zlp = 1;
971 	}
972 
973 	epsize |= DXEPTSIZ_PKTCNT(packets);
974 	epsize |= DXEPTSIZ_XFERSIZE(length);
975 
976 	dev_dbg(hsotg->dev, "%s: %d@%d/%d, 0x%08x => 0x%08x\n",
977 		__func__, packets, length, ureq->length, epsize, epsize_reg);
978 
979 	/* store the request as the current one we're doing */
980 	hs_ep->req = hs_req;
981 
982 	if (using_desc_dma(hsotg)) {
983 		u32 offset = 0;
984 		u32 mps = hs_ep->ep.maxpacket;
985 
986 		/* Adjust length: EP0 - MPS, other OUT EPs - multiple of MPS */
987 		if (!dir_in) {
988 			if (!index)
989 				length = mps;
990 			else if (length % mps)
991 				length += (mps - (length % mps));
992 		}
993 
994 		/*
995 		 * If more data to send, adjust DMA for EP0 out data stage.
996 		 * ureq->dma stays unchanged, hence increment it by already
997 		 * passed passed data count before starting new transaction.
998 		 */
999 		if (!index && hsotg->ep0_state == DWC2_EP0_DATA_OUT &&
1000 		    continuing)
1001 			offset = ureq->actual;
1002 
1003 		/* Fill DDMA chain entries */
1004 		dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset,
1005 						     length);
1006 
1007 		/* write descriptor chain address to control register */
1008 		dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg);
1009 
1010 		dev_dbg(hsotg->dev, "%s: %08x pad => 0x%08x\n",
1011 			__func__, (u32)hs_ep->desc_list_dma, dma_reg);
1012 	} else {
1013 		/* write size / packets */
1014 		dwc2_writel(epsize, hsotg->regs + epsize_reg);
1015 
1016 		if (using_dma(hsotg) && !continuing && (length != 0)) {
1017 			/*
1018 			 * write DMA address to control register, buffer
1019 			 * already synced by dwc2_hsotg_ep_queue().
1020 			 */
1021 
1022 			dwc2_writel(ureq->dma, hsotg->regs + dma_reg);
1023 
1024 			dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n",
1025 				__func__, &ureq->dma, dma_reg);
1026 		}
1027 	}
1028 
1029 	if (hs_ep->isochronous && hs_ep->interval == 1) {
1030 		hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg);
1031 		dwc2_gadget_incr_frame_num(hs_ep);
1032 
1033 		if (hs_ep->target_frame & 0x1)
1034 			ctrl |= DXEPCTL_SETODDFR;
1035 		else
1036 			ctrl |= DXEPCTL_SETEVENFR;
1037 	}
1038 
1039 	ctrl |= DXEPCTL_EPENA;	/* ensure ep enabled */
1040 
1041 	dev_dbg(hsotg->dev, "ep0 state:%d\n", hsotg->ep0_state);
1042 
1043 	/* For Setup request do not clear NAK */
1044 	if (!(index == 0 && hsotg->ep0_state == DWC2_EP0_SETUP))
1045 		ctrl |= DXEPCTL_CNAK;	/* clear NAK set by core */
1046 
1047 	dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
1048 	dwc2_writel(ctrl, hsotg->regs + epctrl_reg);
1049 
1050 	/*
1051 	 * set these, it seems that DMA support increments past the end
1052 	 * of the packet buffer so we need to calculate the length from
1053 	 * this information.
1054 	 */
1055 	hs_ep->size_loaded = length;
1056 	hs_ep->last_load = ureq->actual;
1057 
1058 	if (dir_in && !using_dma(hsotg)) {
1059 		/* set these anyway, we may need them for non-periodic in */
1060 		hs_ep->fifo_load = 0;
1061 
1062 		dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
1063 	}
1064 
1065 	/*
1066 	 * Note, trying to clear the NAK here causes problems with transmit
1067 	 * on the S3C6400 ending up with the TXFIFO becoming full.
1068 	 */
1069 
1070 	/* check ep is enabled */
1071 	if (!(dwc2_readl(hsotg->regs + epctrl_reg) & DXEPCTL_EPENA))
1072 		dev_dbg(hsotg->dev,
1073 			 "ep%d: failed to become enabled (DXEPCTL=0x%08x)?\n",
1074 			 index, dwc2_readl(hsotg->regs + epctrl_reg));
1075 
1076 	dev_dbg(hsotg->dev, "%s: DXEPCTL=0x%08x\n",
1077 		__func__, dwc2_readl(hsotg->regs + epctrl_reg));
1078 
1079 	/* enable ep interrupts */
1080 	dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1);
1081 }
1082 
1083 /**
1084  * dwc2_hsotg_map_dma - map the DMA memory being used for the request
1085  * @hsotg: The device state.
1086  * @hs_ep: The endpoint the request is on.
1087  * @req: The request being processed.
1088  *
1089  * We've been asked to queue a request, so ensure that the memory buffer
1090  * is correctly setup for DMA. If we've been passed an extant DMA address
1091  * then ensure the buffer has been synced to memory. If our buffer has no
1092  * DMA memory, then we map the memory and mark our request to allow us to
1093  * cleanup on completion.
1094  */
1095 static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg,
1096 			     struct dwc2_hsotg_ep *hs_ep,
1097 			     struct usb_request *req)
1098 {
1099 	int ret;
1100 
1101 	ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in);
1102 	if (ret)
1103 		goto dma_error;
1104 
1105 	return 0;
1106 
1107 dma_error:
1108 	dev_err(hsotg->dev, "%s: failed to map buffer %p, %d bytes\n",
1109 		__func__, req->buf, req->length);
1110 
1111 	return -EIO;
1112 }
1113 
1114 static int dwc2_hsotg_handle_unaligned_buf_start(struct dwc2_hsotg *hsotg,
1115 	struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req)
1116 {
1117 	void *req_buf = hs_req->req.buf;
1118 
1119 	/* If dma is not being used or buffer is aligned */
1120 	if (!using_dma(hsotg) || !((long)req_buf & 3))
1121 		return 0;
1122 
1123 	WARN_ON(hs_req->saved_req_buf);
1124 
1125 	dev_dbg(hsotg->dev, "%s: %s: buf=%p length=%d\n", __func__,
1126 			hs_ep->ep.name, req_buf, hs_req->req.length);
1127 
1128 	hs_req->req.buf = kmalloc(hs_req->req.length, GFP_ATOMIC);
1129 	if (!hs_req->req.buf) {
1130 		hs_req->req.buf = req_buf;
1131 		dev_err(hsotg->dev,
1132 			"%s: unable to allocate memory for bounce buffer\n",
1133 			__func__);
1134 		return -ENOMEM;
1135 	}
1136 
1137 	/* Save actual buffer */
1138 	hs_req->saved_req_buf = req_buf;
1139 
1140 	if (hs_ep->dir_in)
1141 		memcpy(hs_req->req.buf, req_buf, hs_req->req.length);
1142 	return 0;
1143 }
1144 
1145 static void dwc2_hsotg_handle_unaligned_buf_complete(struct dwc2_hsotg *hsotg,
1146 	struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req)
1147 {
1148 	/* If dma is not being used or buffer was aligned */
1149 	if (!using_dma(hsotg) || !hs_req->saved_req_buf)
1150 		return;
1151 
1152 	dev_dbg(hsotg->dev, "%s: %s: status=%d actual-length=%d\n", __func__,
1153 		hs_ep->ep.name, hs_req->req.status, hs_req->req.actual);
1154 
1155 	/* Copy data from bounce buffer on successful out transfer */
1156 	if (!hs_ep->dir_in && !hs_req->req.status)
1157 		memcpy(hs_req->saved_req_buf, hs_req->req.buf,
1158 							hs_req->req.actual);
1159 
1160 	/* Free bounce buffer */
1161 	kfree(hs_req->req.buf);
1162 
1163 	hs_req->req.buf = hs_req->saved_req_buf;
1164 	hs_req->saved_req_buf = NULL;
1165 }
1166 
1167 /**
1168  * dwc2_gadget_target_frame_elapsed - Checks target frame
1169  * @hs_ep: The driver endpoint to check
1170  *
1171  * Returns 1 if targeted frame elapsed. If returned 1 then we need to drop
1172  * corresponding transfer.
1173  */
1174 static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep)
1175 {
1176 	struct dwc2_hsotg *hsotg = hs_ep->parent;
1177 	u32 target_frame = hs_ep->target_frame;
1178 	u32 current_frame = dwc2_hsotg_read_frameno(hsotg);
1179 	bool frame_overrun = hs_ep->frame_overrun;
1180 
1181 	if (!frame_overrun && current_frame >= target_frame)
1182 		return true;
1183 
1184 	if (frame_overrun && current_frame >= target_frame &&
1185 	    ((current_frame - target_frame) < DSTS_SOFFN_LIMIT / 2))
1186 		return true;
1187 
1188 	return false;
1189 }
1190 
1191 /*
1192  * dwc2_gadget_set_ep0_desc_chain - Set EP's desc chain pointers
1193  * @hsotg: The driver state
1194  * @hs_ep: the ep descriptor chain is for
1195  *
1196  * Called to update EP0 structure's pointers depend on stage of
1197  * control transfer.
1198  */
1199 static int dwc2_gadget_set_ep0_desc_chain(struct dwc2_hsotg *hsotg,
1200 					  struct dwc2_hsotg_ep *hs_ep)
1201 {
1202 	switch (hsotg->ep0_state) {
1203 	case DWC2_EP0_SETUP:
1204 	case DWC2_EP0_STATUS_OUT:
1205 		hs_ep->desc_list = hsotg->setup_desc[0];
1206 		hs_ep->desc_list_dma = hsotg->setup_desc_dma[0];
1207 		break;
1208 	case DWC2_EP0_DATA_IN:
1209 	case DWC2_EP0_STATUS_IN:
1210 		hs_ep->desc_list = hsotg->ctrl_in_desc;
1211 		hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma;
1212 		break;
1213 	case DWC2_EP0_DATA_OUT:
1214 		hs_ep->desc_list = hsotg->ctrl_out_desc;
1215 		hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma;
1216 		break;
1217 	default:
1218 		dev_err(hsotg->dev, "invalid EP 0 state in queue %d\n",
1219 			hsotg->ep0_state);
1220 		return -EINVAL;
1221 	}
1222 
1223 	return 0;
1224 }
1225 
1226 static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req,
1227 			      gfp_t gfp_flags)
1228 {
1229 	struct dwc2_hsotg_req *hs_req = our_req(req);
1230 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1231 	struct dwc2_hsotg *hs = hs_ep->parent;
1232 	bool first;
1233 	int ret;
1234 
1235 	dev_dbg(hs->dev, "%s: req %p: %d@%p, noi=%d, zero=%d, snok=%d\n",
1236 		ep->name, req, req->length, req->buf, req->no_interrupt,
1237 		req->zero, req->short_not_ok);
1238 
1239 	/* Prevent new request submission when controller is suspended */
1240 	if (hs->lx_state == DWC2_L2) {
1241 		dev_dbg(hs->dev, "%s: don't submit request while suspended\n",
1242 				__func__);
1243 		return -EAGAIN;
1244 	}
1245 
1246 	/* initialise status of the request */
1247 	INIT_LIST_HEAD(&hs_req->queue);
1248 	req->actual = 0;
1249 	req->status = -EINPROGRESS;
1250 
1251 	ret = dwc2_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req);
1252 	if (ret)
1253 		return ret;
1254 
1255 	/* if we're using DMA, sync the buffers as necessary */
1256 	if (using_dma(hs)) {
1257 		ret = dwc2_hsotg_map_dma(hs, hs_ep, req);
1258 		if (ret)
1259 			return ret;
1260 	}
1261 	/* If using descriptor DMA configure EP0 descriptor chain pointers */
1262 	if (using_desc_dma(hs) && !hs_ep->index) {
1263 		ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep);
1264 		if (ret)
1265 			return ret;
1266 	}
1267 
1268 	first = list_empty(&hs_ep->queue);
1269 	list_add_tail(&hs_req->queue, &hs_ep->queue);
1270 
1271 	/*
1272 	 * Handle DDMA isochronous transfers separately - just add new entry
1273 	 * to the half of descriptor chain that is not processed by HW.
1274 	 * Transfer will be started once SW gets either one of NAK or
1275 	 * OutTknEpDis interrupts.
1276 	 */
1277 	if (using_desc_dma(hs) && hs_ep->isochronous &&
1278 	    hs_ep->target_frame != TARGET_FRAME_INITIAL) {
1279 		ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma,
1280 						 hs_req->req.length);
1281 		if (ret)
1282 			dev_dbg(hs->dev, "%s: ISO desc chain full\n", __func__);
1283 
1284 		return 0;
1285 	}
1286 
1287 	if (first) {
1288 		if (!hs_ep->isochronous) {
1289 			dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
1290 			return 0;
1291 		}
1292 
1293 		while (dwc2_gadget_target_frame_elapsed(hs_ep))
1294 			dwc2_gadget_incr_frame_num(hs_ep);
1295 
1296 		if (hs_ep->target_frame != TARGET_FRAME_INITIAL)
1297 			dwc2_hsotg_start_req(hs, hs_ep, hs_req, false);
1298 	}
1299 	return 0;
1300 }
1301 
1302 static int dwc2_hsotg_ep_queue_lock(struct usb_ep *ep, struct usb_request *req,
1303 			      gfp_t gfp_flags)
1304 {
1305 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1306 	struct dwc2_hsotg *hs = hs_ep->parent;
1307 	unsigned long flags = 0;
1308 	int ret = 0;
1309 
1310 	spin_lock_irqsave(&hs->lock, flags);
1311 	ret = dwc2_hsotg_ep_queue(ep, req, gfp_flags);
1312 	spin_unlock_irqrestore(&hs->lock, flags);
1313 
1314 	return ret;
1315 }
1316 
1317 static void dwc2_hsotg_ep_free_request(struct usb_ep *ep,
1318 				      struct usb_request *req)
1319 {
1320 	struct dwc2_hsotg_req *hs_req = our_req(req);
1321 
1322 	kfree(hs_req);
1323 }
1324 
1325 /**
1326  * dwc2_hsotg_complete_oursetup - setup completion callback
1327  * @ep: The endpoint the request was on.
1328  * @req: The request completed.
1329  *
1330  * Called on completion of any requests the driver itself
1331  * submitted that need cleaning up.
1332  */
1333 static void dwc2_hsotg_complete_oursetup(struct usb_ep *ep,
1334 					struct usb_request *req)
1335 {
1336 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1337 	struct dwc2_hsotg *hsotg = hs_ep->parent;
1338 
1339 	dev_dbg(hsotg->dev, "%s: ep %p, req %p\n", __func__, ep, req);
1340 
1341 	dwc2_hsotg_ep_free_request(ep, req);
1342 }
1343 
1344 /**
1345  * ep_from_windex - convert control wIndex value to endpoint
1346  * @hsotg: The driver state.
1347  * @windex: The control request wIndex field (in host order).
1348  *
1349  * Convert the given wIndex into a pointer to an driver endpoint
1350  * structure, or return NULL if it is not a valid endpoint.
1351  */
1352 static struct dwc2_hsotg_ep *ep_from_windex(struct dwc2_hsotg *hsotg,
1353 					   u32 windex)
1354 {
1355 	struct dwc2_hsotg_ep *ep;
1356 	int dir = (windex & USB_DIR_IN) ? 1 : 0;
1357 	int idx = windex & 0x7F;
1358 
1359 	if (windex >= 0x100)
1360 		return NULL;
1361 
1362 	if (idx > hsotg->num_of_eps)
1363 		return NULL;
1364 
1365 	ep = index_to_ep(hsotg, idx, dir);
1366 
1367 	if (idx && ep->dir_in != dir)
1368 		return NULL;
1369 
1370 	return ep;
1371 }
1372 
1373 /**
1374  * dwc2_hsotg_set_test_mode - Enable usb Test Modes
1375  * @hsotg: The driver state.
1376  * @testmode: requested usb test mode
1377  * Enable usb Test Mode requested by the Host.
1378  */
1379 int dwc2_hsotg_set_test_mode(struct dwc2_hsotg *hsotg, int testmode)
1380 {
1381 	int dctl = dwc2_readl(hsotg->regs + DCTL);
1382 
1383 	dctl &= ~DCTL_TSTCTL_MASK;
1384 	switch (testmode) {
1385 	case TEST_J:
1386 	case TEST_K:
1387 	case TEST_SE0_NAK:
1388 	case TEST_PACKET:
1389 	case TEST_FORCE_EN:
1390 		dctl |= testmode << DCTL_TSTCTL_SHIFT;
1391 		break;
1392 	default:
1393 		return -EINVAL;
1394 	}
1395 	dwc2_writel(dctl, hsotg->regs + DCTL);
1396 	return 0;
1397 }
1398 
1399 /**
1400  * dwc2_hsotg_send_reply - send reply to control request
1401  * @hsotg: The device state
1402  * @ep: Endpoint 0
1403  * @buff: Buffer for request
1404  * @length: Length of reply.
1405  *
1406  * Create a request and queue it on the given endpoint. This is useful as
1407  * an internal method of sending replies to certain control requests, etc.
1408  */
1409 static int dwc2_hsotg_send_reply(struct dwc2_hsotg *hsotg,
1410 				struct dwc2_hsotg_ep *ep,
1411 				void *buff,
1412 				int length)
1413 {
1414 	struct usb_request *req;
1415 	int ret;
1416 
1417 	dev_dbg(hsotg->dev, "%s: buff %p, len %d\n", __func__, buff, length);
1418 
1419 	req = dwc2_hsotg_ep_alloc_request(&ep->ep, GFP_ATOMIC);
1420 	hsotg->ep0_reply = req;
1421 	if (!req) {
1422 		dev_warn(hsotg->dev, "%s: cannot alloc req\n", __func__);
1423 		return -ENOMEM;
1424 	}
1425 
1426 	req->buf = hsotg->ep0_buff;
1427 	req->length = length;
1428 	/*
1429 	 * zero flag is for sending zlp in DATA IN stage. It has no impact on
1430 	 * STATUS stage.
1431 	 */
1432 	req->zero = 0;
1433 	req->complete = dwc2_hsotg_complete_oursetup;
1434 
1435 	if (length)
1436 		memcpy(req->buf, buff, length);
1437 
1438 	ret = dwc2_hsotg_ep_queue(&ep->ep, req, GFP_ATOMIC);
1439 	if (ret) {
1440 		dev_warn(hsotg->dev, "%s: cannot queue req\n", __func__);
1441 		return ret;
1442 	}
1443 
1444 	return 0;
1445 }
1446 
1447 /**
1448  * dwc2_hsotg_process_req_status - process request GET_STATUS
1449  * @hsotg: The device state
1450  * @ctrl: USB control request
1451  */
1452 static int dwc2_hsotg_process_req_status(struct dwc2_hsotg *hsotg,
1453 					struct usb_ctrlrequest *ctrl)
1454 {
1455 	struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1456 	struct dwc2_hsotg_ep *ep;
1457 	__le16 reply;
1458 	int ret;
1459 
1460 	dev_dbg(hsotg->dev, "%s: USB_REQ_GET_STATUS\n", __func__);
1461 
1462 	if (!ep0->dir_in) {
1463 		dev_warn(hsotg->dev, "%s: direction out?\n", __func__);
1464 		return -EINVAL;
1465 	}
1466 
1467 	switch (ctrl->bRequestType & USB_RECIP_MASK) {
1468 	case USB_RECIP_DEVICE:
1469 		reply = cpu_to_le16(0); /* bit 0 => self powered,
1470 					 * bit 1 => remote wakeup */
1471 		break;
1472 
1473 	case USB_RECIP_INTERFACE:
1474 		/* currently, the data result should be zero */
1475 		reply = cpu_to_le16(0);
1476 		break;
1477 
1478 	case USB_RECIP_ENDPOINT:
1479 		ep = ep_from_windex(hsotg, le16_to_cpu(ctrl->wIndex));
1480 		if (!ep)
1481 			return -ENOENT;
1482 
1483 		reply = cpu_to_le16(ep->halted ? 1 : 0);
1484 		break;
1485 
1486 	default:
1487 		return 0;
1488 	}
1489 
1490 	if (le16_to_cpu(ctrl->wLength) != 2)
1491 		return -EINVAL;
1492 
1493 	ret = dwc2_hsotg_send_reply(hsotg, ep0, &reply, 2);
1494 	if (ret) {
1495 		dev_err(hsotg->dev, "%s: failed to send reply\n", __func__);
1496 		return ret;
1497 	}
1498 
1499 	return 1;
1500 }
1501 
1502 static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now);
1503 
1504 /**
1505  * get_ep_head - return the first request on the endpoint
1506  * @hs_ep: The controller endpoint to get
1507  *
1508  * Get the first request on the endpoint.
1509  */
1510 static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep)
1511 {
1512 	return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req,
1513 					queue);
1514 }
1515 
1516 /**
1517  * dwc2_gadget_start_next_request - Starts next request from ep queue
1518  * @hs_ep: Endpoint structure
1519  *
1520  * If queue is empty and EP is ISOC-OUT - unmasks OUTTKNEPDIS which is masked
1521  * in its handler. Hence we need to unmask it here to be able to do
1522  * resynchronization.
1523  */
1524 static void dwc2_gadget_start_next_request(struct dwc2_hsotg_ep *hs_ep)
1525 {
1526 	u32 mask;
1527 	struct dwc2_hsotg *hsotg = hs_ep->parent;
1528 	int dir_in = hs_ep->dir_in;
1529 	struct dwc2_hsotg_req *hs_req;
1530 	u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK;
1531 
1532 	if (!list_empty(&hs_ep->queue)) {
1533 		hs_req = get_ep_head(hs_ep);
1534 		dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, false);
1535 		return;
1536 	}
1537 	if (!hs_ep->isochronous)
1538 		return;
1539 
1540 	if (dir_in) {
1541 		dev_dbg(hsotg->dev, "%s: No more ISOC-IN requests\n",
1542 			__func__);
1543 	} else {
1544 		dev_dbg(hsotg->dev, "%s: No more ISOC-OUT requests\n",
1545 			__func__);
1546 		mask = dwc2_readl(hsotg->regs + epmsk_reg);
1547 		mask |= DOEPMSK_OUTTKNEPDISMSK;
1548 		dwc2_writel(mask, hsotg->regs + epmsk_reg);
1549 	}
1550 }
1551 
1552 /**
1553  * dwc2_hsotg_process_req_feature - process request {SET,CLEAR}_FEATURE
1554  * @hsotg: The device state
1555  * @ctrl: USB control request
1556  */
1557 static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg,
1558 					 struct usb_ctrlrequest *ctrl)
1559 {
1560 	struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1561 	struct dwc2_hsotg_req *hs_req;
1562 	bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
1563 	struct dwc2_hsotg_ep *ep;
1564 	int ret;
1565 	bool halted;
1566 	u32 recip;
1567 	u32 wValue;
1568 	u32 wIndex;
1569 
1570 	dev_dbg(hsotg->dev, "%s: %s_FEATURE\n",
1571 		__func__, set ? "SET" : "CLEAR");
1572 
1573 	wValue = le16_to_cpu(ctrl->wValue);
1574 	wIndex = le16_to_cpu(ctrl->wIndex);
1575 	recip = ctrl->bRequestType & USB_RECIP_MASK;
1576 
1577 	switch (recip) {
1578 	case USB_RECIP_DEVICE:
1579 		switch (wValue) {
1580 		case USB_DEVICE_TEST_MODE:
1581 			if ((wIndex & 0xff) != 0)
1582 				return -EINVAL;
1583 			if (!set)
1584 				return -EINVAL;
1585 
1586 			hsotg->test_mode = wIndex >> 8;
1587 			ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
1588 			if (ret) {
1589 				dev_err(hsotg->dev,
1590 					"%s: failed to send reply\n", __func__);
1591 				return ret;
1592 			}
1593 			break;
1594 		default:
1595 			return -ENOENT;
1596 		}
1597 		break;
1598 
1599 	case USB_RECIP_ENDPOINT:
1600 		ep = ep_from_windex(hsotg, wIndex);
1601 		if (!ep) {
1602 			dev_dbg(hsotg->dev, "%s: no endpoint for 0x%04x\n",
1603 				__func__, wIndex);
1604 			return -ENOENT;
1605 		}
1606 
1607 		switch (wValue) {
1608 		case USB_ENDPOINT_HALT:
1609 			halted = ep->halted;
1610 
1611 			dwc2_hsotg_ep_sethalt(&ep->ep, set, true);
1612 
1613 			ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
1614 			if (ret) {
1615 				dev_err(hsotg->dev,
1616 					"%s: failed to send reply\n", __func__);
1617 				return ret;
1618 			}
1619 
1620 			/*
1621 			 * we have to complete all requests for ep if it was
1622 			 * halted, and the halt was cleared by CLEAR_FEATURE
1623 			 */
1624 
1625 			if (!set && halted) {
1626 				/*
1627 				 * If we have request in progress,
1628 				 * then complete it
1629 				 */
1630 				if (ep->req) {
1631 					hs_req = ep->req;
1632 					ep->req = NULL;
1633 					list_del_init(&hs_req->queue);
1634 					if (hs_req->req.complete) {
1635 						spin_unlock(&hsotg->lock);
1636 						usb_gadget_giveback_request(
1637 							&ep->ep, &hs_req->req);
1638 						spin_lock(&hsotg->lock);
1639 					}
1640 				}
1641 
1642 				/* If we have pending request, then start it */
1643 				if (!ep->req) {
1644 					dwc2_gadget_start_next_request(ep);
1645 				}
1646 			}
1647 
1648 			break;
1649 
1650 		default:
1651 			return -ENOENT;
1652 		}
1653 		break;
1654 	default:
1655 		return -ENOENT;
1656 	}
1657 	return 1;
1658 }
1659 
1660 static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg);
1661 
1662 /**
1663  * dwc2_hsotg_stall_ep0 - stall ep0
1664  * @hsotg: The device state
1665  *
1666  * Set stall for ep0 as response for setup request.
1667  */
1668 static void dwc2_hsotg_stall_ep0(struct dwc2_hsotg *hsotg)
1669 {
1670 	struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1671 	u32 reg;
1672 	u32 ctrl;
1673 
1674 	dev_dbg(hsotg->dev, "ep0 stall (dir=%d)\n", ep0->dir_in);
1675 	reg = (ep0->dir_in) ? DIEPCTL0 : DOEPCTL0;
1676 
1677 	/*
1678 	 * DxEPCTL_Stall will be cleared by EP once it has
1679 	 * taken effect, so no need to clear later.
1680 	 */
1681 
1682 	ctrl = dwc2_readl(hsotg->regs + reg);
1683 	ctrl |= DXEPCTL_STALL;
1684 	ctrl |= DXEPCTL_CNAK;
1685 	dwc2_writel(ctrl, hsotg->regs + reg);
1686 
1687 	dev_dbg(hsotg->dev,
1688 		"written DXEPCTL=0x%08x to %08x (DXEPCTL=0x%08x)\n",
1689 		ctrl, reg, dwc2_readl(hsotg->regs + reg));
1690 
1691 	 /*
1692 	  * complete won't be called, so we enqueue
1693 	  * setup request here
1694 	  */
1695 	 dwc2_hsotg_enqueue_setup(hsotg);
1696 }
1697 
1698 /**
1699  * dwc2_hsotg_process_control - process a control request
1700  * @hsotg: The device state
1701  * @ctrl: The control request received
1702  *
1703  * The controller has received the SETUP phase of a control request, and
1704  * needs to work out what to do next (and whether to pass it on to the
1705  * gadget driver).
1706  */
1707 static void dwc2_hsotg_process_control(struct dwc2_hsotg *hsotg,
1708 				      struct usb_ctrlrequest *ctrl)
1709 {
1710 	struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0];
1711 	int ret = 0;
1712 	u32 dcfg;
1713 
1714 	dev_dbg(hsotg->dev,
1715 		"ctrl Type=%02x, Req=%02x, V=%04x, I=%04x, L=%04x\n",
1716 		ctrl->bRequestType, ctrl->bRequest, ctrl->wValue,
1717 		ctrl->wIndex, ctrl->wLength);
1718 
1719 	if (ctrl->wLength == 0) {
1720 		ep0->dir_in = 1;
1721 		hsotg->ep0_state = DWC2_EP0_STATUS_IN;
1722 	} else if (ctrl->bRequestType & USB_DIR_IN) {
1723 		ep0->dir_in = 1;
1724 		hsotg->ep0_state = DWC2_EP0_DATA_IN;
1725 	} else {
1726 		ep0->dir_in = 0;
1727 		hsotg->ep0_state = DWC2_EP0_DATA_OUT;
1728 	}
1729 
1730 	if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1731 		switch (ctrl->bRequest) {
1732 		case USB_REQ_SET_ADDRESS:
1733 			hsotg->connected = 1;
1734 			dcfg = dwc2_readl(hsotg->regs + DCFG);
1735 			dcfg &= ~DCFG_DEVADDR_MASK;
1736 			dcfg |= (le16_to_cpu(ctrl->wValue) <<
1737 				 DCFG_DEVADDR_SHIFT) & DCFG_DEVADDR_MASK;
1738 			dwc2_writel(dcfg, hsotg->regs + DCFG);
1739 
1740 			dev_info(hsotg->dev, "new address %d\n", ctrl->wValue);
1741 
1742 			ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0);
1743 			return;
1744 
1745 		case USB_REQ_GET_STATUS:
1746 			ret = dwc2_hsotg_process_req_status(hsotg, ctrl);
1747 			break;
1748 
1749 		case USB_REQ_CLEAR_FEATURE:
1750 		case USB_REQ_SET_FEATURE:
1751 			ret = dwc2_hsotg_process_req_feature(hsotg, ctrl);
1752 			break;
1753 		}
1754 	}
1755 
1756 	/* as a fallback, try delivering it to the driver to deal with */
1757 
1758 	if (ret == 0 && hsotg->driver) {
1759 		spin_unlock(&hsotg->lock);
1760 		ret = hsotg->driver->setup(&hsotg->gadget, ctrl);
1761 		spin_lock(&hsotg->lock);
1762 		if (ret < 0)
1763 			dev_dbg(hsotg->dev, "driver->setup() ret %d\n", ret);
1764 	}
1765 
1766 	/*
1767 	 * the request is either unhandlable, or is not formatted correctly
1768 	 * so respond with a STALL for the status stage to indicate failure.
1769 	 */
1770 
1771 	if (ret < 0)
1772 		dwc2_hsotg_stall_ep0(hsotg);
1773 }
1774 
1775 /**
1776  * dwc2_hsotg_complete_setup - completion of a setup transfer
1777  * @ep: The endpoint the request was on.
1778  * @req: The request completed.
1779  *
1780  * Called on completion of any requests the driver itself submitted for
1781  * EP0 setup packets
1782  */
1783 static void dwc2_hsotg_complete_setup(struct usb_ep *ep,
1784 				     struct usb_request *req)
1785 {
1786 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
1787 	struct dwc2_hsotg *hsotg = hs_ep->parent;
1788 
1789 	if (req->status < 0) {
1790 		dev_dbg(hsotg->dev, "%s: failed %d\n", __func__, req->status);
1791 		return;
1792 	}
1793 
1794 	spin_lock(&hsotg->lock);
1795 	if (req->actual == 0)
1796 		dwc2_hsotg_enqueue_setup(hsotg);
1797 	else
1798 		dwc2_hsotg_process_control(hsotg, req->buf);
1799 	spin_unlock(&hsotg->lock);
1800 }
1801 
1802 /**
1803  * dwc2_hsotg_enqueue_setup - start a request for EP0 packets
1804  * @hsotg: The device state.
1805  *
1806  * Enqueue a request on EP0 if necessary to received any SETUP packets
1807  * received from the host.
1808  */
1809 static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg)
1810 {
1811 	struct usb_request *req = hsotg->ctrl_req;
1812 	struct dwc2_hsotg_req *hs_req = our_req(req);
1813 	int ret;
1814 
1815 	dev_dbg(hsotg->dev, "%s: queueing setup request\n", __func__);
1816 
1817 	req->zero = 0;
1818 	req->length = 8;
1819 	req->buf = hsotg->ctrl_buff;
1820 	req->complete = dwc2_hsotg_complete_setup;
1821 
1822 	if (!list_empty(&hs_req->queue)) {
1823 		dev_dbg(hsotg->dev, "%s already queued???\n", __func__);
1824 		return;
1825 	}
1826 
1827 	hsotg->eps_out[0]->dir_in = 0;
1828 	hsotg->eps_out[0]->send_zlp = 0;
1829 	hsotg->ep0_state = DWC2_EP0_SETUP;
1830 
1831 	ret = dwc2_hsotg_ep_queue(&hsotg->eps_out[0]->ep, req, GFP_ATOMIC);
1832 	if (ret < 0) {
1833 		dev_err(hsotg->dev, "%s: failed queue (%d)\n", __func__, ret);
1834 		/*
1835 		 * Don't think there's much we can do other than watch the
1836 		 * driver fail.
1837 		 */
1838 	}
1839 }
1840 
1841 static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg,
1842 					struct dwc2_hsotg_ep *hs_ep)
1843 {
1844 	u32 ctrl;
1845 	u8 index = hs_ep->index;
1846 	u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
1847 	u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index);
1848 
1849 	if (hs_ep->dir_in)
1850 		dev_dbg(hsotg->dev, "Sending zero-length packet on ep%d\n",
1851 			index);
1852 	else
1853 		dev_dbg(hsotg->dev, "Receiving zero-length packet on ep%d\n",
1854 			index);
1855 	if (using_desc_dma(hsotg)) {
1856 		/* Not specific buffer needed for ep0 ZLP */
1857 		dma_addr_t dma = hs_ep->desc_list_dma;
1858 
1859 		dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep);
1860 		dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0);
1861 	} else {
1862 		dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
1863 			    DXEPTSIZ_XFERSIZE(0), hsotg->regs +
1864 			    epsiz_reg);
1865 	}
1866 
1867 	ctrl = dwc2_readl(hsotg->regs + epctl_reg);
1868 	ctrl |= DXEPCTL_CNAK;  /* clear NAK set by core */
1869 	ctrl |= DXEPCTL_EPENA; /* ensure ep enabled */
1870 	ctrl |= DXEPCTL_USBACTEP;
1871 	dwc2_writel(ctrl, hsotg->regs + epctl_reg);
1872 }
1873 
1874 /**
1875  * dwc2_hsotg_complete_request - complete a request given to us
1876  * @hsotg: The device state.
1877  * @hs_ep: The endpoint the request was on.
1878  * @hs_req: The request to complete.
1879  * @result: The result code (0 => Ok, otherwise errno)
1880  *
1881  * The given request has finished, so call the necessary completion
1882  * if it has one and then look to see if we can start a new request
1883  * on the endpoint.
1884  *
1885  * Note, expects the ep to already be locked as appropriate.
1886  */
1887 static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg,
1888 				       struct dwc2_hsotg_ep *hs_ep,
1889 				       struct dwc2_hsotg_req *hs_req,
1890 				       int result)
1891 {
1892 
1893 	if (!hs_req) {
1894 		dev_dbg(hsotg->dev, "%s: nothing to complete?\n", __func__);
1895 		return;
1896 	}
1897 
1898 	dev_dbg(hsotg->dev, "complete: ep %p %s, req %p, %d => %p\n",
1899 		hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete);
1900 
1901 	/*
1902 	 * only replace the status if we've not already set an error
1903 	 * from a previous transaction
1904 	 */
1905 
1906 	if (hs_req->req.status == -EINPROGRESS)
1907 		hs_req->req.status = result;
1908 
1909 	if (using_dma(hsotg))
1910 		dwc2_hsotg_unmap_dma(hsotg, hs_ep, hs_req);
1911 
1912 	dwc2_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req);
1913 
1914 	hs_ep->req = NULL;
1915 	list_del_init(&hs_req->queue);
1916 
1917 	/*
1918 	 * call the complete request with the locks off, just in case the
1919 	 * request tries to queue more work for this endpoint.
1920 	 */
1921 
1922 	if (hs_req->req.complete) {
1923 		spin_unlock(&hsotg->lock);
1924 		usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req);
1925 		spin_lock(&hsotg->lock);
1926 	}
1927 
1928 	/* In DDMA don't need to proceed to starting of next ISOC request */
1929 	if (using_desc_dma(hsotg) && hs_ep->isochronous)
1930 		return;
1931 
1932 	/*
1933 	 * Look to see if there is anything else to do. Note, the completion
1934 	 * of the previous request may have caused a new request to be started
1935 	 * so be careful when doing this.
1936 	 */
1937 
1938 	if (!hs_ep->req && result >= 0) {
1939 		dwc2_gadget_start_next_request(hs_ep);
1940 	}
1941 }
1942 
1943 /*
1944  * dwc2_gadget_complete_isoc_request_ddma - complete an isoc request in DDMA
1945  * @hs_ep: The endpoint the request was on.
1946  *
1947  * Get first request from the ep queue, determine descriptor on which complete
1948  * happened. SW based on isoc_chain_num discovers which half of the descriptor
1949  * chain is currently in use by HW, adjusts dma_address and calculates index
1950  * of completed descriptor based on the value of DEPDMA register. Update actual
1951  * length of request, giveback to gadget.
1952  */
1953 static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep)
1954 {
1955 	struct dwc2_hsotg *hsotg = hs_ep->parent;
1956 	struct dwc2_hsotg_req *hs_req;
1957 	struct usb_request *ureq;
1958 	int index;
1959 	dma_addr_t dma_addr;
1960 	u32 dma_reg;
1961 	u32 depdma;
1962 	u32 desc_sts;
1963 	u32 mask;
1964 
1965 	hs_req = get_ep_head(hs_ep);
1966 	if (!hs_req) {
1967 		dev_warn(hsotg->dev, "%s: ISOC EP queue empty\n", __func__);
1968 		return;
1969 	}
1970 	ureq = &hs_req->req;
1971 
1972 	dma_addr = hs_ep->desc_list_dma;
1973 
1974 	/*
1975 	 * If lower half of  descriptor chain is currently use by SW,
1976 	 * that means higher half is being processed by HW, so shift
1977 	 * DMA address to higher half of descriptor chain.
1978 	 */
1979 	if (!hs_ep->isoc_chain_num)
1980 		dma_addr += sizeof(struct dwc2_dma_desc) *
1981 			    (MAX_DMA_DESC_NUM_GENERIC / 2);
1982 
1983 	dma_reg = hs_ep->dir_in ? DIEPDMA(hs_ep->index) : DOEPDMA(hs_ep->index);
1984 	depdma = dwc2_readl(hsotg->regs + dma_reg);
1985 
1986 	index = (depdma - dma_addr) / sizeof(struct dwc2_dma_desc) - 1;
1987 	desc_sts = hs_ep->desc_list[index].status;
1988 
1989 	mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK :
1990 	       DEV_DMA_ISOC_RX_NBYTES_MASK;
1991 	ureq->actual = ureq->length -
1992 		       ((desc_sts & mask) >> DEV_DMA_ISOC_NBYTES_SHIFT);
1993 
1994 	/* Adjust actual length for ISOC Out if length is not align of 4 */
1995 	if (!hs_ep->dir_in && ureq->length & 0x3)
1996 		ureq->actual += 4 - (ureq->length & 0x3);
1997 
1998 	dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
1999 }
2000 
2001 /*
2002  * dwc2_gadget_start_next_isoc_ddma - start next isoc request, if any.
2003  * @hs_ep: The isochronous endpoint to be re-enabled.
2004  *
2005  * If ep has been disabled due to last descriptor servicing (IN endpoint) or
2006  * BNA (OUT endpoint) check the status of other half of descriptor chain that
2007  * was under SW control till HW was busy and restart the endpoint if needed.
2008  */
2009 static void dwc2_gadget_start_next_isoc_ddma(struct dwc2_hsotg_ep *hs_ep)
2010 {
2011 	struct dwc2_hsotg *hsotg = hs_ep->parent;
2012 	u32 depctl;
2013 	u32 dma_reg;
2014 	u32 ctrl;
2015 	u32 dma_addr = hs_ep->desc_list_dma;
2016 	unsigned char index = hs_ep->index;
2017 
2018 	dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index);
2019 	depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index);
2020 
2021 	ctrl = dwc2_readl(hsotg->regs + depctl);
2022 
2023 	/*
2024 	 * EP was disabled if HW has processed last descriptor or BNA was set.
2025 	 * So restart ep if SW has prepared new descriptor chain in ep_queue
2026 	 * routine while HW was busy.
2027 	 */
2028 	if (!(ctrl & DXEPCTL_EPENA)) {
2029 		if (!hs_ep->next_desc) {
2030 			dev_dbg(hsotg->dev, "%s: No more ISOC requests\n",
2031 				__func__);
2032 			return;
2033 		}
2034 
2035 		dma_addr += sizeof(struct dwc2_dma_desc) *
2036 			    (MAX_DMA_DESC_NUM_GENERIC / 2) *
2037 			    hs_ep->isoc_chain_num;
2038 		dwc2_writel(dma_addr, hsotg->regs + dma_reg);
2039 
2040 		ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK;
2041 		dwc2_writel(ctrl, hsotg->regs + depctl);
2042 
2043 		/* Switch ISOC descriptor chain number being processed by SW*/
2044 		hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1;
2045 		hs_ep->next_desc = 0;
2046 
2047 		dev_dbg(hsotg->dev, "%s: Restarted isochronous endpoint\n",
2048 			__func__);
2049 	}
2050 }
2051 
2052 /**
2053  * dwc2_hsotg_rx_data - receive data from the FIFO for an endpoint
2054  * @hsotg: The device state.
2055  * @ep_idx: The endpoint index for the data
2056  * @size: The size of data in the fifo, in bytes
2057  *
2058  * The FIFO status shows there is data to read from the FIFO for a given
2059  * endpoint, so sort out whether we need to read the data into a request
2060  * that has been made for that endpoint.
2061  */
2062 static void dwc2_hsotg_rx_data(struct dwc2_hsotg *hsotg, int ep_idx, int size)
2063 {
2064 	struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx];
2065 	struct dwc2_hsotg_req *hs_req = hs_ep->req;
2066 	void __iomem *fifo = hsotg->regs + EPFIFO(ep_idx);
2067 	int to_read;
2068 	int max_req;
2069 	int read_ptr;
2070 
2071 
2072 	if (!hs_req) {
2073 		u32 epctl = dwc2_readl(hsotg->regs + DOEPCTL(ep_idx));
2074 		int ptr;
2075 
2076 		dev_dbg(hsotg->dev,
2077 			 "%s: FIFO %d bytes on ep%d but no req (DXEPCTl=0x%08x)\n",
2078 			 __func__, size, ep_idx, epctl);
2079 
2080 		/* dump the data from the FIFO, we've nothing we can do */
2081 		for (ptr = 0; ptr < size; ptr += 4)
2082 			(void)dwc2_readl(fifo);
2083 
2084 		return;
2085 	}
2086 
2087 	to_read = size;
2088 	read_ptr = hs_req->req.actual;
2089 	max_req = hs_req->req.length - read_ptr;
2090 
2091 	dev_dbg(hsotg->dev, "%s: read %d/%d, done %d/%d\n",
2092 		__func__, to_read, max_req, read_ptr, hs_req->req.length);
2093 
2094 	if (to_read > max_req) {
2095 		/*
2096 		 * more data appeared than we where willing
2097 		 * to deal with in this request.
2098 		 */
2099 
2100 		/* currently we don't deal this */
2101 		WARN_ON_ONCE(1);
2102 	}
2103 
2104 	hs_ep->total_data += to_read;
2105 	hs_req->req.actual += to_read;
2106 	to_read = DIV_ROUND_UP(to_read, 4);
2107 
2108 	/*
2109 	 * note, we might over-write the buffer end by 3 bytes depending on
2110 	 * alignment of the data.
2111 	 */
2112 	ioread32_rep(fifo, hs_req->req.buf + read_ptr, to_read);
2113 }
2114 
2115 /**
2116  * dwc2_hsotg_ep0_zlp - send/receive zero-length packet on control endpoint
2117  * @hsotg: The device instance
2118  * @dir_in: If IN zlp
2119  *
2120  * Generate a zero-length IN packet request for terminating a SETUP
2121  * transaction.
2122  *
2123  * Note, since we don't write any data to the TxFIFO, then it is
2124  * currently believed that we do not need to wait for any space in
2125  * the TxFIFO.
2126  */
2127 static void dwc2_hsotg_ep0_zlp(struct dwc2_hsotg *hsotg, bool dir_in)
2128 {
2129 	/* eps_out[0] is used in both directions */
2130 	hsotg->eps_out[0]->dir_in = dir_in;
2131 	hsotg->ep0_state = dir_in ? DWC2_EP0_STATUS_IN : DWC2_EP0_STATUS_OUT;
2132 
2133 	dwc2_hsotg_program_zlp(hsotg, hsotg->eps_out[0]);
2134 }
2135 
2136 static void dwc2_hsotg_change_ep_iso_parity(struct dwc2_hsotg *hsotg,
2137 			u32 epctl_reg)
2138 {
2139 	u32 ctrl;
2140 
2141 	ctrl = dwc2_readl(hsotg->regs + epctl_reg);
2142 	if (ctrl & DXEPCTL_EOFRNUM)
2143 		ctrl |= DXEPCTL_SETEVENFR;
2144 	else
2145 		ctrl |= DXEPCTL_SETODDFR;
2146 	dwc2_writel(ctrl, hsotg->regs + epctl_reg);
2147 }
2148 
2149 /*
2150  * dwc2_gadget_get_xfersize_ddma - get transferred bytes amount from desc
2151  * @hs_ep - The endpoint on which transfer went
2152  *
2153  * Iterate over endpoints descriptor chain and get info on bytes remained
2154  * in DMA descriptors after transfer has completed. Used for non isoc EPs.
2155  */
2156 static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep)
2157 {
2158 	struct dwc2_hsotg *hsotg = hs_ep->parent;
2159 	unsigned int bytes_rem = 0;
2160 	struct dwc2_dma_desc *desc = hs_ep->desc_list;
2161 	int i;
2162 	u32 status;
2163 
2164 	if (!desc)
2165 		return -EINVAL;
2166 
2167 	for (i = 0; i < hs_ep->desc_count; ++i) {
2168 		status = desc->status;
2169 		bytes_rem += status & DEV_DMA_NBYTES_MASK;
2170 
2171 		if (status & DEV_DMA_STS_MASK)
2172 			dev_err(hsotg->dev, "descriptor %d closed with %x\n",
2173 				i, status & DEV_DMA_STS_MASK);
2174 	}
2175 
2176 	return bytes_rem;
2177 }
2178 
2179 /**
2180  * dwc2_hsotg_handle_outdone - handle receiving OutDone/SetupDone from RXFIFO
2181  * @hsotg: The device instance
2182  * @epnum: The endpoint received from
2183  *
2184  * The RXFIFO has delivered an OutDone event, which means that the data
2185  * transfer for an OUT endpoint has been completed, either by a short
2186  * packet or by the finish of a transfer.
2187  */
2188 static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum)
2189 {
2190 	u32 epsize = dwc2_readl(hsotg->regs + DOEPTSIZ(epnum));
2191 	struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum];
2192 	struct dwc2_hsotg_req *hs_req = hs_ep->req;
2193 	struct usb_request *req = &hs_req->req;
2194 	unsigned size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
2195 	int result = 0;
2196 
2197 	if (!hs_req) {
2198 		dev_dbg(hsotg->dev, "%s: no request active\n", __func__);
2199 		return;
2200 	}
2201 
2202 	if (epnum == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_OUT) {
2203 		dev_dbg(hsotg->dev, "zlp packet received\n");
2204 		dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2205 		dwc2_hsotg_enqueue_setup(hsotg);
2206 		return;
2207 	}
2208 
2209 	if (using_desc_dma(hsotg))
2210 		size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
2211 
2212 	if (using_dma(hsotg)) {
2213 		unsigned size_done;
2214 
2215 		/*
2216 		 * Calculate the size of the transfer by checking how much
2217 		 * is left in the endpoint size register and then working it
2218 		 * out from the amount we loaded for the transfer.
2219 		 *
2220 		 * We need to do this as DMA pointers are always 32bit aligned
2221 		 * so may overshoot/undershoot the transfer.
2222 		 */
2223 
2224 		size_done = hs_ep->size_loaded - size_left;
2225 		size_done += hs_ep->last_load;
2226 
2227 		req->actual = size_done;
2228 	}
2229 
2230 	/* if there is more request to do, schedule new transfer */
2231 	if (req->actual < req->length && size_left == 0) {
2232 		dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
2233 		return;
2234 	}
2235 
2236 	if (req->actual < req->length && req->short_not_ok) {
2237 		dev_dbg(hsotg->dev, "%s: got %d/%d (short not ok) => error\n",
2238 			__func__, req->actual, req->length);
2239 
2240 		/*
2241 		 * todo - what should we return here? there's no one else
2242 		 * even bothering to check the status.
2243 		 */
2244 	}
2245 
2246 	/* DDMA IN status phase will start from StsPhseRcvd interrupt */
2247 	if (!using_desc_dma(hsotg) && epnum == 0 &&
2248 	    hsotg->ep0_state == DWC2_EP0_DATA_OUT) {
2249 		/* Move to STATUS IN */
2250 		dwc2_hsotg_ep0_zlp(hsotg, true);
2251 		return;
2252 	}
2253 
2254 	/*
2255 	 * Slave mode OUT transfers do not go through XferComplete so
2256 	 * adjust the ISOC parity here.
2257 	 */
2258 	if (!using_dma(hsotg)) {
2259 		if (hs_ep->isochronous && hs_ep->interval == 1)
2260 			dwc2_hsotg_change_ep_iso_parity(hsotg, DOEPCTL(epnum));
2261 		else if (hs_ep->isochronous && hs_ep->interval > 1)
2262 			dwc2_gadget_incr_frame_num(hs_ep);
2263 	}
2264 
2265 	dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, result);
2266 }
2267 
2268 /**
2269  * dwc2_hsotg_handle_rx - RX FIFO has data
2270  * @hsotg: The device instance
2271  *
2272  * The IRQ handler has detected that the RX FIFO has some data in it
2273  * that requires processing, so find out what is in there and do the
2274  * appropriate read.
2275  *
2276  * The RXFIFO is a true FIFO, the packets coming out are still in packet
2277  * chunks, so if you have x packets received on an endpoint you'll get x
2278  * FIFO events delivered, each with a packet's worth of data in it.
2279  *
2280  * When using DMA, we should not be processing events from the RXFIFO
2281  * as the actual data should be sent to the memory directly and we turn
2282  * on the completion interrupts to get notifications of transfer completion.
2283  */
2284 static void dwc2_hsotg_handle_rx(struct dwc2_hsotg *hsotg)
2285 {
2286 	u32 grxstsr = dwc2_readl(hsotg->regs + GRXSTSP);
2287 	u32 epnum, status, size;
2288 
2289 	WARN_ON(using_dma(hsotg));
2290 
2291 	epnum = grxstsr & GRXSTS_EPNUM_MASK;
2292 	status = grxstsr & GRXSTS_PKTSTS_MASK;
2293 
2294 	size = grxstsr & GRXSTS_BYTECNT_MASK;
2295 	size >>= GRXSTS_BYTECNT_SHIFT;
2296 
2297 	dev_dbg(hsotg->dev, "%s: GRXSTSP=0x%08x (%d@%d)\n",
2298 			__func__, grxstsr, size, epnum);
2299 
2300 	switch ((status & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT) {
2301 	case GRXSTS_PKTSTS_GLOBALOUTNAK:
2302 		dev_dbg(hsotg->dev, "GLOBALOUTNAK\n");
2303 		break;
2304 
2305 	case GRXSTS_PKTSTS_OUTDONE:
2306 		dev_dbg(hsotg->dev, "OutDone (Frame=0x%08x)\n",
2307 			dwc2_hsotg_read_frameno(hsotg));
2308 
2309 		if (!using_dma(hsotg))
2310 			dwc2_hsotg_handle_outdone(hsotg, epnum);
2311 		break;
2312 
2313 	case GRXSTS_PKTSTS_SETUPDONE:
2314 		dev_dbg(hsotg->dev,
2315 			"SetupDone (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
2316 			dwc2_hsotg_read_frameno(hsotg),
2317 			dwc2_readl(hsotg->regs + DOEPCTL(0)));
2318 		/*
2319 		 * Call dwc2_hsotg_handle_outdone here if it was not called from
2320 		 * GRXSTS_PKTSTS_OUTDONE. That is, if the core didn't
2321 		 * generate GRXSTS_PKTSTS_OUTDONE for setup packet.
2322 		 */
2323 		if (hsotg->ep0_state == DWC2_EP0_SETUP)
2324 			dwc2_hsotg_handle_outdone(hsotg, epnum);
2325 		break;
2326 
2327 	case GRXSTS_PKTSTS_OUTRX:
2328 		dwc2_hsotg_rx_data(hsotg, epnum, size);
2329 		break;
2330 
2331 	case GRXSTS_PKTSTS_SETUPRX:
2332 		dev_dbg(hsotg->dev,
2333 			"SetupRX (Frame=0x%08x, DOPEPCTL=0x%08x)\n",
2334 			dwc2_hsotg_read_frameno(hsotg),
2335 			dwc2_readl(hsotg->regs + DOEPCTL(0)));
2336 
2337 		WARN_ON(hsotg->ep0_state != DWC2_EP0_SETUP);
2338 
2339 		dwc2_hsotg_rx_data(hsotg, epnum, size);
2340 		break;
2341 
2342 	default:
2343 		dev_warn(hsotg->dev, "%s: unknown status %08x\n",
2344 			 __func__, grxstsr);
2345 
2346 		dwc2_hsotg_dump(hsotg);
2347 		break;
2348 	}
2349 }
2350 
2351 /**
2352  * dwc2_hsotg_ep0_mps - turn max packet size into register setting
2353  * @mps: The maximum packet size in bytes.
2354  */
2355 static u32 dwc2_hsotg_ep0_mps(unsigned int mps)
2356 {
2357 	switch (mps) {
2358 	case 64:
2359 		return D0EPCTL_MPS_64;
2360 	case 32:
2361 		return D0EPCTL_MPS_32;
2362 	case 16:
2363 		return D0EPCTL_MPS_16;
2364 	case 8:
2365 		return D0EPCTL_MPS_8;
2366 	}
2367 
2368 	/* bad max packet size, warn and return invalid result */
2369 	WARN_ON(1);
2370 	return (u32)-1;
2371 }
2372 
2373 /**
2374  * dwc2_hsotg_set_ep_maxpacket - set endpoint's max-packet field
2375  * @hsotg: The driver state.
2376  * @ep: The index number of the endpoint
2377  * @mps: The maximum packet size in bytes
2378  * @mc: The multicount value
2379  *
2380  * Configure the maximum packet size for the given endpoint, updating
2381  * the hardware control registers to reflect this.
2382  */
2383 static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg,
2384 					unsigned int ep, unsigned int mps,
2385 					unsigned int mc, unsigned int dir_in)
2386 {
2387 	struct dwc2_hsotg_ep *hs_ep;
2388 	void __iomem *regs = hsotg->regs;
2389 	u32 reg;
2390 
2391 	hs_ep = index_to_ep(hsotg, ep, dir_in);
2392 	if (!hs_ep)
2393 		return;
2394 
2395 	if (ep == 0) {
2396 		u32 mps_bytes = mps;
2397 
2398 		/* EP0 is a special case */
2399 		mps = dwc2_hsotg_ep0_mps(mps_bytes);
2400 		if (mps > 3)
2401 			goto bad_mps;
2402 		hs_ep->ep.maxpacket = mps_bytes;
2403 		hs_ep->mc = 1;
2404 	} else {
2405 		if (mps > 1024)
2406 			goto bad_mps;
2407 		hs_ep->mc = mc;
2408 		if (mc > 3)
2409 			goto bad_mps;
2410 		hs_ep->ep.maxpacket = mps;
2411 	}
2412 
2413 	if (dir_in) {
2414 		reg = dwc2_readl(regs + DIEPCTL(ep));
2415 		reg &= ~DXEPCTL_MPS_MASK;
2416 		reg |= mps;
2417 		dwc2_writel(reg, regs + DIEPCTL(ep));
2418 	} else {
2419 		reg = dwc2_readl(regs + DOEPCTL(ep));
2420 		reg &= ~DXEPCTL_MPS_MASK;
2421 		reg |= mps;
2422 		dwc2_writel(reg, regs + DOEPCTL(ep));
2423 	}
2424 
2425 	return;
2426 
2427 bad_mps:
2428 	dev_err(hsotg->dev, "ep%d: bad mps of %d\n", ep, mps);
2429 }
2430 
2431 /**
2432  * dwc2_hsotg_txfifo_flush - flush Tx FIFO
2433  * @hsotg: The driver state
2434  * @idx: The index for the endpoint (0..15)
2435  */
2436 static void dwc2_hsotg_txfifo_flush(struct dwc2_hsotg *hsotg, unsigned int idx)
2437 {
2438 	int timeout;
2439 	int val;
2440 
2441 	dwc2_writel(GRSTCTL_TXFNUM(idx) | GRSTCTL_TXFFLSH,
2442 		    hsotg->regs + GRSTCTL);
2443 
2444 	/* wait until the fifo is flushed */
2445 	timeout = 100;
2446 
2447 	while (1) {
2448 		val = dwc2_readl(hsotg->regs + GRSTCTL);
2449 
2450 		if ((val & (GRSTCTL_TXFFLSH)) == 0)
2451 			break;
2452 
2453 		if (--timeout == 0) {
2454 			dev_err(hsotg->dev,
2455 				"%s: timeout flushing fifo (GRSTCTL=%08x)\n",
2456 				__func__, val);
2457 			break;
2458 		}
2459 
2460 		udelay(1);
2461 	}
2462 }
2463 
2464 /**
2465  * dwc2_hsotg_trytx - check to see if anything needs transmitting
2466  * @hsotg: The driver state
2467  * @hs_ep: The driver endpoint to check.
2468  *
2469  * Check to see if there is a request that has data to send, and if so
2470  * make an attempt to write data into the FIFO.
2471  */
2472 static int dwc2_hsotg_trytx(struct dwc2_hsotg *hsotg,
2473 			   struct dwc2_hsotg_ep *hs_ep)
2474 {
2475 	struct dwc2_hsotg_req *hs_req = hs_ep->req;
2476 
2477 	if (!hs_ep->dir_in || !hs_req) {
2478 		/**
2479 		 * if request is not enqueued, we disable interrupts
2480 		 * for endpoints, excepting ep0
2481 		 */
2482 		if (hs_ep->index != 0)
2483 			dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index,
2484 					     hs_ep->dir_in, 0);
2485 		return 0;
2486 	}
2487 
2488 	if (hs_req->req.actual < hs_req->req.length) {
2489 		dev_dbg(hsotg->dev, "trying to write more for ep%d\n",
2490 			hs_ep->index);
2491 		return dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req);
2492 	}
2493 
2494 	return 0;
2495 }
2496 
2497 /**
2498  * dwc2_hsotg_complete_in - complete IN transfer
2499  * @hsotg: The device state.
2500  * @hs_ep: The endpoint that has just completed.
2501  *
2502  * An IN transfer has been completed, update the transfer's state and then
2503  * call the relevant completion routines.
2504  */
2505 static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg,
2506 				  struct dwc2_hsotg_ep *hs_ep)
2507 {
2508 	struct dwc2_hsotg_req *hs_req = hs_ep->req;
2509 	u32 epsize = dwc2_readl(hsotg->regs + DIEPTSIZ(hs_ep->index));
2510 	int size_left, size_done;
2511 
2512 	if (!hs_req) {
2513 		dev_dbg(hsotg->dev, "XferCompl but no req\n");
2514 		return;
2515 	}
2516 
2517 	/* Finish ZLP handling for IN EP0 transactions */
2518 	if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) {
2519 		dev_dbg(hsotg->dev, "zlp packet sent\n");
2520 
2521 		/*
2522 		 * While send zlp for DWC2_EP0_STATUS_IN EP direction was
2523 		 * changed to IN. Change back to complete OUT transfer request
2524 		 */
2525 		hs_ep->dir_in = 0;
2526 
2527 		dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2528 		if (hsotg->test_mode) {
2529 			int ret;
2530 
2531 			ret = dwc2_hsotg_set_test_mode(hsotg, hsotg->test_mode);
2532 			if (ret < 0) {
2533 				dev_dbg(hsotg->dev, "Invalid Test #%d\n",
2534 						hsotg->test_mode);
2535 				dwc2_hsotg_stall_ep0(hsotg);
2536 				return;
2537 			}
2538 		}
2539 		dwc2_hsotg_enqueue_setup(hsotg);
2540 		return;
2541 	}
2542 
2543 	/*
2544 	 * Calculate the size of the transfer by checking how much is left
2545 	 * in the endpoint size register and then working it out from
2546 	 * the amount we loaded for the transfer.
2547 	 *
2548 	 * We do this even for DMA, as the transfer may have incremented
2549 	 * past the end of the buffer (DMA transfers are always 32bit
2550 	 * aligned).
2551 	 */
2552 	if (using_desc_dma(hsotg)) {
2553 		size_left = dwc2_gadget_get_xfersize_ddma(hs_ep);
2554 		if (size_left < 0)
2555 			dev_err(hsotg->dev, "error parsing DDMA results %d\n",
2556 				size_left);
2557 	} else {
2558 		size_left = DXEPTSIZ_XFERSIZE_GET(epsize);
2559 	}
2560 
2561 	size_done = hs_ep->size_loaded - size_left;
2562 	size_done += hs_ep->last_load;
2563 
2564 	if (hs_req->req.actual != size_done)
2565 		dev_dbg(hsotg->dev, "%s: adjusting size done %d => %d\n",
2566 			__func__, hs_req->req.actual, size_done);
2567 
2568 	hs_req->req.actual = size_done;
2569 	dev_dbg(hsotg->dev, "req->length:%d req->actual:%d req->zero:%d\n",
2570 		hs_req->req.length, hs_req->req.actual, hs_req->req.zero);
2571 
2572 	if (!size_left && hs_req->req.actual < hs_req->req.length) {
2573 		dev_dbg(hsotg->dev, "%s trying more for req...\n", __func__);
2574 		dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true);
2575 		return;
2576 	}
2577 
2578 	/* Zlp for all endpoints, for ep0 only in DATA IN stage */
2579 	if (hs_ep->send_zlp) {
2580 		dwc2_hsotg_program_zlp(hsotg, hs_ep);
2581 		hs_ep->send_zlp = 0;
2582 		/* transfer will be completed on next complete interrupt */
2583 		return;
2584 	}
2585 
2586 	if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) {
2587 		/* Move to STATUS OUT */
2588 		dwc2_hsotg_ep0_zlp(hsotg, false);
2589 		return;
2590 	}
2591 
2592 	dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0);
2593 }
2594 
2595 /**
2596  * dwc2_gadget_read_ep_interrupts - reads interrupts for given ep
2597  * @hsotg: The device state.
2598  * @idx: Index of ep.
2599  * @dir_in: Endpoint direction 1-in 0-out.
2600  *
2601  * Reads for endpoint with given index and direction, by masking
2602  * epint_reg with coresponding mask.
2603  */
2604 static u32 dwc2_gadget_read_ep_interrupts(struct dwc2_hsotg *hsotg,
2605 					  unsigned int idx, int dir_in)
2606 {
2607 	u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK;
2608 	u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
2609 	u32 ints;
2610 	u32 mask;
2611 	u32 diepempmsk;
2612 
2613 	mask = dwc2_readl(hsotg->regs + epmsk_reg);
2614 	diepempmsk = dwc2_readl(hsotg->regs + DIEPEMPMSK);
2615 	mask |= ((diepempmsk >> idx) & 0x1) ? DIEPMSK_TXFIFOEMPTY : 0;
2616 	mask |= DXEPINT_SETUP_RCVD;
2617 
2618 	ints = dwc2_readl(hsotg->regs + epint_reg);
2619 	ints &= mask;
2620 	return ints;
2621 }
2622 
2623 /**
2624  * dwc2_gadget_handle_ep_disabled - handle DXEPINT_EPDISBLD
2625  * @hs_ep: The endpoint on which interrupt is asserted.
2626  *
2627  * This interrupt indicates that the endpoint has been disabled per the
2628  * application's request.
2629  *
2630  * For IN endpoints flushes txfifo, in case of BULK clears DCTL_CGNPINNAK,
2631  * in case of ISOC completes current request.
2632  *
2633  * For ISOC-OUT endpoints completes expired requests. If there is remaining
2634  * request starts it.
2635  */
2636 static void dwc2_gadget_handle_ep_disabled(struct dwc2_hsotg_ep *hs_ep)
2637 {
2638 	struct dwc2_hsotg *hsotg = hs_ep->parent;
2639 	struct dwc2_hsotg_req *hs_req;
2640 	unsigned char idx = hs_ep->index;
2641 	int dir_in = hs_ep->dir_in;
2642 	u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
2643 	int dctl = dwc2_readl(hsotg->regs + DCTL);
2644 
2645 	dev_dbg(hsotg->dev, "%s: EPDisbld\n", __func__);
2646 
2647 	if (dir_in) {
2648 		int epctl = dwc2_readl(hsotg->regs + epctl_reg);
2649 
2650 		dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index);
2651 
2652 		if (hs_ep->isochronous) {
2653 			dwc2_hsotg_complete_in(hsotg, hs_ep);
2654 			return;
2655 		}
2656 
2657 		if ((epctl & DXEPCTL_STALL) && (epctl & DXEPCTL_EPTYPE_BULK)) {
2658 			int dctl = dwc2_readl(hsotg->regs + DCTL);
2659 
2660 			dctl |= DCTL_CGNPINNAK;
2661 			dwc2_writel(dctl, hsotg->regs + DCTL);
2662 		}
2663 		return;
2664 	}
2665 
2666 	if (dctl & DCTL_GOUTNAKSTS) {
2667 		dctl |= DCTL_CGOUTNAK;
2668 		dwc2_writel(dctl, hsotg->regs + DCTL);
2669 	}
2670 
2671 	if (!hs_ep->isochronous)
2672 		return;
2673 
2674 	if (list_empty(&hs_ep->queue)) {
2675 		dev_dbg(hsotg->dev, "%s: complete_ep 0x%p, ep->queue empty!\n",
2676 			__func__, hs_ep);
2677 		return;
2678 	}
2679 
2680 	do {
2681 		hs_req = get_ep_head(hs_ep);
2682 		if (hs_req)
2683 			dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req,
2684 						    -ENODATA);
2685 		dwc2_gadget_incr_frame_num(hs_ep);
2686 	} while (dwc2_gadget_target_frame_elapsed(hs_ep));
2687 
2688 	dwc2_gadget_start_next_request(hs_ep);
2689 }
2690 
2691 /**
2692  * dwc2_gadget_handle_out_token_ep_disabled - handle DXEPINT_OUTTKNEPDIS
2693  * @hs_ep: The endpoint on which interrupt is asserted.
2694  *
2695  * This is starting point for ISOC-OUT transfer, synchronization done with
2696  * first out token received from host while corresponding EP is disabled.
2697  *
2698  * Device does not know initial frame in which out token will come. For this
2699  * HW generates OUTTKNEPDIS - out token is received while EP is disabled. Upon
2700  * getting this interrupt SW starts calculation for next transfer frame.
2701  */
2702 static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep)
2703 {
2704 	struct dwc2_hsotg *hsotg = ep->parent;
2705 	int dir_in = ep->dir_in;
2706 	u32 doepmsk;
2707 	u32 tmp;
2708 
2709 	if (dir_in || !ep->isochronous)
2710 		return;
2711 
2712 	/*
2713 	 * Store frame in which irq was asserted here, as
2714 	 * it can change while completing request below.
2715 	 */
2716 	tmp = dwc2_hsotg_read_frameno(hsotg);
2717 
2718 	dwc2_hsotg_complete_request(hsotg, ep, get_ep_head(ep), -ENODATA);
2719 
2720 	if (using_desc_dma(hsotg)) {
2721 		if (ep->target_frame == TARGET_FRAME_INITIAL) {
2722 			/* Start first ISO Out */
2723 			ep->target_frame = tmp;
2724 			dwc2_gadget_start_isoc_ddma(ep);
2725 		}
2726 		return;
2727 	}
2728 
2729 	if (ep->interval > 1 &&
2730 	    ep->target_frame == TARGET_FRAME_INITIAL) {
2731 		u32 dsts;
2732 		u32 ctrl;
2733 
2734 		dsts = dwc2_readl(hsotg->regs + DSTS);
2735 		ep->target_frame = dwc2_hsotg_read_frameno(hsotg);
2736 		dwc2_gadget_incr_frame_num(ep);
2737 
2738 		ctrl = dwc2_readl(hsotg->regs + DOEPCTL(ep->index));
2739 		if (ep->target_frame & 0x1)
2740 			ctrl |= DXEPCTL_SETODDFR;
2741 		else
2742 			ctrl |= DXEPCTL_SETEVENFR;
2743 
2744 		dwc2_writel(ctrl, hsotg->regs + DOEPCTL(ep->index));
2745 	}
2746 
2747 	dwc2_gadget_start_next_request(ep);
2748 	doepmsk = dwc2_readl(hsotg->regs + DOEPMSK);
2749 	doepmsk &= ~DOEPMSK_OUTTKNEPDISMSK;
2750 	dwc2_writel(doepmsk, hsotg->regs + DOEPMSK);
2751 }
2752 
2753 /**
2754 * dwc2_gadget_handle_nak - handle NAK interrupt
2755 * @hs_ep: The endpoint on which interrupt is asserted.
2756 *
2757 * This is starting point for ISOC-IN transfer, synchronization done with
2758 * first IN token received from host while corresponding EP is disabled.
2759 *
2760 * Device does not know when first one token will arrive from host. On first
2761 * token arrival HW generates 2 interrupts: 'in token received while FIFO empty'
2762 * and 'NAK'. NAK interrupt for ISOC-IN means that token has arrived and ZLP was
2763 * sent in response to that as there was no data in FIFO. SW is basing on this
2764 * interrupt to obtain frame in which token has come and then based on the
2765 * interval calculates next frame for transfer.
2766 */
2767 static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep)
2768 {
2769 	struct dwc2_hsotg *hsotg = hs_ep->parent;
2770 	int dir_in = hs_ep->dir_in;
2771 
2772 	if (!dir_in || !hs_ep->isochronous)
2773 		return;
2774 
2775 	if (hs_ep->target_frame == TARGET_FRAME_INITIAL) {
2776 		hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg);
2777 
2778 		if (using_desc_dma(hsotg)) {
2779 			dwc2_gadget_start_isoc_ddma(hs_ep);
2780 			return;
2781 		}
2782 
2783 		if (hs_ep->interval > 1) {
2784 			u32 ctrl = dwc2_readl(hsotg->regs +
2785 					      DIEPCTL(hs_ep->index));
2786 			if (hs_ep->target_frame & 0x1)
2787 				ctrl |= DXEPCTL_SETODDFR;
2788 			else
2789 				ctrl |= DXEPCTL_SETEVENFR;
2790 
2791 			dwc2_writel(ctrl, hsotg->regs + DIEPCTL(hs_ep->index));
2792 		}
2793 
2794 		dwc2_hsotg_complete_request(hsotg, hs_ep,
2795 					    get_ep_head(hs_ep), 0);
2796 	}
2797 
2798 	dwc2_gadget_incr_frame_num(hs_ep);
2799 }
2800 
2801 /**
2802  * dwc2_hsotg_epint - handle an in/out endpoint interrupt
2803  * @hsotg: The driver state
2804  * @idx: The index for the endpoint (0..15)
2805  * @dir_in: Set if this is an IN endpoint
2806  *
2807  * Process and clear any interrupt pending for an individual endpoint
2808  */
2809 static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx,
2810 			    int dir_in)
2811 {
2812 	struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in);
2813 	u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx);
2814 	u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx);
2815 	u32 epsiz_reg = dir_in ? DIEPTSIZ(idx) : DOEPTSIZ(idx);
2816 	u32 ints;
2817 	u32 ctrl;
2818 
2819 	ints = dwc2_gadget_read_ep_interrupts(hsotg, idx, dir_in);
2820 	ctrl = dwc2_readl(hsotg->regs + epctl_reg);
2821 
2822 	/* Clear endpoint interrupts */
2823 	dwc2_writel(ints, hsotg->regs + epint_reg);
2824 
2825 	if (!hs_ep) {
2826 		dev_err(hsotg->dev, "%s:Interrupt for unconfigured ep%d(%s)\n",
2827 					__func__, idx, dir_in ? "in" : "out");
2828 		return;
2829 	}
2830 
2831 	dev_dbg(hsotg->dev, "%s: ep%d(%s) DxEPINT=0x%08x\n",
2832 		__func__, idx, dir_in ? "in" : "out", ints);
2833 
2834 	/* Don't process XferCompl interrupt if it is a setup packet */
2835 	if (idx == 0 && (ints & (DXEPINT_SETUP | DXEPINT_SETUP_RCVD)))
2836 		ints &= ~DXEPINT_XFERCOMPL;
2837 
2838 	/*
2839 	 * Don't process XferCompl interrupt in DDMA if EP0 is still in SETUP
2840 	 * stage and xfercomplete was generated without SETUP phase done
2841 	 * interrupt. SW should parse received setup packet only after host's
2842 	 * exit from setup phase of control transfer.
2843 	 */
2844 	if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in &&
2845 	    hsotg->ep0_state == DWC2_EP0_SETUP && !(ints & DXEPINT_SETUP))
2846 		ints &= ~DXEPINT_XFERCOMPL;
2847 
2848 	if (ints & DXEPINT_XFERCOMPL) {
2849 		dev_dbg(hsotg->dev,
2850 			"%s: XferCompl: DxEPCTL=0x%08x, DXEPTSIZ=%08x\n",
2851 			__func__, dwc2_readl(hsotg->regs + epctl_reg),
2852 			dwc2_readl(hsotg->regs + epsiz_reg));
2853 
2854 		/* In DDMA handle isochronous requests separately */
2855 		if (using_desc_dma(hsotg) && hs_ep->isochronous) {
2856 			dwc2_gadget_complete_isoc_request_ddma(hs_ep);
2857 			/* Try to start next isoc request */
2858 			dwc2_gadget_start_next_isoc_ddma(hs_ep);
2859 		} else if (dir_in) {
2860 			/*
2861 			 * We get OutDone from the FIFO, so we only
2862 			 * need to look at completing IN requests here
2863 			 * if operating slave mode
2864 			 */
2865 			if (hs_ep->isochronous && hs_ep->interval > 1)
2866 				dwc2_gadget_incr_frame_num(hs_ep);
2867 
2868 			dwc2_hsotg_complete_in(hsotg, hs_ep);
2869 			if (ints & DXEPINT_NAKINTRPT)
2870 				ints &= ~DXEPINT_NAKINTRPT;
2871 
2872 			if (idx == 0 && !hs_ep->req)
2873 				dwc2_hsotg_enqueue_setup(hsotg);
2874 		} else if (using_dma(hsotg)) {
2875 			/*
2876 			 * We're using DMA, we need to fire an OutDone here
2877 			 * as we ignore the RXFIFO.
2878 			 */
2879 			if (hs_ep->isochronous && hs_ep->interval > 1)
2880 				dwc2_gadget_incr_frame_num(hs_ep);
2881 
2882 			dwc2_hsotg_handle_outdone(hsotg, idx);
2883 		}
2884 	}
2885 
2886 	if (ints & DXEPINT_EPDISBLD)
2887 		dwc2_gadget_handle_ep_disabled(hs_ep);
2888 
2889 	if (ints & DXEPINT_OUTTKNEPDIS)
2890 		dwc2_gadget_handle_out_token_ep_disabled(hs_ep);
2891 
2892 	if (ints & DXEPINT_NAKINTRPT)
2893 		dwc2_gadget_handle_nak(hs_ep);
2894 
2895 	if (ints & DXEPINT_AHBERR)
2896 		dev_dbg(hsotg->dev, "%s: AHBErr\n", __func__);
2897 
2898 	if (ints & DXEPINT_SETUP) {  /* Setup or Timeout */
2899 		dev_dbg(hsotg->dev, "%s: Setup/Timeout\n",  __func__);
2900 
2901 		if (using_dma(hsotg) && idx == 0) {
2902 			/*
2903 			 * this is the notification we've received a
2904 			 * setup packet. In non-DMA mode we'd get this
2905 			 * from the RXFIFO, instead we need to process
2906 			 * the setup here.
2907 			 */
2908 
2909 			if (dir_in)
2910 				WARN_ON_ONCE(1);
2911 			else
2912 				dwc2_hsotg_handle_outdone(hsotg, 0);
2913 		}
2914 	}
2915 
2916 	if (ints & DXEPINT_STSPHSERCVD) {
2917 		dev_dbg(hsotg->dev, "%s: StsPhseRcvd\n", __func__);
2918 
2919 		/* Move to STATUS IN for DDMA */
2920 		if (using_desc_dma(hsotg))
2921 			dwc2_hsotg_ep0_zlp(hsotg, true);
2922 	}
2923 
2924 	if (ints & DXEPINT_BACK2BACKSETUP)
2925 		dev_dbg(hsotg->dev, "%s: B2BSetup/INEPNakEff\n", __func__);
2926 
2927 	if (ints & DXEPINT_BNAINTR) {
2928 		dev_dbg(hsotg->dev, "%s: BNA interrupt\n", __func__);
2929 
2930 		/*
2931 		 * Try to start next isoc request, if any.
2932 		 * Sometimes the endpoint remains enabled after BNA interrupt
2933 		 * assertion, which is not expected, hence we can enter here
2934 		 * couple of times.
2935 		 */
2936 		if (hs_ep->isochronous)
2937 			dwc2_gadget_start_next_isoc_ddma(hs_ep);
2938 	}
2939 
2940 	if (dir_in && !hs_ep->isochronous) {
2941 		/* not sure if this is important, but we'll clear it anyway */
2942 		if (ints & DXEPINT_INTKNTXFEMP) {
2943 			dev_dbg(hsotg->dev, "%s: ep%d: INTknTXFEmpMsk\n",
2944 				__func__, idx);
2945 		}
2946 
2947 		/* this probably means something bad is happening */
2948 		if (ints & DXEPINT_INTKNEPMIS) {
2949 			dev_warn(hsotg->dev, "%s: ep%d: INTknEP\n",
2950 				 __func__, idx);
2951 		}
2952 
2953 		/* FIFO has space or is empty (see GAHBCFG) */
2954 		if (hsotg->dedicated_fifos &&
2955 		    ints & DXEPINT_TXFEMP) {
2956 			dev_dbg(hsotg->dev, "%s: ep%d: TxFIFOEmpty\n",
2957 				__func__, idx);
2958 			if (!using_dma(hsotg))
2959 				dwc2_hsotg_trytx(hsotg, hs_ep);
2960 		}
2961 	}
2962 }
2963 
2964 /**
2965  * dwc2_hsotg_irq_enumdone - Handle EnumDone interrupt (enumeration done)
2966  * @hsotg: The device state.
2967  *
2968  * Handle updating the device settings after the enumeration phase has
2969  * been completed.
2970  */
2971 static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg)
2972 {
2973 	u32 dsts = dwc2_readl(hsotg->regs + DSTS);
2974 	int ep0_mps = 0, ep_mps = 8;
2975 
2976 	/*
2977 	 * This should signal the finish of the enumeration phase
2978 	 * of the USB handshaking, so we should now know what rate
2979 	 * we connected at.
2980 	 */
2981 
2982 	dev_dbg(hsotg->dev, "EnumDone (DSTS=0x%08x)\n", dsts);
2983 
2984 	/*
2985 	 * note, since we're limited by the size of transfer on EP0, and
2986 	 * it seems IN transfers must be a even number of packets we do
2987 	 * not advertise a 64byte MPS on EP0.
2988 	 */
2989 
2990 	/* catch both EnumSpd_FS and EnumSpd_FS48 */
2991 	switch ((dsts & DSTS_ENUMSPD_MASK) >> DSTS_ENUMSPD_SHIFT) {
2992 	case DSTS_ENUMSPD_FS:
2993 	case DSTS_ENUMSPD_FS48:
2994 		hsotg->gadget.speed = USB_SPEED_FULL;
2995 		ep0_mps = EP0_MPS_LIMIT;
2996 		ep_mps = 1023;
2997 		break;
2998 
2999 	case DSTS_ENUMSPD_HS:
3000 		hsotg->gadget.speed = USB_SPEED_HIGH;
3001 		ep0_mps = EP0_MPS_LIMIT;
3002 		ep_mps = 1024;
3003 		break;
3004 
3005 	case DSTS_ENUMSPD_LS:
3006 		hsotg->gadget.speed = USB_SPEED_LOW;
3007 		ep0_mps = 8;
3008 		ep_mps = 8;
3009 		/*
3010 		 * note, we don't actually support LS in this driver at the
3011 		 * moment, and the documentation seems to imply that it isn't
3012 		 * supported by the PHYs on some of the devices.
3013 		 */
3014 		break;
3015 	}
3016 	dev_info(hsotg->dev, "new device is %s\n",
3017 		 usb_speed_string(hsotg->gadget.speed));
3018 
3019 	/*
3020 	 * we should now know the maximum packet size for an
3021 	 * endpoint, so set the endpoints to a default value.
3022 	 */
3023 
3024 	if (ep0_mps) {
3025 		int i;
3026 		/* Initialize ep0 for both in and out directions */
3027 		dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 1);
3028 		dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 0);
3029 		for (i = 1; i < hsotg->num_of_eps; i++) {
3030 			if (hsotg->eps_in[i])
3031 				dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
3032 							    0, 1);
3033 			if (hsotg->eps_out[i])
3034 				dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps,
3035 							    0, 0);
3036 		}
3037 	}
3038 
3039 	/* ensure after enumeration our EP0 is active */
3040 
3041 	dwc2_hsotg_enqueue_setup(hsotg);
3042 
3043 	dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
3044 		dwc2_readl(hsotg->regs + DIEPCTL0),
3045 		dwc2_readl(hsotg->regs + DOEPCTL0));
3046 }
3047 
3048 /**
3049  * kill_all_requests - remove all requests from the endpoint's queue
3050  * @hsotg: The device state.
3051  * @ep: The endpoint the requests may be on.
3052  * @result: The result code to use.
3053  *
3054  * Go through the requests on the given endpoint and mark them
3055  * completed with the given result code.
3056  */
3057 static void kill_all_requests(struct dwc2_hsotg *hsotg,
3058 			      struct dwc2_hsotg_ep *ep,
3059 			      int result)
3060 {
3061 	struct dwc2_hsotg_req *req, *treq;
3062 	unsigned size;
3063 
3064 	ep->req = NULL;
3065 
3066 	list_for_each_entry_safe(req, treq, &ep->queue, queue)
3067 		dwc2_hsotg_complete_request(hsotg, ep, req,
3068 					   result);
3069 
3070 	if (!hsotg->dedicated_fifos)
3071 		return;
3072 	size = (dwc2_readl(hsotg->regs + DTXFSTS(ep->fifo_index)) & 0xffff) * 4;
3073 	if (size < ep->fifo_size)
3074 		dwc2_hsotg_txfifo_flush(hsotg, ep->fifo_index);
3075 }
3076 
3077 /**
3078  * dwc2_hsotg_disconnect - disconnect service
3079  * @hsotg: The device state.
3080  *
3081  * The device has been disconnected. Remove all current
3082  * transactions and signal the gadget driver that this
3083  * has happened.
3084  */
3085 void dwc2_hsotg_disconnect(struct dwc2_hsotg *hsotg)
3086 {
3087 	unsigned ep;
3088 
3089 	if (!hsotg->connected)
3090 		return;
3091 
3092 	hsotg->connected = 0;
3093 	hsotg->test_mode = 0;
3094 
3095 	for (ep = 0; ep < hsotg->num_of_eps; ep++) {
3096 		if (hsotg->eps_in[ep])
3097 			kill_all_requests(hsotg, hsotg->eps_in[ep],
3098 								-ESHUTDOWN);
3099 		if (hsotg->eps_out[ep])
3100 			kill_all_requests(hsotg, hsotg->eps_out[ep],
3101 								-ESHUTDOWN);
3102 	}
3103 
3104 	call_gadget(hsotg, disconnect);
3105 	hsotg->lx_state = DWC2_L3;
3106 }
3107 
3108 /**
3109  * dwc2_hsotg_irq_fifoempty - TX FIFO empty interrupt handler
3110  * @hsotg: The device state:
3111  * @periodic: True if this is a periodic FIFO interrupt
3112  */
3113 static void dwc2_hsotg_irq_fifoempty(struct dwc2_hsotg *hsotg, bool periodic)
3114 {
3115 	struct dwc2_hsotg_ep *ep;
3116 	int epno, ret;
3117 
3118 	/* look through for any more data to transmit */
3119 	for (epno = 0; epno < hsotg->num_of_eps; epno++) {
3120 		ep = index_to_ep(hsotg, epno, 1);
3121 
3122 		if (!ep)
3123 			continue;
3124 
3125 		if (!ep->dir_in)
3126 			continue;
3127 
3128 		if ((periodic && !ep->periodic) ||
3129 		    (!periodic && ep->periodic))
3130 			continue;
3131 
3132 		ret = dwc2_hsotg_trytx(hsotg, ep);
3133 		if (ret < 0)
3134 			break;
3135 	}
3136 }
3137 
3138 /* IRQ flags which will trigger a retry around the IRQ loop */
3139 #define IRQ_RETRY_MASK (GINTSTS_NPTXFEMP | \
3140 			GINTSTS_PTXFEMP |  \
3141 			GINTSTS_RXFLVL)
3142 
3143 /**
3144  * dwc2_hsotg_core_init - issue softreset to the core
3145  * @hsotg: The device state
3146  *
3147  * Issue a soft reset to the core, and await the core finishing it.
3148  */
3149 void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg,
3150 						bool is_usb_reset)
3151 {
3152 	u32 intmsk;
3153 	u32 val;
3154 	u32 usbcfg;
3155 	u32 dcfg = 0;
3156 
3157 	/* Kill any ep0 requests as controller will be reinitialized */
3158 	kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET);
3159 
3160 	if (!is_usb_reset)
3161 		if (dwc2_core_reset(hsotg))
3162 			return;
3163 
3164 	/*
3165 	 * we must now enable ep0 ready for host detection and then
3166 	 * set configuration.
3167 	 */
3168 
3169 	/* keep other bits untouched (so e.g. forced modes are not lost) */
3170 	usbcfg = dwc2_readl(hsotg->regs + GUSBCFG);
3171 	usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP |
3172 		GUSBCFG_HNPCAP);
3173 
3174 	if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS &&
3175 	    (hsotg->params.speed == DWC2_SPEED_PARAM_FULL ||
3176 	     hsotg->params.speed == DWC2_SPEED_PARAM_LOW)) {
3177 		/* FS/LS Dedicated Transceiver Interface */
3178 		usbcfg |= GUSBCFG_PHYSEL;
3179 	} else {
3180 		/* set the PLL on, remove the HNP/SRP and set the PHY */
3181 		val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5;
3182 		usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) |
3183 			(val << GUSBCFG_USBTRDTIM_SHIFT);
3184 	}
3185 	dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
3186 
3187 	dwc2_hsotg_init_fifo(hsotg);
3188 
3189 	if (!is_usb_reset)
3190 		__orr32(hsotg->regs + DCTL, DCTL_SFTDISCON);
3191 
3192 	dcfg |= DCFG_EPMISCNT(1);
3193 
3194 	switch (hsotg->params.speed) {
3195 	case DWC2_SPEED_PARAM_LOW:
3196 		dcfg |= DCFG_DEVSPD_LS;
3197 		break;
3198 	case DWC2_SPEED_PARAM_FULL:
3199 		if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS)
3200 			dcfg |= DCFG_DEVSPD_FS48;
3201 		else
3202 			dcfg |= DCFG_DEVSPD_FS;
3203 		break;
3204 	default:
3205 		dcfg |= DCFG_DEVSPD_HS;
3206 	}
3207 
3208 	dwc2_writel(dcfg,  hsotg->regs + DCFG);
3209 
3210 	/* Clear any pending OTG interrupts */
3211 	dwc2_writel(0xffffffff, hsotg->regs + GOTGINT);
3212 
3213 	/* Clear any pending interrupts */
3214 	dwc2_writel(0xffffffff, hsotg->regs + GINTSTS);
3215 	intmsk = GINTSTS_ERLYSUSP | GINTSTS_SESSREQINT |
3216 		GINTSTS_GOUTNAKEFF | GINTSTS_GINNAKEFF |
3217 		GINTSTS_USBRST | GINTSTS_RESETDET |
3218 		GINTSTS_ENUMDONE | GINTSTS_OTGINT |
3219 		GINTSTS_USBSUSP | GINTSTS_WKUPINT;
3220 
3221 	if (!using_desc_dma(hsotg))
3222 		intmsk |= GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT;
3223 
3224 	if (hsotg->params.external_id_pin_ctl <= 0)
3225 		intmsk |= GINTSTS_CONIDSTSCHNG;
3226 
3227 	dwc2_writel(intmsk, hsotg->regs + GINTMSK);
3228 
3229 	if (using_dma(hsotg)) {
3230 		dwc2_writel(GAHBCFG_GLBL_INTR_EN | GAHBCFG_DMA_EN |
3231 			    (GAHBCFG_HBSTLEN_INCR4 << GAHBCFG_HBSTLEN_SHIFT),
3232 			    hsotg->regs + GAHBCFG);
3233 
3234 		/* Set DDMA mode support in the core if needed */
3235 		if (using_desc_dma(hsotg))
3236 			__orr32(hsotg->regs + DCFG, DCFG_DESCDMA_EN);
3237 
3238 	} else {
3239 		dwc2_writel(((hsotg->dedicated_fifos) ?
3240 						(GAHBCFG_NP_TXF_EMP_LVL |
3241 						 GAHBCFG_P_TXF_EMP_LVL) : 0) |
3242 			    GAHBCFG_GLBL_INTR_EN, hsotg->regs + GAHBCFG);
3243 	}
3244 
3245 	/*
3246 	 * If INTknTXFEmpMsk is enabled, it's important to disable ep interrupts
3247 	 * when we have no data to transfer. Otherwise we get being flooded by
3248 	 * interrupts.
3249 	 */
3250 
3251 	dwc2_writel(((hsotg->dedicated_fifos && !using_dma(hsotg)) ?
3252 		DIEPMSK_TXFIFOEMPTY | DIEPMSK_INTKNTXFEMPMSK : 0) |
3253 		DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK |
3254 		DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK,
3255 		hsotg->regs + DIEPMSK);
3256 
3257 	/*
3258 	 * don't need XferCompl, we get that from RXFIFO in slave mode. In
3259 	 * DMA mode we may need this and StsPhseRcvd.
3260 	 */
3261 	dwc2_writel((using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK |
3262 		DOEPMSK_STSPHSERCVDMSK) : 0) |
3263 		DOEPMSK_EPDISBLDMSK | DOEPMSK_AHBERRMSK |
3264 		DOEPMSK_SETUPMSK,
3265 		hsotg->regs + DOEPMSK);
3266 
3267 	/* Enable BNA interrupt for DDMA */
3268 	if (using_desc_dma(hsotg))
3269 		__orr32(hsotg->regs + DOEPMSK, DOEPMSK_BNAMSK);
3270 
3271 	dwc2_writel(0, hsotg->regs + DAINTMSK);
3272 
3273 	dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
3274 		dwc2_readl(hsotg->regs + DIEPCTL0),
3275 		dwc2_readl(hsotg->regs + DOEPCTL0));
3276 
3277 	/* enable in and out endpoint interrupts */
3278 	dwc2_hsotg_en_gsint(hsotg, GINTSTS_OEPINT | GINTSTS_IEPINT);
3279 
3280 	/*
3281 	 * Enable the RXFIFO when in slave mode, as this is how we collect
3282 	 * the data. In DMA mode, we get events from the FIFO but also
3283 	 * things we cannot process, so do not use it.
3284 	 */
3285 	if (!using_dma(hsotg))
3286 		dwc2_hsotg_en_gsint(hsotg, GINTSTS_RXFLVL);
3287 
3288 	/* Enable interrupts for EP0 in and out */
3289 	dwc2_hsotg_ctrl_epint(hsotg, 0, 0, 1);
3290 	dwc2_hsotg_ctrl_epint(hsotg, 0, 1, 1);
3291 
3292 	if (!is_usb_reset) {
3293 		__orr32(hsotg->regs + DCTL, DCTL_PWRONPRGDONE);
3294 		udelay(10);  /* see openiboot */
3295 		__bic32(hsotg->regs + DCTL, DCTL_PWRONPRGDONE);
3296 	}
3297 
3298 	dev_dbg(hsotg->dev, "DCTL=0x%08x\n", dwc2_readl(hsotg->regs + DCTL));
3299 
3300 	/*
3301 	 * DxEPCTL_USBActEp says RO in manual, but seems to be set by
3302 	 * writing to the EPCTL register..
3303 	 */
3304 
3305 	/* set to read 1 8byte packet */
3306 	dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) |
3307 	       DXEPTSIZ_XFERSIZE(8), hsotg->regs + DOEPTSIZ0);
3308 
3309 	dwc2_writel(dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
3310 	       DXEPCTL_CNAK | DXEPCTL_EPENA |
3311 	       DXEPCTL_USBACTEP,
3312 	       hsotg->regs + DOEPCTL0);
3313 
3314 	/* enable, but don't activate EP0in */
3315 	dwc2_writel(dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) |
3316 	       DXEPCTL_USBACTEP, hsotg->regs + DIEPCTL0);
3317 
3318 	dwc2_hsotg_enqueue_setup(hsotg);
3319 
3320 	dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n",
3321 		dwc2_readl(hsotg->regs + DIEPCTL0),
3322 		dwc2_readl(hsotg->regs + DOEPCTL0));
3323 
3324 	/* clear global NAKs */
3325 	val = DCTL_CGOUTNAK | DCTL_CGNPINNAK;
3326 	if (!is_usb_reset)
3327 		val |= DCTL_SFTDISCON;
3328 	__orr32(hsotg->regs + DCTL, val);
3329 
3330 	/* must be at-least 3ms to allow bus to see disconnect */
3331 	mdelay(3);
3332 
3333 	hsotg->lx_state = DWC2_L0;
3334 }
3335 
3336 static void dwc2_hsotg_core_disconnect(struct dwc2_hsotg *hsotg)
3337 {
3338 	/* set the soft-disconnect bit */
3339 	__orr32(hsotg->regs + DCTL, DCTL_SFTDISCON);
3340 }
3341 
3342 void dwc2_hsotg_core_connect(struct dwc2_hsotg *hsotg)
3343 {
3344 	/* remove the soft-disconnect and let's go */
3345 	__bic32(hsotg->regs + DCTL, DCTL_SFTDISCON);
3346 }
3347 
3348 /**
3349  * dwc2_gadget_handle_incomplete_isoc_in - handle incomplete ISO IN Interrupt.
3350  * @hsotg: The device state:
3351  *
3352  * This interrupt indicates one of the following conditions occurred while
3353  * transmitting an ISOC transaction.
3354  * - Corrupted IN Token for ISOC EP.
3355  * - Packet not complete in FIFO.
3356  *
3357  * The following actions will be taken:
3358  * - Determine the EP
3359  * - Disable EP; when 'Endpoint Disabled' interrupt is received Flush FIFO
3360  */
3361 static void dwc2_gadget_handle_incomplete_isoc_in(struct dwc2_hsotg *hsotg)
3362 {
3363 	struct dwc2_hsotg_ep *hs_ep;
3364 	u32 epctrl;
3365 	u32 idx;
3366 
3367 	dev_dbg(hsotg->dev, "Incomplete isoc in interrupt received:\n");
3368 
3369 	for (idx = 1; idx <= hsotg->num_of_eps; idx++) {
3370 		hs_ep = hsotg->eps_in[idx];
3371 		epctrl = dwc2_readl(hsotg->regs + DIEPCTL(idx));
3372 		if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous &&
3373 		    dwc2_gadget_target_frame_elapsed(hs_ep)) {
3374 			epctrl |= DXEPCTL_SNAK;
3375 			epctrl |= DXEPCTL_EPDIS;
3376 			dwc2_writel(epctrl, hsotg->regs + DIEPCTL(idx));
3377 		}
3378 	}
3379 
3380 	/* Clear interrupt */
3381 	dwc2_writel(GINTSTS_INCOMPL_SOIN, hsotg->regs + GINTSTS);
3382 }
3383 
3384 /**
3385  * dwc2_gadget_handle_incomplete_isoc_out - handle incomplete ISO OUT Interrupt
3386  * @hsotg: The device state:
3387  *
3388  * This interrupt indicates one of the following conditions occurred while
3389  * transmitting an ISOC transaction.
3390  * - Corrupted OUT Token for ISOC EP.
3391  * - Packet not complete in FIFO.
3392  *
3393  * The following actions will be taken:
3394  * - Determine the EP
3395  * - Set DCTL_SGOUTNAK and unmask GOUTNAKEFF if target frame elapsed.
3396  */
3397 static void dwc2_gadget_handle_incomplete_isoc_out(struct dwc2_hsotg *hsotg)
3398 {
3399 	u32 gintsts;
3400 	u32 gintmsk;
3401 	u32 epctrl;
3402 	struct dwc2_hsotg_ep *hs_ep;
3403 	int idx;
3404 
3405 	dev_dbg(hsotg->dev, "%s: GINTSTS_INCOMPL_SOOUT\n", __func__);
3406 
3407 	for (idx = 1; idx <= hsotg->num_of_eps; idx++) {
3408 		hs_ep = hsotg->eps_out[idx];
3409 		epctrl = dwc2_readl(hsotg->regs + DOEPCTL(idx));
3410 		if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous &&
3411 		    dwc2_gadget_target_frame_elapsed(hs_ep)) {
3412 			/* Unmask GOUTNAKEFF interrupt */
3413 			gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
3414 			gintmsk |= GINTSTS_GOUTNAKEFF;
3415 			dwc2_writel(gintmsk, hsotg->regs + GINTMSK);
3416 
3417 			gintsts = dwc2_readl(hsotg->regs + GINTSTS);
3418 			if (!(gintsts & GINTSTS_GOUTNAKEFF))
3419 				__orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK);
3420 		}
3421 	}
3422 
3423 	/* Clear interrupt */
3424 	dwc2_writel(GINTSTS_INCOMPL_SOOUT, hsotg->regs + GINTSTS);
3425 }
3426 
3427 /**
3428  * dwc2_hsotg_irq - handle device interrupt
3429  * @irq: The IRQ number triggered
3430  * @pw: The pw value when registered the handler.
3431  */
3432 static irqreturn_t dwc2_hsotg_irq(int irq, void *pw)
3433 {
3434 	struct dwc2_hsotg *hsotg = pw;
3435 	int retry_count = 8;
3436 	u32 gintsts;
3437 	u32 gintmsk;
3438 
3439 	if (!dwc2_is_device_mode(hsotg))
3440 		return IRQ_NONE;
3441 
3442 	spin_lock(&hsotg->lock);
3443 irq_retry:
3444 	gintsts = dwc2_readl(hsotg->regs + GINTSTS);
3445 	gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
3446 
3447 	dev_dbg(hsotg->dev, "%s: %08x %08x (%08x) retry %d\n",
3448 		__func__, gintsts, gintsts & gintmsk, gintmsk, retry_count);
3449 
3450 	gintsts &= gintmsk;
3451 
3452 	if (gintsts & GINTSTS_RESETDET) {
3453 		dev_dbg(hsotg->dev, "%s: USBRstDet\n", __func__);
3454 
3455 		dwc2_writel(GINTSTS_RESETDET, hsotg->regs + GINTSTS);
3456 
3457 		/* This event must be used only if controller is suspended */
3458 		if (hsotg->lx_state == DWC2_L2) {
3459 			dwc2_exit_hibernation(hsotg, true);
3460 			hsotg->lx_state = DWC2_L0;
3461 		}
3462 	}
3463 
3464 	if (gintsts & (GINTSTS_USBRST | GINTSTS_RESETDET)) {
3465 
3466 		u32 usb_status = dwc2_readl(hsotg->regs + GOTGCTL);
3467 		u32 connected = hsotg->connected;
3468 
3469 		dev_dbg(hsotg->dev, "%s: USBRst\n", __func__);
3470 		dev_dbg(hsotg->dev, "GNPTXSTS=%08x\n",
3471 			dwc2_readl(hsotg->regs + GNPTXSTS));
3472 
3473 		dwc2_writel(GINTSTS_USBRST, hsotg->regs + GINTSTS);
3474 
3475 		/* Report disconnection if it is not already done. */
3476 		dwc2_hsotg_disconnect(hsotg);
3477 
3478 		if (usb_status & GOTGCTL_BSESVLD && connected)
3479 			dwc2_hsotg_core_init_disconnected(hsotg, true);
3480 	}
3481 
3482 	if (gintsts & GINTSTS_ENUMDONE) {
3483 		dwc2_writel(GINTSTS_ENUMDONE, hsotg->regs + GINTSTS);
3484 
3485 		dwc2_hsotg_irq_enumdone(hsotg);
3486 	}
3487 
3488 	if (gintsts & (GINTSTS_OEPINT | GINTSTS_IEPINT)) {
3489 		u32 daint = dwc2_readl(hsotg->regs + DAINT);
3490 		u32 daintmsk = dwc2_readl(hsotg->regs + DAINTMSK);
3491 		u32 daint_out, daint_in;
3492 		int ep;
3493 
3494 		daint &= daintmsk;
3495 		daint_out = daint >> DAINT_OUTEP_SHIFT;
3496 		daint_in = daint & ~(daint_out << DAINT_OUTEP_SHIFT);
3497 
3498 		dev_dbg(hsotg->dev, "%s: daint=%08x\n", __func__, daint);
3499 
3500 		for (ep = 0; ep < hsotg->num_of_eps && daint_out;
3501 						ep++, daint_out >>= 1) {
3502 			if (daint_out & 1)
3503 				dwc2_hsotg_epint(hsotg, ep, 0);
3504 		}
3505 
3506 		for (ep = 0; ep < hsotg->num_of_eps  && daint_in;
3507 						ep++, daint_in >>= 1) {
3508 			if (daint_in & 1)
3509 				dwc2_hsotg_epint(hsotg, ep, 1);
3510 		}
3511 	}
3512 
3513 	/* check both FIFOs */
3514 
3515 	if (gintsts & GINTSTS_NPTXFEMP) {
3516 		dev_dbg(hsotg->dev, "NPTxFEmp\n");
3517 
3518 		/*
3519 		 * Disable the interrupt to stop it happening again
3520 		 * unless one of these endpoint routines decides that
3521 		 * it needs re-enabling
3522 		 */
3523 
3524 		dwc2_hsotg_disable_gsint(hsotg, GINTSTS_NPTXFEMP);
3525 		dwc2_hsotg_irq_fifoempty(hsotg, false);
3526 	}
3527 
3528 	if (gintsts & GINTSTS_PTXFEMP) {
3529 		dev_dbg(hsotg->dev, "PTxFEmp\n");
3530 
3531 		/* See note in GINTSTS_NPTxFEmp */
3532 
3533 		dwc2_hsotg_disable_gsint(hsotg, GINTSTS_PTXFEMP);
3534 		dwc2_hsotg_irq_fifoempty(hsotg, true);
3535 	}
3536 
3537 	if (gintsts & GINTSTS_RXFLVL) {
3538 		/*
3539 		 * note, since GINTSTS_RxFLvl doubles as FIFO-not-empty,
3540 		 * we need to retry dwc2_hsotg_handle_rx if this is still
3541 		 * set.
3542 		 */
3543 
3544 		dwc2_hsotg_handle_rx(hsotg);
3545 	}
3546 
3547 	if (gintsts & GINTSTS_ERLYSUSP) {
3548 		dev_dbg(hsotg->dev, "GINTSTS_ErlySusp\n");
3549 		dwc2_writel(GINTSTS_ERLYSUSP, hsotg->regs + GINTSTS);
3550 	}
3551 
3552 	/*
3553 	 * these next two seem to crop-up occasionally causing the core
3554 	 * to shutdown the USB transfer, so try clearing them and logging
3555 	 * the occurrence.
3556 	 */
3557 
3558 	if (gintsts & GINTSTS_GOUTNAKEFF) {
3559 		u8 idx;
3560 		u32 epctrl;
3561 		u32 gintmsk;
3562 		struct dwc2_hsotg_ep *hs_ep;
3563 
3564 		/* Mask this interrupt */
3565 		gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
3566 		gintmsk &= ~GINTSTS_GOUTNAKEFF;
3567 		dwc2_writel(gintmsk, hsotg->regs + GINTMSK);
3568 
3569 		dev_dbg(hsotg->dev, "GOUTNakEff triggered\n");
3570 		for (idx = 1; idx <= hsotg->num_of_eps; idx++) {
3571 			hs_ep = hsotg->eps_out[idx];
3572 			epctrl = dwc2_readl(hsotg->regs + DOEPCTL(idx));
3573 
3574 			if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous) {
3575 				epctrl |= DXEPCTL_SNAK;
3576 				epctrl |= DXEPCTL_EPDIS;
3577 				dwc2_writel(epctrl, hsotg->regs + DOEPCTL(idx));
3578 			}
3579 		}
3580 
3581 		/* This interrupt bit is cleared in DXEPINT_EPDISBLD handler */
3582 	}
3583 
3584 	if (gintsts & GINTSTS_GINNAKEFF) {
3585 		dev_info(hsotg->dev, "GINNakEff triggered\n");
3586 
3587 		__orr32(hsotg->regs + DCTL, DCTL_CGNPINNAK);
3588 
3589 		dwc2_hsotg_dump(hsotg);
3590 	}
3591 
3592 	if (gintsts & GINTSTS_INCOMPL_SOIN)
3593 		dwc2_gadget_handle_incomplete_isoc_in(hsotg);
3594 
3595 	if (gintsts & GINTSTS_INCOMPL_SOOUT)
3596 		dwc2_gadget_handle_incomplete_isoc_out(hsotg);
3597 
3598 	/*
3599 	 * if we've had fifo events, we should try and go around the
3600 	 * loop again to see if there's any point in returning yet.
3601 	 */
3602 
3603 	if (gintsts & IRQ_RETRY_MASK && --retry_count > 0)
3604 			goto irq_retry;
3605 
3606 	spin_unlock(&hsotg->lock);
3607 
3608 	return IRQ_HANDLED;
3609 }
3610 
3611 static int dwc2_hsotg_wait_bit_set(struct dwc2_hsotg *hs_otg, u32 reg,
3612 				   u32 bit, u32 timeout)
3613 {
3614 	u32 i;
3615 
3616 	for (i = 0; i < timeout; i++) {
3617 		if (dwc2_readl(hs_otg->regs + reg) & bit)
3618 			return 0;
3619 		udelay(1);
3620 	}
3621 
3622 	return -ETIMEDOUT;
3623 }
3624 
3625 static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg,
3626 				   struct dwc2_hsotg_ep *hs_ep)
3627 {
3628 	u32 epctrl_reg;
3629 	u32 epint_reg;
3630 
3631 	epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) :
3632 		DOEPCTL(hs_ep->index);
3633 	epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) :
3634 		DOEPINT(hs_ep->index);
3635 
3636 	dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__,
3637 		hs_ep->name);
3638 
3639 	if (hs_ep->dir_in) {
3640 		if (hsotg->dedicated_fifos || hs_ep->periodic) {
3641 			__orr32(hsotg->regs + epctrl_reg, DXEPCTL_SNAK);
3642 			/* Wait for Nak effect */
3643 			if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg,
3644 						    DXEPINT_INEPNAKEFF, 100))
3645 				dev_warn(hsotg->dev,
3646 					 "%s: timeout DIEPINT.NAKEFF\n",
3647 					 __func__);
3648 		} else {
3649 			__orr32(hsotg->regs + DCTL, DCTL_SGNPINNAK);
3650 			/* Wait for Nak effect */
3651 			if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3652 						    GINTSTS_GINNAKEFF, 100))
3653 				dev_warn(hsotg->dev,
3654 					 "%s: timeout GINTSTS.GINNAKEFF\n",
3655 					 __func__);
3656 		}
3657 	} else {
3658 		if (!(dwc2_readl(hsotg->regs + GINTSTS) & GINTSTS_GOUTNAKEFF))
3659 			__orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK);
3660 
3661 		/* Wait for global nak to take effect */
3662 		if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS,
3663 					    GINTSTS_GOUTNAKEFF, 100))
3664 			dev_warn(hsotg->dev, "%s: timeout GINTSTS.GOUTNAKEFF\n",
3665 				 __func__);
3666 	}
3667 
3668 	/* Disable ep */
3669 	__orr32(hsotg->regs + epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK);
3670 
3671 	/* Wait for ep to be disabled */
3672 	if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100))
3673 		dev_warn(hsotg->dev,
3674 			 "%s: timeout DOEPCTL.EPDisable\n", __func__);
3675 
3676 	/* Clear EPDISBLD interrupt */
3677 	__orr32(hsotg->regs + epint_reg, DXEPINT_EPDISBLD);
3678 
3679 	if (hs_ep->dir_in) {
3680 		unsigned short fifo_index;
3681 
3682 		if (hsotg->dedicated_fifos || hs_ep->periodic)
3683 			fifo_index = hs_ep->fifo_index;
3684 		else
3685 			fifo_index = 0;
3686 
3687 		/* Flush TX FIFO */
3688 		dwc2_flush_tx_fifo(hsotg, fifo_index);
3689 
3690 		/* Clear Global In NP NAK in Shared FIFO for non periodic ep */
3691 		if (!hsotg->dedicated_fifos && !hs_ep->periodic)
3692 			__orr32(hsotg->regs + DCTL, DCTL_CGNPINNAK);
3693 
3694 	} else {
3695 		/* Remove global NAKs */
3696 		__orr32(hsotg->regs + DCTL, DCTL_CGOUTNAK);
3697 	}
3698 }
3699 
3700 /**
3701  * dwc2_hsotg_ep_enable - enable the given endpoint
3702  * @ep: The USB endpint to configure
3703  * @desc: The USB endpoint descriptor to configure with.
3704  *
3705  * This is called from the USB gadget code's usb_ep_enable().
3706  */
3707 static int dwc2_hsotg_ep_enable(struct usb_ep *ep,
3708 			       const struct usb_endpoint_descriptor *desc)
3709 {
3710 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
3711 	struct dwc2_hsotg *hsotg = hs_ep->parent;
3712 	unsigned long flags;
3713 	unsigned int index = hs_ep->index;
3714 	u32 epctrl_reg;
3715 	u32 epctrl;
3716 	u32 mps;
3717 	u32 mc;
3718 	u32 mask;
3719 	unsigned int dir_in;
3720 	unsigned int i, val, size;
3721 	int ret = 0;
3722 
3723 	dev_dbg(hsotg->dev,
3724 		"%s: ep %s: a 0x%02x, attr 0x%02x, mps 0x%04x, intr %d\n",
3725 		__func__, ep->name, desc->bEndpointAddress, desc->bmAttributes,
3726 		desc->wMaxPacketSize, desc->bInterval);
3727 
3728 	/* not to be called for EP0 */
3729 	if (index == 0) {
3730 		dev_err(hsotg->dev, "%s: called for EP 0\n", __func__);
3731 		return -EINVAL;
3732 	}
3733 
3734 	dir_in = (desc->bEndpointAddress & USB_ENDPOINT_DIR_MASK) ? 1 : 0;
3735 	if (dir_in != hs_ep->dir_in) {
3736 		dev_err(hsotg->dev, "%s: direction mismatch!\n", __func__);
3737 		return -EINVAL;
3738 	}
3739 
3740 	mps = usb_endpoint_maxp(desc);
3741 	mc = usb_endpoint_maxp_mult(desc);
3742 
3743 	/* note, we handle this here instead of dwc2_hsotg_set_ep_maxpacket */
3744 
3745 	epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
3746 	epctrl = dwc2_readl(hsotg->regs + epctrl_reg);
3747 
3748 	dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n",
3749 		__func__, epctrl, epctrl_reg);
3750 
3751 	/* Allocate DMA descriptor chain for non-ctrl endpoints */
3752 	if (using_desc_dma(hsotg)) {
3753 		hs_ep->desc_list = dma_alloc_coherent(hsotg->dev,
3754 			MAX_DMA_DESC_NUM_GENERIC *
3755 			sizeof(struct dwc2_dma_desc),
3756 			&hs_ep->desc_list_dma, GFP_KERNEL);
3757 		if (!hs_ep->desc_list) {
3758 			ret = -ENOMEM;
3759 			goto error2;
3760 		}
3761 	}
3762 
3763 	spin_lock_irqsave(&hsotg->lock, flags);
3764 
3765 	epctrl &= ~(DXEPCTL_EPTYPE_MASK | DXEPCTL_MPS_MASK);
3766 	epctrl |= DXEPCTL_MPS(mps);
3767 
3768 	/*
3769 	 * mark the endpoint as active, otherwise the core may ignore
3770 	 * transactions entirely for this endpoint
3771 	 */
3772 	epctrl |= DXEPCTL_USBACTEP;
3773 
3774 	/* update the endpoint state */
3775 	dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in);
3776 
3777 	/* default, set to non-periodic */
3778 	hs_ep->isochronous = 0;
3779 	hs_ep->periodic = 0;
3780 	hs_ep->halted = 0;
3781 	hs_ep->interval = desc->bInterval;
3782 
3783 	switch (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) {
3784 	case USB_ENDPOINT_XFER_ISOC:
3785 		epctrl |= DXEPCTL_EPTYPE_ISO;
3786 		epctrl |= DXEPCTL_SETEVENFR;
3787 		hs_ep->isochronous = 1;
3788 		hs_ep->interval = 1 << (desc->bInterval - 1);
3789 		hs_ep->target_frame = TARGET_FRAME_INITIAL;
3790 		hs_ep->isoc_chain_num = 0;
3791 		hs_ep->next_desc = 0;
3792 		if (dir_in) {
3793 			hs_ep->periodic = 1;
3794 			mask = dwc2_readl(hsotg->regs + DIEPMSK);
3795 			mask |= DIEPMSK_NAKMSK;
3796 			dwc2_writel(mask, hsotg->regs + DIEPMSK);
3797 		} else {
3798 			mask = dwc2_readl(hsotg->regs + DOEPMSK);
3799 			mask |= DOEPMSK_OUTTKNEPDISMSK;
3800 			dwc2_writel(mask, hsotg->regs + DOEPMSK);
3801 		}
3802 		break;
3803 
3804 	case USB_ENDPOINT_XFER_BULK:
3805 		epctrl |= DXEPCTL_EPTYPE_BULK;
3806 		break;
3807 
3808 	case USB_ENDPOINT_XFER_INT:
3809 		if (dir_in)
3810 			hs_ep->periodic = 1;
3811 
3812 		if (hsotg->gadget.speed == USB_SPEED_HIGH)
3813 			hs_ep->interval = 1 << (desc->bInterval - 1);
3814 
3815 		epctrl |= DXEPCTL_EPTYPE_INTERRUPT;
3816 		break;
3817 
3818 	case USB_ENDPOINT_XFER_CONTROL:
3819 		epctrl |= DXEPCTL_EPTYPE_CONTROL;
3820 		break;
3821 	}
3822 
3823 	/*
3824 	 * if the hardware has dedicated fifos, we must give each IN EP
3825 	 * a unique tx-fifo even if it is non-periodic.
3826 	 */
3827 	if (dir_in && hsotg->dedicated_fifos) {
3828 		u32 fifo_index = 0;
3829 		u32 fifo_size = UINT_MAX;
3830 		size = hs_ep->ep.maxpacket*hs_ep->mc;
3831 		for (i = 1; i < hsotg->num_of_eps; ++i) {
3832 			if (hsotg->fifo_map & (1<<i))
3833 				continue;
3834 			val = dwc2_readl(hsotg->regs + DPTXFSIZN(i));
3835 			val = (val >> FIFOSIZE_DEPTH_SHIFT)*4;
3836 			if (val < size)
3837 				continue;
3838 			/* Search for smallest acceptable fifo */
3839 			if (val < fifo_size) {
3840 				fifo_size = val;
3841 				fifo_index = i;
3842 			}
3843 		}
3844 		if (!fifo_index) {
3845 			dev_err(hsotg->dev,
3846 				"%s: No suitable fifo found\n", __func__);
3847 			ret = -ENOMEM;
3848 			goto error1;
3849 		}
3850 		hsotg->fifo_map |= 1 << fifo_index;
3851 		epctrl |= DXEPCTL_TXFNUM(fifo_index);
3852 		hs_ep->fifo_index = fifo_index;
3853 		hs_ep->fifo_size = fifo_size;
3854 	}
3855 
3856 	/* for non control endpoints, set PID to D0 */
3857 	if (index && !hs_ep->isochronous)
3858 		epctrl |= DXEPCTL_SETD0PID;
3859 
3860 	dev_dbg(hsotg->dev, "%s: write DxEPCTL=0x%08x\n",
3861 		__func__, epctrl);
3862 
3863 	dwc2_writel(epctrl, hsotg->regs + epctrl_reg);
3864 	dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x\n",
3865 		__func__, dwc2_readl(hsotg->regs + epctrl_reg));
3866 
3867 	/* enable the endpoint interrupt */
3868 	dwc2_hsotg_ctrl_epint(hsotg, index, dir_in, 1);
3869 
3870 error1:
3871 	spin_unlock_irqrestore(&hsotg->lock, flags);
3872 
3873 error2:
3874 	if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) {
3875 		dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC *
3876 			sizeof(struct dwc2_dma_desc),
3877 			hs_ep->desc_list, hs_ep->desc_list_dma);
3878 		hs_ep->desc_list = NULL;
3879 	}
3880 
3881 	return ret;
3882 }
3883 
3884 /**
3885  * dwc2_hsotg_ep_disable - disable given endpoint
3886  * @ep: The endpoint to disable.
3887  */
3888 static int dwc2_hsotg_ep_disable(struct usb_ep *ep)
3889 {
3890 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
3891 	struct dwc2_hsotg *hsotg = hs_ep->parent;
3892 	int dir_in = hs_ep->dir_in;
3893 	int index = hs_ep->index;
3894 	unsigned long flags;
3895 	u32 epctrl_reg;
3896 	u32 ctrl;
3897 
3898 	dev_dbg(hsotg->dev, "%s(ep %p)\n", __func__, ep);
3899 
3900 	if (ep == &hsotg->eps_out[0]->ep) {
3901 		dev_err(hsotg->dev, "%s: called for ep0\n", __func__);
3902 		return -EINVAL;
3903 	}
3904 
3905 	/* Remove DMA memory allocated for non-control Endpoints */
3906 	if (using_desc_dma(hsotg)) {
3907 		dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC *
3908 				  sizeof(struct dwc2_dma_desc),
3909 				  hs_ep->desc_list, hs_ep->desc_list_dma);
3910 		hs_ep->desc_list = NULL;
3911 	}
3912 
3913 	epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index);
3914 
3915 	spin_lock_irqsave(&hsotg->lock, flags);
3916 
3917 	ctrl = dwc2_readl(hsotg->regs + epctrl_reg);
3918 
3919 	if (ctrl & DXEPCTL_EPENA)
3920 		dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep);
3921 
3922 	ctrl &= ~DXEPCTL_EPENA;
3923 	ctrl &= ~DXEPCTL_USBACTEP;
3924 	ctrl |= DXEPCTL_SNAK;
3925 
3926 	dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl);
3927 	dwc2_writel(ctrl, hsotg->regs + epctrl_reg);
3928 
3929 	/* disable endpoint interrupts */
3930 	dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0);
3931 
3932 	/* terminate all requests with shutdown */
3933 	kill_all_requests(hsotg, hs_ep, -ESHUTDOWN);
3934 
3935 	hsotg->fifo_map &= ~(1 << hs_ep->fifo_index);
3936 	hs_ep->fifo_index = 0;
3937 	hs_ep->fifo_size = 0;
3938 
3939 	spin_unlock_irqrestore(&hsotg->lock, flags);
3940 	return 0;
3941 }
3942 
3943 /**
3944  * on_list - check request is on the given endpoint
3945  * @ep: The endpoint to check.
3946  * @test: The request to test if it is on the endpoint.
3947  */
3948 static bool on_list(struct dwc2_hsotg_ep *ep, struct dwc2_hsotg_req *test)
3949 {
3950 	struct dwc2_hsotg_req *req, *treq;
3951 
3952 	list_for_each_entry_safe(req, treq, &ep->queue, queue) {
3953 		if (req == test)
3954 			return true;
3955 	}
3956 
3957 	return false;
3958 }
3959 
3960 /**
3961  * dwc2_hsotg_ep_dequeue - dequeue given endpoint
3962  * @ep: The endpoint to dequeue.
3963  * @req: The request to be removed from a queue.
3964  */
3965 static int dwc2_hsotg_ep_dequeue(struct usb_ep *ep, struct usb_request *req)
3966 {
3967 	struct dwc2_hsotg_req *hs_req = our_req(req);
3968 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
3969 	struct dwc2_hsotg *hs = hs_ep->parent;
3970 	unsigned long flags;
3971 
3972 	dev_dbg(hs->dev, "ep_dequeue(%p,%p)\n", ep, req);
3973 
3974 	spin_lock_irqsave(&hs->lock, flags);
3975 
3976 	if (!on_list(hs_ep, hs_req)) {
3977 		spin_unlock_irqrestore(&hs->lock, flags);
3978 		return -EINVAL;
3979 	}
3980 
3981 	/* Dequeue already started request */
3982 	if (req == &hs_ep->req->req)
3983 		dwc2_hsotg_ep_stop_xfr(hs, hs_ep);
3984 
3985 	dwc2_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET);
3986 	spin_unlock_irqrestore(&hs->lock, flags);
3987 
3988 	return 0;
3989 }
3990 
3991 /**
3992  * dwc2_hsotg_ep_sethalt - set halt on a given endpoint
3993  * @ep: The endpoint to set halt.
3994  * @value: Set or unset the halt.
3995  * @now: If true, stall the endpoint now. Otherwise return -EAGAIN if
3996  *       the endpoint is busy processing requests.
3997  *
3998  * We need to stall the endpoint immediately if request comes from set_feature
3999  * protocol command handler.
4000  */
4001 static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now)
4002 {
4003 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4004 	struct dwc2_hsotg *hs = hs_ep->parent;
4005 	int index = hs_ep->index;
4006 	u32 epreg;
4007 	u32 epctl;
4008 	u32 xfertype;
4009 
4010 	dev_info(hs->dev, "%s(ep %p %s, %d)\n", __func__, ep, ep->name, value);
4011 
4012 	if (index == 0) {
4013 		if (value)
4014 			dwc2_hsotg_stall_ep0(hs);
4015 		else
4016 			dev_warn(hs->dev,
4017 				 "%s: can't clear halt on ep0\n", __func__);
4018 		return 0;
4019 	}
4020 
4021 	if (hs_ep->isochronous) {
4022 		dev_err(hs->dev, "%s is Isochronous Endpoint\n", ep->name);
4023 		return -EINVAL;
4024 	}
4025 
4026 	if (!now && value && !list_empty(&hs_ep->queue)) {
4027 		dev_dbg(hs->dev, "%s request is pending, cannot halt\n",
4028 			ep->name);
4029 		return -EAGAIN;
4030 	}
4031 
4032 	if (hs_ep->dir_in) {
4033 		epreg = DIEPCTL(index);
4034 		epctl = dwc2_readl(hs->regs + epreg);
4035 
4036 		if (value) {
4037 			epctl |= DXEPCTL_STALL | DXEPCTL_SNAK;
4038 			if (epctl & DXEPCTL_EPENA)
4039 				epctl |= DXEPCTL_EPDIS;
4040 		} else {
4041 			epctl &= ~DXEPCTL_STALL;
4042 			xfertype = epctl & DXEPCTL_EPTYPE_MASK;
4043 			if (xfertype == DXEPCTL_EPTYPE_BULK ||
4044 				xfertype == DXEPCTL_EPTYPE_INTERRUPT)
4045 					epctl |= DXEPCTL_SETD0PID;
4046 		}
4047 		dwc2_writel(epctl, hs->regs + epreg);
4048 	} else {
4049 
4050 		epreg = DOEPCTL(index);
4051 		epctl = dwc2_readl(hs->regs + epreg);
4052 
4053 		if (value)
4054 			epctl |= DXEPCTL_STALL;
4055 		else {
4056 			epctl &= ~DXEPCTL_STALL;
4057 			xfertype = epctl & DXEPCTL_EPTYPE_MASK;
4058 			if (xfertype == DXEPCTL_EPTYPE_BULK ||
4059 				xfertype == DXEPCTL_EPTYPE_INTERRUPT)
4060 					epctl |= DXEPCTL_SETD0PID;
4061 		}
4062 		dwc2_writel(epctl, hs->regs + epreg);
4063 	}
4064 
4065 	hs_ep->halted = value;
4066 
4067 	return 0;
4068 }
4069 
4070 /**
4071  * dwc2_hsotg_ep_sethalt_lock - set halt on a given endpoint with lock held
4072  * @ep: The endpoint to set halt.
4073  * @value: Set or unset the halt.
4074  */
4075 static int dwc2_hsotg_ep_sethalt_lock(struct usb_ep *ep, int value)
4076 {
4077 	struct dwc2_hsotg_ep *hs_ep = our_ep(ep);
4078 	struct dwc2_hsotg *hs = hs_ep->parent;
4079 	unsigned long flags = 0;
4080 	int ret = 0;
4081 
4082 	spin_lock_irqsave(&hs->lock, flags);
4083 	ret = dwc2_hsotg_ep_sethalt(ep, value, false);
4084 	spin_unlock_irqrestore(&hs->lock, flags);
4085 
4086 	return ret;
4087 }
4088 
4089 static struct usb_ep_ops dwc2_hsotg_ep_ops = {
4090 	.enable		= dwc2_hsotg_ep_enable,
4091 	.disable	= dwc2_hsotg_ep_disable,
4092 	.alloc_request	= dwc2_hsotg_ep_alloc_request,
4093 	.free_request	= dwc2_hsotg_ep_free_request,
4094 	.queue		= dwc2_hsotg_ep_queue_lock,
4095 	.dequeue	= dwc2_hsotg_ep_dequeue,
4096 	.set_halt	= dwc2_hsotg_ep_sethalt_lock,
4097 	/* note, don't believe we have any call for the fifo routines */
4098 };
4099 
4100 /**
4101  * dwc2_hsotg_init - initalize the usb core
4102  * @hsotg: The driver state
4103  */
4104 static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg)
4105 {
4106 	u32 trdtim;
4107 	u32 usbcfg;
4108 	/* unmask subset of endpoint interrupts */
4109 
4110 	dwc2_writel(DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK |
4111 		    DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK,
4112 		    hsotg->regs + DIEPMSK);
4113 
4114 	dwc2_writel(DOEPMSK_SETUPMSK | DOEPMSK_AHBERRMSK |
4115 		    DOEPMSK_EPDISBLDMSK | DOEPMSK_XFERCOMPLMSK,
4116 		    hsotg->regs + DOEPMSK);
4117 
4118 	dwc2_writel(0, hsotg->regs + DAINTMSK);
4119 
4120 	/* Be in disconnected state until gadget is registered */
4121 	__orr32(hsotg->regs + DCTL, DCTL_SFTDISCON);
4122 
4123 	/* setup fifos */
4124 
4125 	dev_dbg(hsotg->dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n",
4126 		dwc2_readl(hsotg->regs + GRXFSIZ),
4127 		dwc2_readl(hsotg->regs + GNPTXFSIZ));
4128 
4129 	dwc2_hsotg_init_fifo(hsotg);
4130 
4131 	/* keep other bits untouched (so e.g. forced modes are not lost) */
4132 	usbcfg = dwc2_readl(hsotg->regs + GUSBCFG);
4133 	usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP |
4134 		GUSBCFG_HNPCAP);
4135 
4136 	/* set the PLL on, remove the HNP/SRP and set the PHY */
4137 	trdtim = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5;
4138 	usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) |
4139 		(trdtim << GUSBCFG_USBTRDTIM_SHIFT);
4140 	dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
4141 
4142 	if (using_dma(hsotg))
4143 		__orr32(hsotg->regs + GAHBCFG, GAHBCFG_DMA_EN);
4144 }
4145 
4146 /**
4147  * dwc2_hsotg_udc_start - prepare the udc for work
4148  * @gadget: The usb gadget state
4149  * @driver: The usb gadget driver
4150  *
4151  * Perform initialization to prepare udc device and driver
4152  * to work.
4153  */
4154 static int dwc2_hsotg_udc_start(struct usb_gadget *gadget,
4155 			   struct usb_gadget_driver *driver)
4156 {
4157 	struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4158 	unsigned long flags;
4159 	int ret;
4160 
4161 	if (!hsotg) {
4162 		pr_err("%s: called with no device\n", __func__);
4163 		return -ENODEV;
4164 	}
4165 
4166 	if (!driver) {
4167 		dev_err(hsotg->dev, "%s: no driver\n", __func__);
4168 		return -EINVAL;
4169 	}
4170 
4171 	if (driver->max_speed < USB_SPEED_FULL)
4172 		dev_err(hsotg->dev, "%s: bad speed\n", __func__);
4173 
4174 	if (!driver->setup) {
4175 		dev_err(hsotg->dev, "%s: missing entry points\n", __func__);
4176 		return -EINVAL;
4177 	}
4178 
4179 	WARN_ON(hsotg->driver);
4180 
4181 	driver->driver.bus = NULL;
4182 	hsotg->driver = driver;
4183 	hsotg->gadget.dev.of_node = hsotg->dev->of_node;
4184 	hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4185 
4186 	if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL) {
4187 		ret = dwc2_lowlevel_hw_enable(hsotg);
4188 		if (ret)
4189 			goto err;
4190 	}
4191 
4192 	if (!IS_ERR_OR_NULL(hsotg->uphy))
4193 		otg_set_peripheral(hsotg->uphy->otg, &hsotg->gadget);
4194 
4195 	spin_lock_irqsave(&hsotg->lock, flags);
4196 	if (dwc2_hw_is_device(hsotg)) {
4197 		dwc2_hsotg_init(hsotg);
4198 		dwc2_hsotg_core_init_disconnected(hsotg, false);
4199 	}
4200 
4201 	hsotg->enabled = 0;
4202 	spin_unlock_irqrestore(&hsotg->lock, flags);
4203 
4204 	dev_info(hsotg->dev, "bound driver %s\n", driver->driver.name);
4205 
4206 	return 0;
4207 
4208 err:
4209 	hsotg->driver = NULL;
4210 	return ret;
4211 }
4212 
4213 /**
4214  * dwc2_hsotg_udc_stop - stop the udc
4215  * @gadget: The usb gadget state
4216  * @driver: The usb gadget driver
4217  *
4218  * Stop udc hw block and stay tunned for future transmissions
4219  */
4220 static int dwc2_hsotg_udc_stop(struct usb_gadget *gadget)
4221 {
4222 	struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4223 	unsigned long flags = 0;
4224 	int ep;
4225 
4226 	if (!hsotg)
4227 		return -ENODEV;
4228 
4229 	/* all endpoints should be shutdown */
4230 	for (ep = 1; ep < hsotg->num_of_eps; ep++) {
4231 		if (hsotg->eps_in[ep])
4232 			dwc2_hsotg_ep_disable(&hsotg->eps_in[ep]->ep);
4233 		if (hsotg->eps_out[ep])
4234 			dwc2_hsotg_ep_disable(&hsotg->eps_out[ep]->ep);
4235 	}
4236 
4237 	spin_lock_irqsave(&hsotg->lock, flags);
4238 
4239 	hsotg->driver = NULL;
4240 	hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4241 	hsotg->enabled = 0;
4242 
4243 	spin_unlock_irqrestore(&hsotg->lock, flags);
4244 
4245 	if (!IS_ERR_OR_NULL(hsotg->uphy))
4246 		otg_set_peripheral(hsotg->uphy->otg, NULL);
4247 
4248 	if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
4249 		dwc2_lowlevel_hw_disable(hsotg);
4250 
4251 	return 0;
4252 }
4253 
4254 /**
4255  * dwc2_hsotg_gadget_getframe - read the frame number
4256  * @gadget: The usb gadget state
4257  *
4258  * Read the {micro} frame number
4259  */
4260 static int dwc2_hsotg_gadget_getframe(struct usb_gadget *gadget)
4261 {
4262 	return dwc2_hsotg_read_frameno(to_hsotg(gadget));
4263 }
4264 
4265 /**
4266  * dwc2_hsotg_pullup - connect/disconnect the USB PHY
4267  * @gadget: The usb gadget state
4268  * @is_on: Current state of the USB PHY
4269  *
4270  * Connect/Disconnect the USB PHY pullup
4271  */
4272 static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on)
4273 {
4274 	struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4275 	unsigned long flags = 0;
4276 
4277 	dev_dbg(hsotg->dev, "%s: is_on: %d op_state: %d\n", __func__, is_on,
4278 			hsotg->op_state);
4279 
4280 	/* Don't modify pullup state while in host mode */
4281 	if (hsotg->op_state != OTG_STATE_B_PERIPHERAL) {
4282 		hsotg->enabled = is_on;
4283 		return 0;
4284 	}
4285 
4286 	spin_lock_irqsave(&hsotg->lock, flags);
4287 	if (is_on) {
4288 		hsotg->enabled = 1;
4289 		dwc2_hsotg_core_init_disconnected(hsotg, false);
4290 		dwc2_hsotg_core_connect(hsotg);
4291 	} else {
4292 		dwc2_hsotg_core_disconnect(hsotg);
4293 		dwc2_hsotg_disconnect(hsotg);
4294 		hsotg->enabled = 0;
4295 	}
4296 
4297 	hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4298 	spin_unlock_irqrestore(&hsotg->lock, flags);
4299 
4300 	return 0;
4301 }
4302 
4303 static int dwc2_hsotg_vbus_session(struct usb_gadget *gadget, int is_active)
4304 {
4305 	struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4306 	unsigned long flags;
4307 
4308 	dev_dbg(hsotg->dev, "%s: is_active: %d\n", __func__, is_active);
4309 	spin_lock_irqsave(&hsotg->lock, flags);
4310 
4311 	/*
4312 	 * If controller is hibernated, it must exit from hibernation
4313 	 * before being initialized / de-initialized
4314 	 */
4315 	if (hsotg->lx_state == DWC2_L2)
4316 		dwc2_exit_hibernation(hsotg, false);
4317 
4318 	if (is_active) {
4319 		hsotg->op_state = OTG_STATE_B_PERIPHERAL;
4320 
4321 		dwc2_hsotg_core_init_disconnected(hsotg, false);
4322 		if (hsotg->enabled)
4323 			dwc2_hsotg_core_connect(hsotg);
4324 	} else {
4325 		dwc2_hsotg_core_disconnect(hsotg);
4326 		dwc2_hsotg_disconnect(hsotg);
4327 	}
4328 
4329 	spin_unlock_irqrestore(&hsotg->lock, flags);
4330 	return 0;
4331 }
4332 
4333 /**
4334  * dwc2_hsotg_vbus_draw - report bMaxPower field
4335  * @gadget: The usb gadget state
4336  * @mA: Amount of current
4337  *
4338  * Report how much power the device may consume to the phy.
4339  */
4340 static int dwc2_hsotg_vbus_draw(struct usb_gadget *gadget, unsigned mA)
4341 {
4342 	struct dwc2_hsotg *hsotg = to_hsotg(gadget);
4343 
4344 	if (IS_ERR_OR_NULL(hsotg->uphy))
4345 		return -ENOTSUPP;
4346 	return usb_phy_set_power(hsotg->uphy, mA);
4347 }
4348 
4349 static const struct usb_gadget_ops dwc2_hsotg_gadget_ops = {
4350 	.get_frame	= dwc2_hsotg_gadget_getframe,
4351 	.udc_start		= dwc2_hsotg_udc_start,
4352 	.udc_stop		= dwc2_hsotg_udc_stop,
4353 	.pullup                 = dwc2_hsotg_pullup,
4354 	.vbus_session		= dwc2_hsotg_vbus_session,
4355 	.vbus_draw		= dwc2_hsotg_vbus_draw,
4356 };
4357 
4358 /**
4359  * dwc2_hsotg_initep - initialise a single endpoint
4360  * @hsotg: The device state.
4361  * @hs_ep: The endpoint to be initialised.
4362  * @epnum: The endpoint number
4363  *
4364  * Initialise the given endpoint (as part of the probe and device state
4365  * creation) to give to the gadget driver. Setup the endpoint name, any
4366  * direction information and other state that may be required.
4367  */
4368 static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg,
4369 				       struct dwc2_hsotg_ep *hs_ep,
4370 				       int epnum,
4371 				       bool dir_in)
4372 {
4373 	char *dir;
4374 
4375 	if (epnum == 0)
4376 		dir = "";
4377 	else if (dir_in)
4378 		dir = "in";
4379 	else
4380 		dir = "out";
4381 
4382 	hs_ep->dir_in = dir_in;
4383 	hs_ep->index = epnum;
4384 
4385 	snprintf(hs_ep->name, sizeof(hs_ep->name), "ep%d%s", epnum, dir);
4386 
4387 	INIT_LIST_HEAD(&hs_ep->queue);
4388 	INIT_LIST_HEAD(&hs_ep->ep.ep_list);
4389 
4390 	/* add to the list of endpoints known by the gadget driver */
4391 	if (epnum)
4392 		list_add_tail(&hs_ep->ep.ep_list, &hsotg->gadget.ep_list);
4393 
4394 	hs_ep->parent = hsotg;
4395 	hs_ep->ep.name = hs_ep->name;
4396 
4397 	if (hsotg->params.speed == DWC2_SPEED_PARAM_LOW)
4398 		usb_ep_set_maxpacket_limit(&hs_ep->ep, 8);
4399 	else
4400 		usb_ep_set_maxpacket_limit(&hs_ep->ep,
4401 					   epnum ? 1024 : EP0_MPS_LIMIT);
4402 	hs_ep->ep.ops = &dwc2_hsotg_ep_ops;
4403 
4404 	if (epnum == 0) {
4405 		hs_ep->ep.caps.type_control = true;
4406 	} else {
4407 		if (hsotg->params.speed != DWC2_SPEED_PARAM_LOW) {
4408 			hs_ep->ep.caps.type_iso = true;
4409 			hs_ep->ep.caps.type_bulk = true;
4410 		}
4411 		hs_ep->ep.caps.type_int = true;
4412 	}
4413 
4414 	if (dir_in)
4415 		hs_ep->ep.caps.dir_in = true;
4416 	else
4417 		hs_ep->ep.caps.dir_out = true;
4418 
4419 	/*
4420 	 * if we're using dma, we need to set the next-endpoint pointer
4421 	 * to be something valid.
4422 	 */
4423 
4424 	if (using_dma(hsotg)) {
4425 		u32 next = DXEPCTL_NEXTEP((epnum + 1) % 15);
4426 		if (dir_in)
4427 			dwc2_writel(next, hsotg->regs + DIEPCTL(epnum));
4428 		else
4429 			dwc2_writel(next, hsotg->regs + DOEPCTL(epnum));
4430 	}
4431 }
4432 
4433 /**
4434  * dwc2_hsotg_hw_cfg - read HW configuration registers
4435  * @param: The device state
4436  *
4437  * Read the USB core HW configuration registers
4438  */
4439 static int dwc2_hsotg_hw_cfg(struct dwc2_hsotg *hsotg)
4440 {
4441 	u32 cfg;
4442 	u32 ep_type;
4443 	u32 i;
4444 
4445 	/* check hardware configuration */
4446 
4447 	hsotg->num_of_eps = hsotg->hw_params.num_dev_ep;
4448 
4449 	/* Add ep0 */
4450 	hsotg->num_of_eps++;
4451 
4452 	hsotg->eps_in[0] = devm_kzalloc(hsotg->dev, sizeof(struct dwc2_hsotg_ep),
4453 								GFP_KERNEL);
4454 	if (!hsotg->eps_in[0])
4455 		return -ENOMEM;
4456 	/* Same dwc2_hsotg_ep is used in both directions for ep0 */
4457 	hsotg->eps_out[0] = hsotg->eps_in[0];
4458 
4459 	cfg = hsotg->hw_params.dev_ep_dirs;
4460 	for (i = 1, cfg >>= 2; i < hsotg->num_of_eps; i++, cfg >>= 2) {
4461 		ep_type = cfg & 3;
4462 		/* Direction in or both */
4463 		if (!(ep_type & 2)) {
4464 			hsotg->eps_in[i] = devm_kzalloc(hsotg->dev,
4465 				sizeof(struct dwc2_hsotg_ep), GFP_KERNEL);
4466 			if (!hsotg->eps_in[i])
4467 				return -ENOMEM;
4468 		}
4469 		/* Direction out or both */
4470 		if (!(ep_type & 1)) {
4471 			hsotg->eps_out[i] = devm_kzalloc(hsotg->dev,
4472 				sizeof(struct dwc2_hsotg_ep), GFP_KERNEL);
4473 			if (!hsotg->eps_out[i])
4474 				return -ENOMEM;
4475 		}
4476 	}
4477 
4478 	hsotg->fifo_mem = hsotg->hw_params.total_fifo_size;
4479 	hsotg->dedicated_fifos = hsotg->hw_params.en_multiple_tx_fifo;
4480 
4481 	dev_info(hsotg->dev, "EPs: %d, %s fifos, %d entries in SPRAM\n",
4482 		 hsotg->num_of_eps,
4483 		 hsotg->dedicated_fifos ? "dedicated" : "shared",
4484 		 hsotg->fifo_mem);
4485 	return 0;
4486 }
4487 
4488 /**
4489  * dwc2_hsotg_dump - dump state of the udc
4490  * @param: The device state
4491  */
4492 static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg)
4493 {
4494 #ifdef DEBUG
4495 	struct device *dev = hsotg->dev;
4496 	void __iomem *regs = hsotg->regs;
4497 	u32 val;
4498 	int idx;
4499 
4500 	dev_info(dev, "DCFG=0x%08x, DCTL=0x%08x, DIEPMSK=%08x\n",
4501 		 dwc2_readl(regs + DCFG), dwc2_readl(regs + DCTL),
4502 		 dwc2_readl(regs + DIEPMSK));
4503 
4504 	dev_info(dev, "GAHBCFG=0x%08x, GHWCFG1=0x%08x\n",
4505 		 dwc2_readl(regs + GAHBCFG), dwc2_readl(regs + GHWCFG1));
4506 
4507 	dev_info(dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n",
4508 		 dwc2_readl(regs + GRXFSIZ), dwc2_readl(regs + GNPTXFSIZ));
4509 
4510 	/* show periodic fifo settings */
4511 
4512 	for (idx = 1; idx < hsotg->num_of_eps; idx++) {
4513 		val = dwc2_readl(regs + DPTXFSIZN(idx));
4514 		dev_info(dev, "DPTx[%d] FSize=%d, StAddr=0x%08x\n", idx,
4515 			 val >> FIFOSIZE_DEPTH_SHIFT,
4516 			 val & FIFOSIZE_STARTADDR_MASK);
4517 	}
4518 
4519 	for (idx = 0; idx < hsotg->num_of_eps; idx++) {
4520 		dev_info(dev,
4521 			 "ep%d-in: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n", idx,
4522 			 dwc2_readl(regs + DIEPCTL(idx)),
4523 			 dwc2_readl(regs + DIEPTSIZ(idx)),
4524 			 dwc2_readl(regs + DIEPDMA(idx)));
4525 
4526 		val = dwc2_readl(regs + DOEPCTL(idx));
4527 		dev_info(dev,
4528 			 "ep%d-out: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n",
4529 			 idx, dwc2_readl(regs + DOEPCTL(idx)),
4530 			 dwc2_readl(regs + DOEPTSIZ(idx)),
4531 			 dwc2_readl(regs + DOEPDMA(idx)));
4532 
4533 	}
4534 
4535 	dev_info(dev, "DVBUSDIS=0x%08x, DVBUSPULSE=%08x\n",
4536 		 dwc2_readl(regs + DVBUSDIS), dwc2_readl(regs + DVBUSPULSE));
4537 #endif
4538 }
4539 
4540 /**
4541  * dwc2_gadget_init - init function for gadget
4542  * @dwc2: The data structure for the DWC2 driver.
4543  * @irq: The IRQ number for the controller.
4544  */
4545 int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq)
4546 {
4547 	struct device *dev = hsotg->dev;
4548 	int epnum;
4549 	int ret;
4550 
4551 	/* Dump fifo information */
4552 	dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n",
4553 		hsotg->params.g_np_tx_fifo_size);
4554 	dev_dbg(dev, "RXFIFO size: %d\n", hsotg->params.g_rx_fifo_size);
4555 
4556 	hsotg->gadget.max_speed = USB_SPEED_HIGH;
4557 	hsotg->gadget.ops = &dwc2_hsotg_gadget_ops;
4558 	hsotg->gadget.name = dev_name(dev);
4559 	if (hsotg->dr_mode == USB_DR_MODE_OTG)
4560 		hsotg->gadget.is_otg = 1;
4561 	else if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL)
4562 		hsotg->op_state = OTG_STATE_B_PERIPHERAL;
4563 
4564 	ret = dwc2_hsotg_hw_cfg(hsotg);
4565 	if (ret) {
4566 		dev_err(hsotg->dev, "Hardware configuration failed: %d\n", ret);
4567 		return ret;
4568 	}
4569 
4570 	hsotg->ctrl_buff = devm_kzalloc(hsotg->dev,
4571 			DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
4572 	if (!hsotg->ctrl_buff)
4573 		return -ENOMEM;
4574 
4575 	hsotg->ep0_buff = devm_kzalloc(hsotg->dev,
4576 			DWC2_CTRL_BUFF_SIZE, GFP_KERNEL);
4577 	if (!hsotg->ep0_buff)
4578 		return -ENOMEM;
4579 
4580 	if (using_desc_dma(hsotg)) {
4581 		ret = dwc2_gadget_alloc_ctrl_desc_chains(hsotg);
4582 		if (ret < 0)
4583 			return ret;
4584 	}
4585 
4586 	ret = devm_request_irq(hsotg->dev, irq, dwc2_hsotg_irq, IRQF_SHARED,
4587 				dev_name(hsotg->dev), hsotg);
4588 	if (ret < 0) {
4589 		dev_err(dev, "cannot claim IRQ for gadget\n");
4590 		return ret;
4591 	}
4592 
4593 	/* hsotg->num_of_eps holds number of EPs other than ep0 */
4594 
4595 	if (hsotg->num_of_eps == 0) {
4596 		dev_err(dev, "wrong number of EPs (zero)\n");
4597 		return -EINVAL;
4598 	}
4599 
4600 	/* setup endpoint information */
4601 
4602 	INIT_LIST_HEAD(&hsotg->gadget.ep_list);
4603 	hsotg->gadget.ep0 = &hsotg->eps_out[0]->ep;
4604 
4605 	/* allocate EP0 request */
4606 
4607 	hsotg->ctrl_req = dwc2_hsotg_ep_alloc_request(&hsotg->eps_out[0]->ep,
4608 						     GFP_KERNEL);
4609 	if (!hsotg->ctrl_req) {
4610 		dev_err(dev, "failed to allocate ctrl req\n");
4611 		return -ENOMEM;
4612 	}
4613 
4614 	/* initialise the endpoints now the core has been initialised */
4615 	for (epnum = 0; epnum < hsotg->num_of_eps; epnum++) {
4616 		if (hsotg->eps_in[epnum])
4617 			dwc2_hsotg_initep(hsotg, hsotg->eps_in[epnum],
4618 								epnum, 1);
4619 		if (hsotg->eps_out[epnum])
4620 			dwc2_hsotg_initep(hsotg, hsotg->eps_out[epnum],
4621 								epnum, 0);
4622 	}
4623 
4624 	ret = usb_add_gadget_udc(dev, &hsotg->gadget);
4625 	if (ret)
4626 		return ret;
4627 
4628 	dwc2_hsotg_dump(hsotg);
4629 
4630 	return 0;
4631 }
4632 
4633 /**
4634  * dwc2_hsotg_remove - remove function for hsotg driver
4635  * @pdev: The platform information for the driver
4636  */
4637 int dwc2_hsotg_remove(struct dwc2_hsotg *hsotg)
4638 {
4639 	usb_del_gadget_udc(&hsotg->gadget);
4640 
4641 	return 0;
4642 }
4643 
4644 int dwc2_hsotg_suspend(struct dwc2_hsotg *hsotg)
4645 {
4646 	unsigned long flags;
4647 
4648 	if (hsotg->lx_state != DWC2_L0)
4649 		return 0;
4650 
4651 	if (hsotg->driver) {
4652 		int ep;
4653 
4654 		dev_info(hsotg->dev, "suspending usb gadget %s\n",
4655 			 hsotg->driver->driver.name);
4656 
4657 		spin_lock_irqsave(&hsotg->lock, flags);
4658 		if (hsotg->enabled)
4659 			dwc2_hsotg_core_disconnect(hsotg);
4660 		dwc2_hsotg_disconnect(hsotg);
4661 		hsotg->gadget.speed = USB_SPEED_UNKNOWN;
4662 		spin_unlock_irqrestore(&hsotg->lock, flags);
4663 
4664 		for (ep = 0; ep < hsotg->num_of_eps; ep++) {
4665 			if (hsotg->eps_in[ep])
4666 				dwc2_hsotg_ep_disable(&hsotg->eps_in[ep]->ep);
4667 			if (hsotg->eps_out[ep])
4668 				dwc2_hsotg_ep_disable(&hsotg->eps_out[ep]->ep);
4669 		}
4670 	}
4671 
4672 	return 0;
4673 }
4674 
4675 int dwc2_hsotg_resume(struct dwc2_hsotg *hsotg)
4676 {
4677 	unsigned long flags;
4678 
4679 	if (hsotg->lx_state == DWC2_L2)
4680 		return 0;
4681 
4682 	if (hsotg->driver) {
4683 		dev_info(hsotg->dev, "resuming usb gadget %s\n",
4684 			 hsotg->driver->driver.name);
4685 
4686 		spin_lock_irqsave(&hsotg->lock, flags);
4687 		dwc2_hsotg_core_init_disconnected(hsotg, false);
4688 		if (hsotg->enabled)
4689 			dwc2_hsotg_core_connect(hsotg);
4690 		spin_unlock_irqrestore(&hsotg->lock, flags);
4691 	}
4692 
4693 	return 0;
4694 }
4695 
4696 /**
4697  * dwc2_backup_device_registers() - Backup controller device registers.
4698  * When suspending usb bus, registers needs to be backuped
4699  * if controller power is disabled once suspended.
4700  *
4701  * @hsotg: Programming view of the DWC_otg controller
4702  */
4703 int dwc2_backup_device_registers(struct dwc2_hsotg *hsotg)
4704 {
4705 	struct dwc2_dregs_backup *dr;
4706 	int i;
4707 
4708 	dev_dbg(hsotg->dev, "%s\n", __func__);
4709 
4710 	/* Backup dev regs */
4711 	dr = &hsotg->dr_backup;
4712 
4713 	dr->dcfg = dwc2_readl(hsotg->regs + DCFG);
4714 	dr->dctl = dwc2_readl(hsotg->regs + DCTL);
4715 	dr->daintmsk = dwc2_readl(hsotg->regs + DAINTMSK);
4716 	dr->diepmsk = dwc2_readl(hsotg->regs + DIEPMSK);
4717 	dr->doepmsk = dwc2_readl(hsotg->regs + DOEPMSK);
4718 
4719 	for (i = 0; i < hsotg->num_of_eps; i++) {
4720 		/* Backup IN EPs */
4721 		dr->diepctl[i] = dwc2_readl(hsotg->regs + DIEPCTL(i));
4722 
4723 		/* Ensure DATA PID is correctly configured */
4724 		if (dr->diepctl[i] & DXEPCTL_DPID)
4725 			dr->diepctl[i] |= DXEPCTL_SETD1PID;
4726 		else
4727 			dr->diepctl[i] |= DXEPCTL_SETD0PID;
4728 
4729 		dr->dieptsiz[i] = dwc2_readl(hsotg->regs + DIEPTSIZ(i));
4730 		dr->diepdma[i] = dwc2_readl(hsotg->regs + DIEPDMA(i));
4731 
4732 		/* Backup OUT EPs */
4733 		dr->doepctl[i] = dwc2_readl(hsotg->regs + DOEPCTL(i));
4734 
4735 		/* Ensure DATA PID is correctly configured */
4736 		if (dr->doepctl[i] & DXEPCTL_DPID)
4737 			dr->doepctl[i] |= DXEPCTL_SETD1PID;
4738 		else
4739 			dr->doepctl[i] |= DXEPCTL_SETD0PID;
4740 
4741 		dr->doeptsiz[i] = dwc2_readl(hsotg->regs + DOEPTSIZ(i));
4742 		dr->doepdma[i] = dwc2_readl(hsotg->regs + DOEPDMA(i));
4743 	}
4744 	dr->valid = true;
4745 	return 0;
4746 }
4747 
4748 /**
4749  * dwc2_restore_device_registers() - Restore controller device registers.
4750  * When resuming usb bus, device registers needs to be restored
4751  * if controller power were disabled.
4752  *
4753  * @hsotg: Programming view of the DWC_otg controller
4754  */
4755 int dwc2_restore_device_registers(struct dwc2_hsotg *hsotg)
4756 {
4757 	struct dwc2_dregs_backup *dr;
4758 	u32 dctl;
4759 	int i;
4760 
4761 	dev_dbg(hsotg->dev, "%s\n", __func__);
4762 
4763 	/* Restore dev regs */
4764 	dr = &hsotg->dr_backup;
4765 	if (!dr->valid) {
4766 		dev_err(hsotg->dev, "%s: no device registers to restore\n",
4767 			__func__);
4768 		return -EINVAL;
4769 	}
4770 	dr->valid = false;
4771 
4772 	dwc2_writel(dr->dcfg, hsotg->regs + DCFG);
4773 	dwc2_writel(dr->dctl, hsotg->regs + DCTL);
4774 	dwc2_writel(dr->daintmsk, hsotg->regs + DAINTMSK);
4775 	dwc2_writel(dr->diepmsk, hsotg->regs + DIEPMSK);
4776 	dwc2_writel(dr->doepmsk, hsotg->regs + DOEPMSK);
4777 
4778 	for (i = 0; i < hsotg->num_of_eps; i++) {
4779 		/* Restore IN EPs */
4780 		dwc2_writel(dr->diepctl[i], hsotg->regs + DIEPCTL(i));
4781 		dwc2_writel(dr->dieptsiz[i], hsotg->regs + DIEPTSIZ(i));
4782 		dwc2_writel(dr->diepdma[i], hsotg->regs + DIEPDMA(i));
4783 
4784 		/* Restore OUT EPs */
4785 		dwc2_writel(dr->doepctl[i], hsotg->regs + DOEPCTL(i));
4786 		dwc2_writel(dr->doeptsiz[i], hsotg->regs + DOEPTSIZ(i));
4787 		dwc2_writel(dr->doepdma[i], hsotg->regs + DOEPDMA(i));
4788 	}
4789 
4790 	/* Set the Power-On Programming done bit */
4791 	dctl = dwc2_readl(hsotg->regs + DCTL);
4792 	dctl |= DCTL_PWRONPRGDONE;
4793 	dwc2_writel(dctl, hsotg->regs + DCTL);
4794 
4795 	return 0;
4796 }
4797