xref: /openbmc/linux/drivers/usb/dwc3/gadget.c (revision 77423a62)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4  *
5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6  *
7  * Authors: Felipe Balbi <balbi@ti.com>,
8  *	    Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9  */
10 
11 #include <linux/kernel.h>
12 #include <linux/delay.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <linux/platform_device.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/list.h>
20 #include <linux/dma-mapping.h>
21 
22 #include <linux/usb/ch9.h>
23 #include <linux/usb/gadget.h>
24 
25 #include "debug.h"
26 #include "core.h"
27 #include "gadget.h"
28 #include "io.h"
29 
30 #define DWC3_ALIGN_FRAME(d, n)	(((d)->frame_number + ((d)->interval * (n))) \
31 					& ~((d)->interval - 1))
32 
33 /**
34  * dwc3_gadget_set_test_mode - enables usb2 test modes
35  * @dwc: pointer to our context structure
36  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
37  *
38  * Caller should take care of locking. This function will return 0 on
39  * success or -EINVAL if wrong Test Selector is passed.
40  */
41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42 {
43 	u32		reg;
44 
45 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47 
48 	switch (mode) {
49 	case USB_TEST_J:
50 	case USB_TEST_K:
51 	case USB_TEST_SE0_NAK:
52 	case USB_TEST_PACKET:
53 	case USB_TEST_FORCE_ENABLE:
54 		reg |= mode << 1;
55 		break;
56 	default:
57 		return -EINVAL;
58 	}
59 
60 	dwc3_gadget_dctl_write_safe(dwc, reg);
61 
62 	return 0;
63 }
64 
65 /**
66  * dwc3_gadget_get_link_state - gets current state of usb link
67  * @dwc: pointer to our context structure
68  *
69  * Caller should take care of locking. This function will
70  * return the link state on success (>= 0) or -ETIMEDOUT.
71  */
72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73 {
74 	u32		reg;
75 
76 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77 
78 	return DWC3_DSTS_USBLNKST(reg);
79 }
80 
81 /**
82  * dwc3_gadget_set_link_state - sets usb link to a particular state
83  * @dwc: pointer to our context structure
84  * @state: the state to put link into
85  *
86  * Caller should take care of locking. This function will
87  * return 0 on success or -ETIMEDOUT.
88  */
89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90 {
91 	int		retries = 10000;
92 	u32		reg;
93 
94 	/*
95 	 * Wait until device controller is ready. Only applies to 1.94a and
96 	 * later RTL.
97 	 */
98 	if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99 		while (--retries) {
100 			reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 			if (reg & DWC3_DSTS_DCNRD)
102 				udelay(5);
103 			else
104 				break;
105 		}
106 
107 		if (retries <= 0)
108 			return -ETIMEDOUT;
109 	}
110 
111 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113 
114 	/* set no action before sending new link state change */
115 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116 
117 	/* set requested state */
118 	reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
120 
121 	/*
122 	 * The following code is racy when called from dwc3_gadget_wakeup,
123 	 * and is not needed, at least on newer versions
124 	 */
125 	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126 		return 0;
127 
128 	/* wait for a change in DSTS */
129 	retries = 10000;
130 	while (--retries) {
131 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
132 
133 		if (DWC3_DSTS_USBLNKST(reg) == state)
134 			return 0;
135 
136 		udelay(5);
137 	}
138 
139 	return -ETIMEDOUT;
140 }
141 
142 /**
143  * dwc3_ep_inc_trb - increment a trb index.
144  * @index: Pointer to the TRB index to increment.
145  *
146  * The index should never point to the link TRB. After incrementing,
147  * if it is point to the link TRB, wrap around to the beginning. The
148  * link TRB is always at the last TRB entry.
149  */
150 static void dwc3_ep_inc_trb(u8 *index)
151 {
152 	(*index)++;
153 	if (*index == (DWC3_TRB_NUM - 1))
154 		*index = 0;
155 }
156 
157 /**
158  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
159  * @dep: The endpoint whose enqueue pointer we're incrementing
160  */
161 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
162 {
163 	dwc3_ep_inc_trb(&dep->trb_enqueue);
164 }
165 
166 /**
167  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
168  * @dep: The endpoint whose enqueue pointer we're incrementing
169  */
170 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
171 {
172 	dwc3_ep_inc_trb(&dep->trb_dequeue);
173 }
174 
175 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
176 		struct dwc3_request *req, int status)
177 {
178 	struct dwc3			*dwc = dep->dwc;
179 
180 	list_del(&req->list);
181 	req->remaining = 0;
182 	req->needs_extra_trb = false;
183 
184 	if (req->request.status == -EINPROGRESS)
185 		req->request.status = status;
186 
187 	if (req->trb)
188 		usb_gadget_unmap_request_by_dev(dwc->sysdev,
189 				&req->request, req->direction);
190 
191 	req->trb = NULL;
192 	trace_dwc3_gadget_giveback(req);
193 
194 	if (dep->number > 1)
195 		pm_runtime_put(dwc->dev);
196 }
197 
198 /**
199  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
200  * @dep: The endpoint to whom the request belongs to
201  * @req: The request we're giving back
202  * @status: completion code for the request
203  *
204  * Must be called with controller's lock held and interrupts disabled. This
205  * function will unmap @req and call its ->complete() callback to notify upper
206  * layers that it has completed.
207  */
208 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
209 		int status)
210 {
211 	struct dwc3			*dwc = dep->dwc;
212 
213 	dwc3_gadget_del_and_unmap_request(dep, req, status);
214 	req->status = DWC3_REQUEST_STATUS_COMPLETED;
215 
216 	spin_unlock(&dwc->lock);
217 	usb_gadget_giveback_request(&dep->endpoint, &req->request);
218 	spin_lock(&dwc->lock);
219 }
220 
221 /**
222  * dwc3_send_gadget_generic_command - issue a generic command for the controller
223  * @dwc: pointer to the controller context
224  * @cmd: the command to be issued
225  * @param: command parameter
226  *
227  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
228  * and wait for its completion.
229  */
230 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
231 		u32 param)
232 {
233 	u32		timeout = 500;
234 	int		status = 0;
235 	int		ret = 0;
236 	u32		reg;
237 
238 	dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
239 	dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
240 
241 	do {
242 		reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
243 		if (!(reg & DWC3_DGCMD_CMDACT)) {
244 			status = DWC3_DGCMD_STATUS(reg);
245 			if (status)
246 				ret = -EINVAL;
247 			break;
248 		}
249 	} while (--timeout);
250 
251 	if (!timeout) {
252 		ret = -ETIMEDOUT;
253 		status = -ETIMEDOUT;
254 	}
255 
256 	trace_dwc3_gadget_generic_cmd(cmd, param, status);
257 
258 	return ret;
259 }
260 
261 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
262 
263 /**
264  * dwc3_send_gadget_ep_cmd - issue an endpoint command
265  * @dep: the endpoint to which the command is going to be issued
266  * @cmd: the command to be issued
267  * @params: parameters to the command
268  *
269  * Caller should handle locking. This function will issue @cmd with given
270  * @params to @dep and wait for its completion.
271  */
272 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
273 		struct dwc3_gadget_ep_cmd_params *params)
274 {
275 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
276 	struct dwc3		*dwc = dep->dwc;
277 	u32			timeout = 5000;
278 	u32			saved_config = 0;
279 	u32			reg;
280 
281 	int			cmd_status = 0;
282 	int			ret = -EINVAL;
283 
284 	/*
285 	 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
286 	 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
287 	 * endpoint command.
288 	 *
289 	 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
290 	 * settings. Restore them after the command is completed.
291 	 *
292 	 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
293 	 */
294 	if (dwc->gadget->speed <= USB_SPEED_HIGH) {
295 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
296 		if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
297 			saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
298 			reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
299 		}
300 
301 		if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
302 			saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
303 			reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
304 		}
305 
306 		if (saved_config)
307 			dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
308 	}
309 
310 	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
311 		int link_state;
312 
313 		/*
314 		 * Initiate remote wakeup if the link state is in U3 when
315 		 * operating in SS/SSP or L1/L2 when operating in HS/FS. If the
316 		 * link state is in U1/U2, no remote wakeup is needed. The Start
317 		 * Transfer command will initiate the link recovery.
318 		 */
319 		link_state = dwc3_gadget_get_link_state(dwc);
320 		switch (link_state) {
321 		case DWC3_LINK_STATE_U2:
322 			if (dwc->gadget->speed >= USB_SPEED_SUPER)
323 				break;
324 
325 			fallthrough;
326 		case DWC3_LINK_STATE_U3:
327 			ret = __dwc3_gadget_wakeup(dwc);
328 			dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
329 					ret);
330 			break;
331 		}
332 	}
333 
334 	/*
335 	 * For some commands such as Update Transfer command, DEPCMDPARn
336 	 * registers are reserved. Since the driver often sends Update Transfer
337 	 * command, don't write to DEPCMDPARn to avoid register write delays and
338 	 * improve performance.
339 	 */
340 	if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
341 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
342 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
343 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
344 	}
345 
346 	/*
347 	 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
348 	 * not relying on XferNotReady, we can make use of a special "No
349 	 * Response Update Transfer" command where we should clear both CmdAct
350 	 * and CmdIOC bits.
351 	 *
352 	 * With this, we don't need to wait for command completion and can
353 	 * straight away issue further commands to the endpoint.
354 	 *
355 	 * NOTICE: We're making an assumption that control endpoints will never
356 	 * make use of Update Transfer command. This is a safe assumption
357 	 * because we can never have more than one request at a time with
358 	 * Control Endpoints. If anybody changes that assumption, this chunk
359 	 * needs to be updated accordingly.
360 	 */
361 	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
362 			!usb_endpoint_xfer_isoc(desc))
363 		cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
364 	else
365 		cmd |= DWC3_DEPCMD_CMDACT;
366 
367 	dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
368 
369 	if (!(cmd & DWC3_DEPCMD_CMDACT)) {
370 		ret = 0;
371 		goto skip_status;
372 	}
373 
374 	do {
375 		reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
376 		if (!(reg & DWC3_DEPCMD_CMDACT)) {
377 			cmd_status = DWC3_DEPCMD_STATUS(reg);
378 
379 			switch (cmd_status) {
380 			case 0:
381 				ret = 0;
382 				break;
383 			case DEPEVT_TRANSFER_NO_RESOURCE:
384 				dev_WARN(dwc->dev, "No resource for %s\n",
385 					 dep->name);
386 				ret = -EINVAL;
387 				break;
388 			case DEPEVT_TRANSFER_BUS_EXPIRY:
389 				/*
390 				 * SW issues START TRANSFER command to
391 				 * isochronous ep with future frame interval. If
392 				 * future interval time has already passed when
393 				 * core receives the command, it will respond
394 				 * with an error status of 'Bus Expiry'.
395 				 *
396 				 * Instead of always returning -EINVAL, let's
397 				 * give a hint to the gadget driver that this is
398 				 * the case by returning -EAGAIN.
399 				 */
400 				ret = -EAGAIN;
401 				break;
402 			default:
403 				dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
404 			}
405 
406 			break;
407 		}
408 	} while (--timeout);
409 
410 	if (timeout == 0) {
411 		ret = -ETIMEDOUT;
412 		cmd_status = -ETIMEDOUT;
413 	}
414 
415 skip_status:
416 	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
417 
418 	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
419 		if (ret == 0)
420 			dep->flags |= DWC3_EP_TRANSFER_STARTED;
421 
422 		if (ret != -ETIMEDOUT)
423 			dwc3_gadget_ep_get_transfer_index(dep);
424 	}
425 
426 	if (saved_config) {
427 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
428 		reg |= saved_config;
429 		dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
430 	}
431 
432 	return ret;
433 }
434 
435 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
436 {
437 	struct dwc3 *dwc = dep->dwc;
438 	struct dwc3_gadget_ep_cmd_params params;
439 	u32 cmd = DWC3_DEPCMD_CLEARSTALL;
440 
441 	/*
442 	 * As of core revision 2.60a the recommended programming model
443 	 * is to set the ClearPendIN bit when issuing a Clear Stall EP
444 	 * command for IN endpoints. This is to prevent an issue where
445 	 * some (non-compliant) hosts may not send ACK TPs for pending
446 	 * IN transfers due to a mishandled error condition. Synopsys
447 	 * STAR 9000614252.
448 	 */
449 	if (dep->direction &&
450 	    !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
451 	    (dwc->gadget->speed >= USB_SPEED_SUPER))
452 		cmd |= DWC3_DEPCMD_CLEARPENDIN;
453 
454 	memset(&params, 0, sizeof(params));
455 
456 	return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
457 }
458 
459 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
460 		struct dwc3_trb *trb)
461 {
462 	u32		offset = (char *) trb - (char *) dep->trb_pool;
463 
464 	return dep->trb_pool_dma + offset;
465 }
466 
467 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
468 {
469 	struct dwc3		*dwc = dep->dwc;
470 
471 	if (dep->trb_pool)
472 		return 0;
473 
474 	dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
475 			sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
476 			&dep->trb_pool_dma, GFP_KERNEL);
477 	if (!dep->trb_pool) {
478 		dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
479 				dep->name);
480 		return -ENOMEM;
481 	}
482 
483 	return 0;
484 }
485 
486 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
487 {
488 	struct dwc3		*dwc = dep->dwc;
489 
490 	dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
491 			dep->trb_pool, dep->trb_pool_dma);
492 
493 	dep->trb_pool = NULL;
494 	dep->trb_pool_dma = 0;
495 }
496 
497 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
498 {
499 	struct dwc3_gadget_ep_cmd_params params;
500 
501 	memset(&params, 0x00, sizeof(params));
502 
503 	params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
504 
505 	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
506 			&params);
507 }
508 
509 /**
510  * dwc3_gadget_start_config - configure ep resources
511  * @dep: endpoint that is being enabled
512  *
513  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
514  * completion, it will set Transfer Resource for all available endpoints.
515  *
516  * The assignment of transfer resources cannot perfectly follow the data book
517  * due to the fact that the controller driver does not have all knowledge of the
518  * configuration in advance. It is given this information piecemeal by the
519  * composite gadget framework after every SET_CONFIGURATION and
520  * SET_INTERFACE. Trying to follow the databook programming model in this
521  * scenario can cause errors. For two reasons:
522  *
523  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
524  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
525  * incorrect in the scenario of multiple interfaces.
526  *
527  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
528  * endpoint on alt setting (8.1.6).
529  *
530  * The following simplified method is used instead:
531  *
532  * All hardware endpoints can be assigned a transfer resource and this setting
533  * will stay persistent until either a core reset or hibernation. So whenever we
534  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
535  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
536  * guaranteed that there are as many transfer resources as endpoints.
537  *
538  * This function is called for each endpoint when it is being enabled but is
539  * triggered only when called for EP0-out, which always happens first, and which
540  * should only happen in one of the above conditions.
541  */
542 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
543 {
544 	struct dwc3_gadget_ep_cmd_params params;
545 	struct dwc3		*dwc;
546 	u32			cmd;
547 	int			i;
548 	int			ret;
549 
550 	if (dep->number)
551 		return 0;
552 
553 	memset(&params, 0x00, sizeof(params));
554 	cmd = DWC3_DEPCMD_DEPSTARTCFG;
555 	dwc = dep->dwc;
556 
557 	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
558 	if (ret)
559 		return ret;
560 
561 	for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
562 		struct dwc3_ep *dep = dwc->eps[i];
563 
564 		if (!dep)
565 			continue;
566 
567 		ret = dwc3_gadget_set_xfer_resource(dep);
568 		if (ret)
569 			return ret;
570 	}
571 
572 	return 0;
573 }
574 
575 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
576 {
577 	const struct usb_ss_ep_comp_descriptor *comp_desc;
578 	const struct usb_endpoint_descriptor *desc;
579 	struct dwc3_gadget_ep_cmd_params params;
580 	struct dwc3 *dwc = dep->dwc;
581 
582 	comp_desc = dep->endpoint.comp_desc;
583 	desc = dep->endpoint.desc;
584 
585 	memset(&params, 0x00, sizeof(params));
586 
587 	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
588 		| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
589 
590 	/* Burst size is only needed in SuperSpeed mode */
591 	if (dwc->gadget->speed >= USB_SPEED_SUPER) {
592 		u32 burst = dep->endpoint.maxburst;
593 
594 		params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
595 	}
596 
597 	params.param0 |= action;
598 	if (action == DWC3_DEPCFG_ACTION_RESTORE)
599 		params.param2 |= dep->saved_state;
600 
601 	if (usb_endpoint_xfer_control(desc))
602 		params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
603 
604 	if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
605 		params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
606 
607 	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
608 		params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
609 			| DWC3_DEPCFG_XFER_COMPLETE_EN
610 			| DWC3_DEPCFG_STREAM_EVENT_EN;
611 		dep->stream_capable = true;
612 	}
613 
614 	if (!usb_endpoint_xfer_control(desc))
615 		params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
616 
617 	/*
618 	 * We are doing 1:1 mapping for endpoints, meaning
619 	 * Physical Endpoints 2 maps to Logical Endpoint 2 and
620 	 * so on. We consider the direction bit as part of the physical
621 	 * endpoint number. So USB endpoint 0x81 is 0x03.
622 	 */
623 	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
624 
625 	/*
626 	 * We must use the lower 16 TX FIFOs even though
627 	 * HW might have more
628 	 */
629 	if (dep->direction)
630 		params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
631 
632 	if (desc->bInterval) {
633 		u8 bInterval_m1;
634 
635 		/*
636 		 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
637 		 *
638 		 * NOTE: The programming guide incorrectly stated bInterval_m1
639 		 * must be set to 0 when operating in fullspeed. Internally the
640 		 * controller does not have this limitation. See DWC_usb3x
641 		 * programming guide section 3.2.2.1.
642 		 */
643 		bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
644 
645 		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
646 		    dwc->gadget->speed == USB_SPEED_FULL)
647 			dep->interval = desc->bInterval;
648 		else
649 			dep->interval = 1 << (desc->bInterval - 1);
650 
651 		params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
652 	}
653 
654 	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
655 }
656 
657 /**
658  * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
659  * @dwc: pointer to the DWC3 context
660  * @mult: multiplier to be used when calculating the fifo_size
661  *
662  * Calculates the size value based on the equation below:
663  *
664  * DWC3 revision 280A and prior:
665  * fifo_size = mult * (max_packet / mdwidth) + 1;
666  *
667  * DWC3 revision 290A and onwards:
668  * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
669  *
670  * The max packet size is set to 1024, as the txfifo requirements mainly apply
671  * to super speed USB use cases.  However, it is safe to overestimate the fifo
672  * allocations for other scenarios, i.e. high speed USB.
673  */
674 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
675 {
676 	int max_packet = 1024;
677 	int fifo_size;
678 	int mdwidth;
679 
680 	mdwidth = dwc3_mdwidth(dwc);
681 
682 	/* MDWIDTH is represented in bits, we need it in bytes */
683 	mdwidth >>= 3;
684 
685 	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
686 		fifo_size = mult * (max_packet / mdwidth) + 1;
687 	else
688 		fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
689 	return fifo_size;
690 }
691 
692 /**
693  * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
694  * @dwc: pointer to the DWC3 context
695  *
696  * Iterates through all the endpoint registers and clears the previous txfifo
697  * allocations.
698  */
699 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
700 {
701 	struct dwc3_ep *dep;
702 	int fifo_depth;
703 	int size;
704 	int num;
705 
706 	if (!dwc->do_fifo_resize)
707 		return;
708 
709 	/* Read ep0IN related TXFIFO size */
710 	dep = dwc->eps[1];
711 	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
712 	if (DWC3_IP_IS(DWC3))
713 		fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
714 	else
715 		fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
716 
717 	dwc->last_fifo_depth = fifo_depth;
718 	/* Clear existing TXFIFO for all IN eps except ep0 */
719 	for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
720 	     num += 2) {
721 		dep = dwc->eps[num];
722 		/* Don't change TXFRAMNUM on usb31 version */
723 		size = DWC3_IP_IS(DWC3) ? 0 :
724 			dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
725 				   DWC31_GTXFIFOSIZ_TXFRAMNUM;
726 
727 		dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
728 		dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
729 	}
730 	dwc->num_ep_resized = 0;
731 }
732 
733 /*
734  * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
735  * @dwc: pointer to our context structure
736  *
737  * This function will a best effort FIFO allocation in order
738  * to improve FIFO usage and throughput, while still allowing
739  * us to enable as many endpoints as possible.
740  *
741  * Keep in mind that this operation will be highly dependent
742  * on the configured size for RAM1 - which contains TxFifo -,
743  * the amount of endpoints enabled on coreConsultant tool, and
744  * the width of the Master Bus.
745  *
746  * In general, FIFO depths are represented with the following equation:
747  *
748  * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
749  *
750  * In conjunction with dwc3_gadget_check_config(), this resizing logic will
751  * ensure that all endpoints will have enough internal memory for one max
752  * packet per endpoint.
753  */
754 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
755 {
756 	struct dwc3 *dwc = dep->dwc;
757 	int fifo_0_start;
758 	int ram1_depth;
759 	int fifo_size;
760 	int min_depth;
761 	int num_in_ep;
762 	int remaining;
763 	int num_fifos = 1;
764 	int fifo;
765 	int tmp;
766 
767 	if (!dwc->do_fifo_resize)
768 		return 0;
769 
770 	/* resize IN endpoints except ep0 */
771 	if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
772 		return 0;
773 
774 	/* bail if already resized */
775 	if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
776 		return 0;
777 
778 	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
779 
780 	if ((dep->endpoint.maxburst > 1 &&
781 	     usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||
782 	    usb_endpoint_xfer_isoc(dep->endpoint.desc))
783 		num_fifos = 3;
784 
785 	if (dep->endpoint.maxburst > 6 &&
786 	    (usb_endpoint_xfer_bulk(dep->endpoint.desc) ||
787 	     usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
788 		num_fifos = dwc->tx_fifo_resize_max_num;
789 
790 	/* FIFO size for a single buffer */
791 	fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
792 
793 	/* Calculate the number of remaining EPs w/o any FIFO */
794 	num_in_ep = dwc->max_cfg_eps;
795 	num_in_ep -= dwc->num_ep_resized;
796 
797 	/* Reserve at least one FIFO for the number of IN EPs */
798 	min_depth = num_in_ep * (fifo + 1);
799 	remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
800 	remaining = max_t(int, 0, remaining);
801 	/*
802 	 * We've already reserved 1 FIFO per EP, so check what we can fit in
803 	 * addition to it.  If there is not enough remaining space, allocate
804 	 * all the remaining space to the EP.
805 	 */
806 	fifo_size = (num_fifos - 1) * fifo;
807 	if (remaining < fifo_size)
808 		fifo_size = remaining;
809 
810 	fifo_size += fifo;
811 	/* Last increment according to the TX FIFO size equation */
812 	fifo_size++;
813 
814 	/* Check if TXFIFOs start at non-zero addr */
815 	tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
816 	fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
817 
818 	fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
819 	if (DWC3_IP_IS(DWC3))
820 		dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
821 	else
822 		dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
823 
824 	/* Check fifo size allocation doesn't exceed available RAM size. */
825 	if (dwc->last_fifo_depth >= ram1_depth) {
826 		dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
827 			dwc->last_fifo_depth, ram1_depth,
828 			dep->endpoint.name, fifo_size);
829 		if (DWC3_IP_IS(DWC3))
830 			fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
831 		else
832 			fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
833 
834 		dwc->last_fifo_depth -= fifo_size;
835 		return -ENOMEM;
836 	}
837 
838 	dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
839 	dep->flags |= DWC3_EP_TXFIFO_RESIZED;
840 	dwc->num_ep_resized++;
841 
842 	return 0;
843 }
844 
845 /**
846  * __dwc3_gadget_ep_enable - initializes a hw endpoint
847  * @dep: endpoint to be initialized
848  * @action: one of INIT, MODIFY or RESTORE
849  *
850  * Caller should take care of locking. Execute all necessary commands to
851  * initialize a HW endpoint so it can be used by a gadget driver.
852  */
853 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
854 {
855 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
856 	struct dwc3		*dwc = dep->dwc;
857 
858 	u32			reg;
859 	int			ret;
860 
861 	if (!(dep->flags & DWC3_EP_ENABLED)) {
862 		ret = dwc3_gadget_resize_tx_fifos(dep);
863 		if (ret)
864 			return ret;
865 
866 		ret = dwc3_gadget_start_config(dep);
867 		if (ret)
868 			return ret;
869 	}
870 
871 	ret = dwc3_gadget_set_ep_config(dep, action);
872 	if (ret)
873 		return ret;
874 
875 	if (!(dep->flags & DWC3_EP_ENABLED)) {
876 		struct dwc3_trb	*trb_st_hw;
877 		struct dwc3_trb	*trb_link;
878 
879 		dep->type = usb_endpoint_type(desc);
880 		dep->flags |= DWC3_EP_ENABLED;
881 
882 		reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
883 		reg |= DWC3_DALEPENA_EP(dep->number);
884 		dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
885 
886 		dep->trb_dequeue = 0;
887 		dep->trb_enqueue = 0;
888 
889 		if (usb_endpoint_xfer_control(desc))
890 			goto out;
891 
892 		/* Initialize the TRB ring */
893 		memset(dep->trb_pool, 0,
894 		       sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
895 
896 		/* Link TRB. The HWO bit is never reset */
897 		trb_st_hw = &dep->trb_pool[0];
898 
899 		trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
900 		trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
901 		trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
902 		trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
903 		trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
904 	}
905 
906 	/*
907 	 * Issue StartTransfer here with no-op TRB so we can always rely on No
908 	 * Response Update Transfer command.
909 	 */
910 	if (usb_endpoint_xfer_bulk(desc) ||
911 			usb_endpoint_xfer_int(desc)) {
912 		struct dwc3_gadget_ep_cmd_params params;
913 		struct dwc3_trb	*trb;
914 		dma_addr_t trb_dma;
915 		u32 cmd;
916 
917 		memset(&params, 0, sizeof(params));
918 		trb = &dep->trb_pool[0];
919 		trb_dma = dwc3_trb_dma_offset(dep, trb);
920 
921 		params.param0 = upper_32_bits(trb_dma);
922 		params.param1 = lower_32_bits(trb_dma);
923 
924 		cmd = DWC3_DEPCMD_STARTTRANSFER;
925 
926 		ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
927 		if (ret < 0)
928 			return ret;
929 
930 		if (dep->stream_capable) {
931 			/*
932 			 * For streams, at start, there maybe a race where the
933 			 * host primes the endpoint before the function driver
934 			 * queues a request to initiate a stream. In that case,
935 			 * the controller will not see the prime to generate the
936 			 * ERDY and start stream. To workaround this, issue a
937 			 * no-op TRB as normal, but end it immediately. As a
938 			 * result, when the function driver queues the request,
939 			 * the next START_TRANSFER command will cause the
940 			 * controller to generate an ERDY to initiate the
941 			 * stream.
942 			 */
943 			dwc3_stop_active_transfer(dep, true, true);
944 
945 			/*
946 			 * All stream eps will reinitiate stream on NoStream
947 			 * rejection until we can determine that the host can
948 			 * prime after the first transfer.
949 			 *
950 			 * However, if the controller is capable of
951 			 * TXF_FLUSH_BYPASS, then IN direction endpoints will
952 			 * automatically restart the stream without the driver
953 			 * initiation.
954 			 */
955 			if (!dep->direction ||
956 			    !(dwc->hwparams.hwparams9 &
957 			      DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
958 				dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
959 		}
960 	}
961 
962 out:
963 	trace_dwc3_gadget_ep_enable(dep);
964 
965 	return 0;
966 }
967 
968 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
969 {
970 	struct dwc3_request		*req;
971 
972 	dwc3_stop_active_transfer(dep, true, false);
973 
974 	/* - giveback all requests to gadget driver */
975 	while (!list_empty(&dep->started_list)) {
976 		req = next_request(&dep->started_list);
977 
978 		dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
979 	}
980 
981 	while (!list_empty(&dep->pending_list)) {
982 		req = next_request(&dep->pending_list);
983 
984 		dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
985 	}
986 
987 	while (!list_empty(&dep->cancelled_list)) {
988 		req = next_request(&dep->cancelled_list);
989 
990 		dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
991 	}
992 }
993 
994 /**
995  * __dwc3_gadget_ep_disable - disables a hw endpoint
996  * @dep: the endpoint to disable
997  *
998  * This function undoes what __dwc3_gadget_ep_enable did and also removes
999  * requests which are currently being processed by the hardware and those which
1000  * are not yet scheduled.
1001  *
1002  * Caller should take care of locking.
1003  */
1004 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1005 {
1006 	struct dwc3		*dwc = dep->dwc;
1007 	u32			reg;
1008 
1009 	trace_dwc3_gadget_ep_disable(dep);
1010 
1011 	/* make sure HW endpoint isn't stalled */
1012 	if (dep->flags & DWC3_EP_STALL)
1013 		__dwc3_gadget_ep_set_halt(dep, 0, false);
1014 
1015 	reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
1016 	reg &= ~DWC3_DALEPENA_EP(dep->number);
1017 	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
1018 
1019 	/* Clear out the ep descriptors for non-ep0 */
1020 	if (dep->number > 1) {
1021 		dep->endpoint.comp_desc = NULL;
1022 		dep->endpoint.desc = NULL;
1023 	}
1024 
1025 	dwc3_remove_requests(dwc, dep);
1026 
1027 	dep->stream_capable = false;
1028 	dep->type = 0;
1029 	dep->flags &= DWC3_EP_TXFIFO_RESIZED;
1030 
1031 	return 0;
1032 }
1033 
1034 /* -------------------------------------------------------------------------- */
1035 
1036 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1037 		const struct usb_endpoint_descriptor *desc)
1038 {
1039 	return -EINVAL;
1040 }
1041 
1042 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1043 {
1044 	return -EINVAL;
1045 }
1046 
1047 /* -------------------------------------------------------------------------- */
1048 
1049 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1050 		const struct usb_endpoint_descriptor *desc)
1051 {
1052 	struct dwc3_ep			*dep;
1053 	struct dwc3			*dwc;
1054 	unsigned long			flags;
1055 	int				ret;
1056 
1057 	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1058 		pr_debug("dwc3: invalid parameters\n");
1059 		return -EINVAL;
1060 	}
1061 
1062 	if (!desc->wMaxPacketSize) {
1063 		pr_debug("dwc3: missing wMaxPacketSize\n");
1064 		return -EINVAL;
1065 	}
1066 
1067 	dep = to_dwc3_ep(ep);
1068 	dwc = dep->dwc;
1069 
1070 	if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1071 					"%s is already enabled\n",
1072 					dep->name))
1073 		return 0;
1074 
1075 	spin_lock_irqsave(&dwc->lock, flags);
1076 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1077 	spin_unlock_irqrestore(&dwc->lock, flags);
1078 
1079 	return ret;
1080 }
1081 
1082 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1083 {
1084 	struct dwc3_ep			*dep;
1085 	struct dwc3			*dwc;
1086 	unsigned long			flags;
1087 	int				ret;
1088 
1089 	if (!ep) {
1090 		pr_debug("dwc3: invalid parameters\n");
1091 		return -EINVAL;
1092 	}
1093 
1094 	dep = to_dwc3_ep(ep);
1095 	dwc = dep->dwc;
1096 
1097 	if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1098 					"%s is already disabled\n",
1099 					dep->name))
1100 		return 0;
1101 
1102 	spin_lock_irqsave(&dwc->lock, flags);
1103 	ret = __dwc3_gadget_ep_disable(dep);
1104 	spin_unlock_irqrestore(&dwc->lock, flags);
1105 
1106 	return ret;
1107 }
1108 
1109 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1110 		gfp_t gfp_flags)
1111 {
1112 	struct dwc3_request		*req;
1113 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1114 
1115 	req = kzalloc(sizeof(*req), gfp_flags);
1116 	if (!req)
1117 		return NULL;
1118 
1119 	req->direction	= dep->direction;
1120 	req->epnum	= dep->number;
1121 	req->dep	= dep;
1122 	req->status	= DWC3_REQUEST_STATUS_UNKNOWN;
1123 
1124 	trace_dwc3_alloc_request(req);
1125 
1126 	return &req->request;
1127 }
1128 
1129 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1130 		struct usb_request *request)
1131 {
1132 	struct dwc3_request		*req = to_dwc3_request(request);
1133 
1134 	trace_dwc3_free_request(req);
1135 	kfree(req);
1136 }
1137 
1138 /**
1139  * dwc3_ep_prev_trb - returns the previous TRB in the ring
1140  * @dep: The endpoint with the TRB ring
1141  * @index: The index of the current TRB in the ring
1142  *
1143  * Returns the TRB prior to the one pointed to by the index. If the
1144  * index is 0, we will wrap backwards, skip the link TRB, and return
1145  * the one just before that.
1146  */
1147 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1148 {
1149 	u8 tmp = index;
1150 
1151 	if (!tmp)
1152 		tmp = DWC3_TRB_NUM - 1;
1153 
1154 	return &dep->trb_pool[tmp - 1];
1155 }
1156 
1157 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1158 {
1159 	u8			trbs_left;
1160 
1161 	/*
1162 	 * If the enqueue & dequeue are equal then the TRB ring is either full
1163 	 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1164 	 * pending to be processed by the driver.
1165 	 */
1166 	if (dep->trb_enqueue == dep->trb_dequeue) {
1167 		/*
1168 		 * If there is any request remained in the started_list at
1169 		 * this point, that means there is no TRB available.
1170 		 */
1171 		if (!list_empty(&dep->started_list))
1172 			return 0;
1173 
1174 		return DWC3_TRB_NUM - 1;
1175 	}
1176 
1177 	trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1178 	trbs_left &= (DWC3_TRB_NUM - 1);
1179 
1180 	if (dep->trb_dequeue < dep->trb_enqueue)
1181 		trbs_left--;
1182 
1183 	return trbs_left;
1184 }
1185 
1186 /**
1187  * dwc3_prepare_one_trb - setup one TRB from one request
1188  * @dep: endpoint for which this request is prepared
1189  * @req: dwc3_request pointer
1190  * @trb_length: buffer size of the TRB
1191  * @chain: should this TRB be chained to the next?
1192  * @node: only for isochronous endpoints. First TRB needs different type.
1193  * @use_bounce_buffer: set to use bounce buffer
1194  * @must_interrupt: set to interrupt on TRB completion
1195  */
1196 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1197 		struct dwc3_request *req, unsigned int trb_length,
1198 		unsigned int chain, unsigned int node, bool use_bounce_buffer,
1199 		bool must_interrupt)
1200 {
1201 	struct dwc3_trb		*trb;
1202 	dma_addr_t		dma;
1203 	unsigned int		stream_id = req->request.stream_id;
1204 	unsigned int		short_not_ok = req->request.short_not_ok;
1205 	unsigned int		no_interrupt = req->request.no_interrupt;
1206 	unsigned int		is_last = req->request.is_last;
1207 	struct dwc3		*dwc = dep->dwc;
1208 	struct usb_gadget	*gadget = dwc->gadget;
1209 	enum usb_device_speed	speed = gadget->speed;
1210 
1211 	if (use_bounce_buffer)
1212 		dma = dep->dwc->bounce_addr;
1213 	else if (req->request.num_sgs > 0)
1214 		dma = sg_dma_address(req->start_sg);
1215 	else
1216 		dma = req->request.dma;
1217 
1218 	trb = &dep->trb_pool[dep->trb_enqueue];
1219 
1220 	if (!req->trb) {
1221 		dwc3_gadget_move_started_request(req);
1222 		req->trb = trb;
1223 		req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1224 	}
1225 
1226 	req->num_trbs++;
1227 
1228 	trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1229 	trb->bpl = lower_32_bits(dma);
1230 	trb->bph = upper_32_bits(dma);
1231 
1232 	switch (usb_endpoint_type(dep->endpoint.desc)) {
1233 	case USB_ENDPOINT_XFER_CONTROL:
1234 		trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1235 		break;
1236 
1237 	case USB_ENDPOINT_XFER_ISOC:
1238 		if (!node) {
1239 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1240 
1241 			/*
1242 			 * USB Specification 2.0 Section 5.9.2 states that: "If
1243 			 * there is only a single transaction in the microframe,
1244 			 * only a DATA0 data packet PID is used.  If there are
1245 			 * two transactions per microframe, DATA1 is used for
1246 			 * the first transaction data packet and DATA0 is used
1247 			 * for the second transaction data packet.  If there are
1248 			 * three transactions per microframe, DATA2 is used for
1249 			 * the first transaction data packet, DATA1 is used for
1250 			 * the second, and DATA0 is used for the third."
1251 			 *
1252 			 * IOW, we should satisfy the following cases:
1253 			 *
1254 			 * 1) length <= maxpacket
1255 			 *	- DATA0
1256 			 *
1257 			 * 2) maxpacket < length <= (2 * maxpacket)
1258 			 *	- DATA1, DATA0
1259 			 *
1260 			 * 3) (2 * maxpacket) < length <= (3 * maxpacket)
1261 			 *	- DATA2, DATA1, DATA0
1262 			 */
1263 			if (speed == USB_SPEED_HIGH) {
1264 				struct usb_ep *ep = &dep->endpoint;
1265 				unsigned int mult = 2;
1266 				unsigned int maxp = usb_endpoint_maxp(ep->desc);
1267 
1268 				if (req->request.length <= (2 * maxp))
1269 					mult--;
1270 
1271 				if (req->request.length <= maxp)
1272 					mult--;
1273 
1274 				trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1275 			}
1276 		} else {
1277 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1278 		}
1279 
1280 		/* always enable Interrupt on Missed ISOC */
1281 		trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1282 		break;
1283 
1284 	case USB_ENDPOINT_XFER_BULK:
1285 	case USB_ENDPOINT_XFER_INT:
1286 		trb->ctrl = DWC3_TRBCTL_NORMAL;
1287 		break;
1288 	default:
1289 		/*
1290 		 * This is only possible with faulty memory because we
1291 		 * checked it already :)
1292 		 */
1293 		dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1294 				usb_endpoint_type(dep->endpoint.desc));
1295 	}
1296 
1297 	/*
1298 	 * Enable Continue on Short Packet
1299 	 * when endpoint is not a stream capable
1300 	 */
1301 	if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1302 		if (!dep->stream_capable)
1303 			trb->ctrl |= DWC3_TRB_CTRL_CSP;
1304 
1305 		if (short_not_ok)
1306 			trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1307 	}
1308 
1309 	/* All TRBs setup for MST must set CSP=1 when LST=0 */
1310 	if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1311 		trb->ctrl |= DWC3_TRB_CTRL_CSP;
1312 
1313 	if ((!no_interrupt && !chain) || must_interrupt)
1314 		trb->ctrl |= DWC3_TRB_CTRL_IOC;
1315 
1316 	if (chain)
1317 		trb->ctrl |= DWC3_TRB_CTRL_CHN;
1318 	else if (dep->stream_capable && is_last &&
1319 		 !DWC3_MST_CAPABLE(&dwc->hwparams))
1320 		trb->ctrl |= DWC3_TRB_CTRL_LST;
1321 
1322 	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1323 		trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1324 
1325 	/*
1326 	 * As per data book 4.2.3.2TRB Control Bit Rules section
1327 	 *
1328 	 * The controller autonomously checks the HWO field of a TRB to determine if the
1329 	 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1330 	 * is valid before setting the HWO field to '1'. In most systems, this means that
1331 	 * software must update the fourth DWORD of a TRB last.
1332 	 *
1333 	 * However there is a possibility of CPU re-ordering here which can cause
1334 	 * controller to observe the HWO bit set prematurely.
1335 	 * Add a write memory barrier to prevent CPU re-ordering.
1336 	 */
1337 	wmb();
1338 	trb->ctrl |= DWC3_TRB_CTRL_HWO;
1339 
1340 	dwc3_ep_inc_enq(dep);
1341 
1342 	trace_dwc3_prepare_trb(dep, trb);
1343 }
1344 
1345 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1346 {
1347 	unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1348 	unsigned int rem = req->request.length % maxp;
1349 
1350 	if ((req->request.length && req->request.zero && !rem &&
1351 			!usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1352 			(!req->direction && rem))
1353 		return true;
1354 
1355 	return false;
1356 }
1357 
1358 /**
1359  * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1360  * @dep: The endpoint that the request belongs to
1361  * @req: The request to prepare
1362  * @entry_length: The last SG entry size
1363  * @node: Indicates whether this is not the first entry (for isoc only)
1364  *
1365  * Return the number of TRBs prepared.
1366  */
1367 static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1368 		struct dwc3_request *req, unsigned int entry_length,
1369 		unsigned int node)
1370 {
1371 	unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1372 	unsigned int rem = req->request.length % maxp;
1373 	unsigned int num_trbs = 1;
1374 
1375 	if (dwc3_needs_extra_trb(dep, req))
1376 		num_trbs++;
1377 
1378 	if (dwc3_calc_trbs_left(dep) < num_trbs)
1379 		return 0;
1380 
1381 	req->needs_extra_trb = num_trbs > 1;
1382 
1383 	/* Prepare a normal TRB */
1384 	if (req->direction || req->request.length)
1385 		dwc3_prepare_one_trb(dep, req, entry_length,
1386 				req->needs_extra_trb, node, false, false);
1387 
1388 	/* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1389 	if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1390 		dwc3_prepare_one_trb(dep, req,
1391 				req->direction ? 0 : maxp - rem,
1392 				false, 1, true, false);
1393 
1394 	return num_trbs;
1395 }
1396 
1397 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1398 		struct dwc3_request *req)
1399 {
1400 	struct scatterlist *sg = req->start_sg;
1401 	struct scatterlist *s;
1402 	int		i;
1403 	unsigned int length = req->request.length;
1404 	unsigned int remaining = req->request.num_mapped_sgs
1405 		- req->num_queued_sgs;
1406 	unsigned int num_trbs = req->num_trbs;
1407 	bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1408 
1409 	/*
1410 	 * If we resume preparing the request, then get the remaining length of
1411 	 * the request and resume where we left off.
1412 	 */
1413 	for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1414 		length -= sg_dma_len(s);
1415 
1416 	for_each_sg(sg, s, remaining, i) {
1417 		unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1418 		unsigned int trb_length;
1419 		bool must_interrupt = false;
1420 		bool last_sg = false;
1421 
1422 		trb_length = min_t(unsigned int, length, sg_dma_len(s));
1423 
1424 		length -= trb_length;
1425 
1426 		/*
1427 		 * IOMMU driver is coalescing the list of sgs which shares a
1428 		 * page boundary into one and giving it to USB driver. With
1429 		 * this the number of sgs mapped is not equal to the number of
1430 		 * sgs passed. So mark the chain bit to false if it isthe last
1431 		 * mapped sg.
1432 		 */
1433 		if ((i == remaining - 1) || !length)
1434 			last_sg = true;
1435 
1436 		if (!num_trbs_left)
1437 			break;
1438 
1439 		if (last_sg) {
1440 			if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1441 				break;
1442 		} else {
1443 			/*
1444 			 * Look ahead to check if we have enough TRBs for the
1445 			 * next SG entry. If not, set interrupt on this TRB to
1446 			 * resume preparing the next SG entry when more TRBs are
1447 			 * free.
1448 			 */
1449 			if (num_trbs_left == 1 || (needs_extra_trb &&
1450 					num_trbs_left <= 2 &&
1451 					sg_dma_len(sg_next(s)) >= length))
1452 				must_interrupt = true;
1453 
1454 			dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1455 					must_interrupt);
1456 		}
1457 
1458 		/*
1459 		 * There can be a situation where all sgs in sglist are not
1460 		 * queued because of insufficient trb number. To handle this
1461 		 * case, update start_sg to next sg to be queued, so that
1462 		 * we have free trbs we can continue queuing from where we
1463 		 * previously stopped
1464 		 */
1465 		if (!last_sg)
1466 			req->start_sg = sg_next(s);
1467 
1468 		req->num_queued_sgs++;
1469 		req->num_pending_sgs--;
1470 
1471 		/*
1472 		 * The number of pending SG entries may not correspond to the
1473 		 * number of mapped SG entries. If all the data are queued, then
1474 		 * don't include unused SG entries.
1475 		 */
1476 		if (length == 0) {
1477 			req->num_pending_sgs = 0;
1478 			break;
1479 		}
1480 
1481 		if (must_interrupt)
1482 			break;
1483 	}
1484 
1485 	return req->num_trbs - num_trbs;
1486 }
1487 
1488 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1489 		struct dwc3_request *req)
1490 {
1491 	return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1492 }
1493 
1494 /*
1495  * dwc3_prepare_trbs - setup TRBs from requests
1496  * @dep: endpoint for which requests are being prepared
1497  *
1498  * The function goes through the requests list and sets up TRBs for the
1499  * transfers. The function returns once there are no more TRBs available or
1500  * it runs out of requests.
1501  *
1502  * Returns the number of TRBs prepared or negative errno.
1503  */
1504 static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1505 {
1506 	struct dwc3_request	*req, *n;
1507 	int			ret = 0;
1508 
1509 	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1510 
1511 	/*
1512 	 * We can get in a situation where there's a request in the started list
1513 	 * but there weren't enough TRBs to fully kick it in the first time
1514 	 * around, so it has been waiting for more TRBs to be freed up.
1515 	 *
1516 	 * In that case, we should check if we have a request with pending_sgs
1517 	 * in the started list and prepare TRBs for that request first,
1518 	 * otherwise we will prepare TRBs completely out of order and that will
1519 	 * break things.
1520 	 */
1521 	list_for_each_entry(req, &dep->started_list, list) {
1522 		if (req->num_pending_sgs > 0) {
1523 			ret = dwc3_prepare_trbs_sg(dep, req);
1524 			if (!ret || req->num_pending_sgs)
1525 				return ret;
1526 		}
1527 
1528 		if (!dwc3_calc_trbs_left(dep))
1529 			return ret;
1530 
1531 		/*
1532 		 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1533 		 * burst capability may try to read and use TRBs beyond the
1534 		 * active transfer instead of stopping.
1535 		 */
1536 		if (dep->stream_capable && req->request.is_last &&
1537 		    !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1538 			return ret;
1539 	}
1540 
1541 	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1542 		struct dwc3	*dwc = dep->dwc;
1543 
1544 		ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1545 						    dep->direction);
1546 		if (ret)
1547 			return ret;
1548 
1549 		req->sg			= req->request.sg;
1550 		req->start_sg		= req->sg;
1551 		req->num_queued_sgs	= 0;
1552 		req->num_pending_sgs	= req->request.num_mapped_sgs;
1553 
1554 		if (req->num_pending_sgs > 0) {
1555 			ret = dwc3_prepare_trbs_sg(dep, req);
1556 			if (req->num_pending_sgs)
1557 				return ret;
1558 		} else {
1559 			ret = dwc3_prepare_trbs_linear(dep, req);
1560 		}
1561 
1562 		if (!ret || !dwc3_calc_trbs_left(dep))
1563 			return ret;
1564 
1565 		/*
1566 		 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1567 		 * burst capability may try to read and use TRBs beyond the
1568 		 * active transfer instead of stopping.
1569 		 */
1570 		if (dep->stream_capable && req->request.is_last &&
1571 		    !DWC3_MST_CAPABLE(&dwc->hwparams))
1572 			return ret;
1573 	}
1574 
1575 	return ret;
1576 }
1577 
1578 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1579 
1580 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1581 {
1582 	struct dwc3_gadget_ep_cmd_params params;
1583 	struct dwc3_request		*req;
1584 	int				starting;
1585 	int				ret;
1586 	u32				cmd;
1587 
1588 	/*
1589 	 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1590 	 * This happens when we need to stop and restart a transfer such as in
1591 	 * the case of reinitiating a stream or retrying an isoc transfer.
1592 	 */
1593 	ret = dwc3_prepare_trbs(dep);
1594 	if (ret < 0)
1595 		return ret;
1596 
1597 	starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1598 
1599 	/*
1600 	 * If there's no new TRB prepared and we don't need to restart a
1601 	 * transfer, there's no need to update the transfer.
1602 	 */
1603 	if (!ret && !starting)
1604 		return ret;
1605 
1606 	req = next_request(&dep->started_list);
1607 	if (!req) {
1608 		dep->flags |= DWC3_EP_PENDING_REQUEST;
1609 		return 0;
1610 	}
1611 
1612 	memset(&params, 0, sizeof(params));
1613 
1614 	if (starting) {
1615 		params.param0 = upper_32_bits(req->trb_dma);
1616 		params.param1 = lower_32_bits(req->trb_dma);
1617 		cmd = DWC3_DEPCMD_STARTTRANSFER;
1618 
1619 		if (dep->stream_capable)
1620 			cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1621 
1622 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1623 			cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1624 	} else {
1625 		cmd = DWC3_DEPCMD_UPDATETRANSFER |
1626 			DWC3_DEPCMD_PARAM(dep->resource_index);
1627 	}
1628 
1629 	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1630 	if (ret < 0) {
1631 		struct dwc3_request *tmp;
1632 
1633 		if (ret == -EAGAIN)
1634 			return ret;
1635 
1636 		dwc3_stop_active_transfer(dep, true, true);
1637 
1638 		list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1639 			dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1640 
1641 		/* If ep isn't started, then there's no end transfer pending */
1642 		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1643 			dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1644 
1645 		return ret;
1646 	}
1647 
1648 	if (dep->stream_capable && req->request.is_last &&
1649 	    !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1650 		dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1651 
1652 	return 0;
1653 }
1654 
1655 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1656 {
1657 	u32			reg;
1658 
1659 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1660 	return DWC3_DSTS_SOFFN(reg);
1661 }
1662 
1663 /**
1664  * __dwc3_stop_active_transfer - stop the current active transfer
1665  * @dep: isoc endpoint
1666  * @force: set forcerm bit in the command
1667  * @interrupt: command complete interrupt after End Transfer command
1668  *
1669  * When setting force, the ForceRM bit will be set. In that case
1670  * the controller won't update the TRB progress on command
1671  * completion. It also won't clear the HWO bit in the TRB.
1672  * The command will also not complete immediately in that case.
1673  */
1674 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1675 {
1676 	struct dwc3_gadget_ep_cmd_params params;
1677 	u32 cmd;
1678 	int ret;
1679 
1680 	cmd = DWC3_DEPCMD_ENDTRANSFER;
1681 	cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1682 	cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1683 	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1684 	memset(&params, 0, sizeof(params));
1685 	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1686 	WARN_ON_ONCE(ret);
1687 	dep->resource_index = 0;
1688 
1689 	if (!interrupt)
1690 		dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1691 	else if (!ret)
1692 		dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1693 
1694 	return ret;
1695 }
1696 
1697 /**
1698  * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1699  * @dep: isoc endpoint
1700  *
1701  * This function tests for the correct combination of BIT[15:14] from the 16-bit
1702  * microframe number reported by the XferNotReady event for the future frame
1703  * number to start the isoc transfer.
1704  *
1705  * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1706  * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1707  * XferNotReady event are invalid. The driver uses this number to schedule the
1708  * isochronous transfer and passes it to the START TRANSFER command. Because
1709  * this number is invalid, the command may fail. If BIT[15:14] matches the
1710  * internal 16-bit microframe, the START TRANSFER command will pass and the
1711  * transfer will start at the scheduled time, if it is off by 1, the command
1712  * will still pass, but the transfer will start 2 seconds in the future. For all
1713  * other conditions, the START TRANSFER command will fail with bus-expiry.
1714  *
1715  * In order to workaround this issue, we can test for the correct combination of
1716  * BIT[15:14] by sending START TRANSFER commands with different values of
1717  * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1718  * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1719  * As the result, within the 4 possible combinations for BIT[15:14], there will
1720  * be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1721  * command status will result in a 2-second delay start. The smaller BIT[15:14]
1722  * value is the correct combination.
1723  *
1724  * Since there are only 4 outcomes and the results are ordered, we can simply
1725  * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1726  * deduce the smaller successful combination.
1727  *
1728  * Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1729  * of BIT[15:14]. The correct combination is as follow:
1730  *
1731  * if test0 fails and test1 passes, BIT[15:14] is 'b01
1732  * if test0 fails and test1 fails, BIT[15:14] is 'b10
1733  * if test0 passes and test1 fails, BIT[15:14] is 'b11
1734  * if test0 passes and test1 passes, BIT[15:14] is 'b00
1735  *
1736  * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1737  * endpoints.
1738  */
1739 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1740 {
1741 	int cmd_status = 0;
1742 	bool test0;
1743 	bool test1;
1744 
1745 	while (dep->combo_num < 2) {
1746 		struct dwc3_gadget_ep_cmd_params params;
1747 		u32 test_frame_number;
1748 		u32 cmd;
1749 
1750 		/*
1751 		 * Check if we can start isoc transfer on the next interval or
1752 		 * 4 uframes in the future with BIT[15:14] as dep->combo_num
1753 		 */
1754 		test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1755 		test_frame_number |= dep->combo_num << 14;
1756 		test_frame_number += max_t(u32, 4, dep->interval);
1757 
1758 		params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1759 		params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1760 
1761 		cmd = DWC3_DEPCMD_STARTTRANSFER;
1762 		cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1763 		cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1764 
1765 		/* Redo if some other failure beside bus-expiry is received */
1766 		if (cmd_status && cmd_status != -EAGAIN) {
1767 			dep->start_cmd_status = 0;
1768 			dep->combo_num = 0;
1769 			return 0;
1770 		}
1771 
1772 		/* Store the first test status */
1773 		if (dep->combo_num == 0)
1774 			dep->start_cmd_status = cmd_status;
1775 
1776 		dep->combo_num++;
1777 
1778 		/*
1779 		 * End the transfer if the START_TRANSFER command is successful
1780 		 * to wait for the next XferNotReady to test the command again
1781 		 */
1782 		if (cmd_status == 0) {
1783 			dwc3_stop_active_transfer(dep, true, true);
1784 			return 0;
1785 		}
1786 	}
1787 
1788 	/* test0 and test1 are both completed at this point */
1789 	test0 = (dep->start_cmd_status == 0);
1790 	test1 = (cmd_status == 0);
1791 
1792 	if (!test0 && test1)
1793 		dep->combo_num = 1;
1794 	else if (!test0 && !test1)
1795 		dep->combo_num = 2;
1796 	else if (test0 && !test1)
1797 		dep->combo_num = 3;
1798 	else if (test0 && test1)
1799 		dep->combo_num = 0;
1800 
1801 	dep->frame_number &= DWC3_FRNUMBER_MASK;
1802 	dep->frame_number |= dep->combo_num << 14;
1803 	dep->frame_number += max_t(u32, 4, dep->interval);
1804 
1805 	/* Reinitialize test variables */
1806 	dep->start_cmd_status = 0;
1807 	dep->combo_num = 0;
1808 
1809 	return __dwc3_gadget_kick_transfer(dep);
1810 }
1811 
1812 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1813 {
1814 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1815 	struct dwc3 *dwc = dep->dwc;
1816 	int ret;
1817 	int i;
1818 
1819 	if (list_empty(&dep->pending_list) &&
1820 	    list_empty(&dep->started_list)) {
1821 		dep->flags |= DWC3_EP_PENDING_REQUEST;
1822 		return -EAGAIN;
1823 	}
1824 
1825 	if (!dwc->dis_start_transfer_quirk &&
1826 	    (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1827 	     DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1828 		if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1829 			return dwc3_gadget_start_isoc_quirk(dep);
1830 	}
1831 
1832 	if (desc->bInterval <= 14 &&
1833 	    dwc->gadget->speed >= USB_SPEED_HIGH) {
1834 		u32 frame = __dwc3_gadget_get_frame(dwc);
1835 		bool rollover = frame <
1836 				(dep->frame_number & DWC3_FRNUMBER_MASK);
1837 
1838 		/*
1839 		 * frame_number is set from XferNotReady and may be already
1840 		 * out of date. DSTS only provides the lower 14 bit of the
1841 		 * current frame number. So add the upper two bits of
1842 		 * frame_number and handle a possible rollover.
1843 		 * This will provide the correct frame_number unless more than
1844 		 * rollover has happened since XferNotReady.
1845 		 */
1846 
1847 		dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1848 				     frame;
1849 		if (rollover)
1850 			dep->frame_number += BIT(14);
1851 	}
1852 
1853 	for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1854 		int future_interval = i + 1;
1855 
1856 		/* Give the controller at least 500us to schedule transfers */
1857 		if (desc->bInterval < 3)
1858 			future_interval += 3 - desc->bInterval;
1859 
1860 		dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1861 
1862 		ret = __dwc3_gadget_kick_transfer(dep);
1863 		if (ret != -EAGAIN)
1864 			break;
1865 	}
1866 
1867 	/*
1868 	 * After a number of unsuccessful start attempts due to bus-expiry
1869 	 * status, issue END_TRANSFER command and retry on the next XferNotReady
1870 	 * event.
1871 	 */
1872 	if (ret == -EAGAIN)
1873 		ret = __dwc3_stop_active_transfer(dep, false, true);
1874 
1875 	return ret;
1876 }
1877 
1878 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1879 {
1880 	struct dwc3		*dwc = dep->dwc;
1881 
1882 	if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1883 		dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1884 				dep->name);
1885 		return -ESHUTDOWN;
1886 	}
1887 
1888 	if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1889 				&req->request, req->dep->name))
1890 		return -EINVAL;
1891 
1892 	if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1893 				"%s: request %pK already in flight\n",
1894 				dep->name, &req->request))
1895 		return -EINVAL;
1896 
1897 	pm_runtime_get(dwc->dev);
1898 
1899 	req->request.actual	= 0;
1900 	req->request.status	= -EINPROGRESS;
1901 
1902 	trace_dwc3_ep_queue(req);
1903 
1904 	list_add_tail(&req->list, &dep->pending_list);
1905 	req->status = DWC3_REQUEST_STATUS_QUEUED;
1906 
1907 	if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1908 		return 0;
1909 
1910 	/*
1911 	 * Start the transfer only after the END_TRANSFER is completed
1912 	 * and endpoint STALL is cleared.
1913 	 */
1914 	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1915 	    (dep->flags & DWC3_EP_WEDGE) ||
1916 	    (dep->flags & DWC3_EP_DELAY_STOP) ||
1917 	    (dep->flags & DWC3_EP_STALL)) {
1918 		dep->flags |= DWC3_EP_DELAY_START;
1919 		return 0;
1920 	}
1921 
1922 	/*
1923 	 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1924 	 * wait for a XferNotReady event so we will know what's the current
1925 	 * (micro-)frame number.
1926 	 *
1927 	 * Without this trick, we are very, very likely gonna get Bus Expiry
1928 	 * errors which will force us issue EndTransfer command.
1929 	 */
1930 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1931 		if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1932 			if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1933 				return __dwc3_gadget_start_isoc(dep);
1934 
1935 			return 0;
1936 		}
1937 	}
1938 
1939 	__dwc3_gadget_kick_transfer(dep);
1940 
1941 	return 0;
1942 }
1943 
1944 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1945 	gfp_t gfp_flags)
1946 {
1947 	struct dwc3_request		*req = to_dwc3_request(request);
1948 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1949 	struct dwc3			*dwc = dep->dwc;
1950 
1951 	unsigned long			flags;
1952 
1953 	int				ret;
1954 
1955 	spin_lock_irqsave(&dwc->lock, flags);
1956 	ret = __dwc3_gadget_ep_queue(dep, req);
1957 	spin_unlock_irqrestore(&dwc->lock, flags);
1958 
1959 	return ret;
1960 }
1961 
1962 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
1963 {
1964 	int i;
1965 
1966 	/* If req->trb is not set, then the request has not started */
1967 	if (!req->trb)
1968 		return;
1969 
1970 	/*
1971 	 * If request was already started, this means we had to
1972 	 * stop the transfer. With that we also need to ignore
1973 	 * all TRBs used by the request, however TRBs can only
1974 	 * be modified after completion of END_TRANSFER
1975 	 * command. So what we do here is that we wait for
1976 	 * END_TRANSFER completion and only after that, we jump
1977 	 * over TRBs by clearing HWO and incrementing dequeue
1978 	 * pointer.
1979 	 */
1980 	for (i = 0; i < req->num_trbs; i++) {
1981 		struct dwc3_trb *trb;
1982 
1983 		trb = &dep->trb_pool[dep->trb_dequeue];
1984 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1985 		dwc3_ep_inc_deq(dep);
1986 	}
1987 
1988 	req->num_trbs = 0;
1989 }
1990 
1991 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
1992 {
1993 	struct dwc3_request		*req;
1994 	struct dwc3			*dwc = dep->dwc;
1995 
1996 	while (!list_empty(&dep->cancelled_list)) {
1997 		req = next_request(&dep->cancelled_list);
1998 		dwc3_gadget_ep_skip_trbs(dep, req);
1999 		switch (req->status) {
2000 		case DWC3_REQUEST_STATUS_DISCONNECTED:
2001 			dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
2002 			break;
2003 		case DWC3_REQUEST_STATUS_DEQUEUED:
2004 			dwc3_gadget_giveback(dep, req, -ECONNRESET);
2005 			break;
2006 		case DWC3_REQUEST_STATUS_STALLED:
2007 			dwc3_gadget_giveback(dep, req, -EPIPE);
2008 			break;
2009 		default:
2010 			dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2011 			dwc3_gadget_giveback(dep, req, -ECONNRESET);
2012 			break;
2013 		}
2014 		/*
2015 		 * The endpoint is disabled, let the dwc3_remove_requests()
2016 		 * handle the cleanup.
2017 		 */
2018 		if (!dep->endpoint.desc)
2019 			break;
2020 	}
2021 }
2022 
2023 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2024 		struct usb_request *request)
2025 {
2026 	struct dwc3_request		*req = to_dwc3_request(request);
2027 	struct dwc3_request		*r = NULL;
2028 
2029 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2030 	struct dwc3			*dwc = dep->dwc;
2031 
2032 	unsigned long			flags;
2033 	int				ret = 0;
2034 
2035 	trace_dwc3_ep_dequeue(req);
2036 
2037 	spin_lock_irqsave(&dwc->lock, flags);
2038 
2039 	list_for_each_entry(r, &dep->cancelled_list, list) {
2040 		if (r == req)
2041 			goto out;
2042 	}
2043 
2044 	list_for_each_entry(r, &dep->pending_list, list) {
2045 		if (r == req) {
2046 			dwc3_gadget_giveback(dep, req, -ECONNRESET);
2047 			goto out;
2048 		}
2049 	}
2050 
2051 	list_for_each_entry(r, &dep->started_list, list) {
2052 		if (r == req) {
2053 			struct dwc3_request *t;
2054 
2055 			/* wait until it is processed */
2056 			dwc3_stop_active_transfer(dep, true, true);
2057 
2058 			/*
2059 			 * Remove any started request if the transfer is
2060 			 * cancelled.
2061 			 */
2062 			list_for_each_entry_safe(r, t, &dep->started_list, list)
2063 				dwc3_gadget_move_cancelled_request(r,
2064 						DWC3_REQUEST_STATUS_DEQUEUED);
2065 
2066 			dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2067 
2068 			goto out;
2069 		}
2070 	}
2071 
2072 	dev_err(dwc->dev, "request %pK was not queued to %s\n",
2073 		request, ep->name);
2074 	ret = -EINVAL;
2075 out:
2076 	spin_unlock_irqrestore(&dwc->lock, flags);
2077 
2078 	return ret;
2079 }
2080 
2081 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2082 {
2083 	struct dwc3_gadget_ep_cmd_params	params;
2084 	struct dwc3				*dwc = dep->dwc;
2085 	struct dwc3_request			*req;
2086 	struct dwc3_request			*tmp;
2087 	int					ret;
2088 
2089 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2090 		dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2091 		return -EINVAL;
2092 	}
2093 
2094 	memset(&params, 0x00, sizeof(params));
2095 
2096 	if (value) {
2097 		struct dwc3_trb *trb;
2098 
2099 		unsigned int transfer_in_flight;
2100 		unsigned int started;
2101 
2102 		if (dep->number > 1)
2103 			trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
2104 		else
2105 			trb = &dwc->ep0_trb[dep->trb_enqueue];
2106 
2107 		transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2108 		started = !list_empty(&dep->started_list);
2109 
2110 		if (!protocol && ((dep->direction && transfer_in_flight) ||
2111 				(!dep->direction && started))) {
2112 			return -EAGAIN;
2113 		}
2114 
2115 		ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2116 				&params);
2117 		if (ret)
2118 			dev_err(dwc->dev, "failed to set STALL on %s\n",
2119 					dep->name);
2120 		else
2121 			dep->flags |= DWC3_EP_STALL;
2122 	} else {
2123 		/*
2124 		 * Don't issue CLEAR_STALL command to control endpoints. The
2125 		 * controller automatically clears the STALL when it receives
2126 		 * the SETUP token.
2127 		 */
2128 		if (dep->number <= 1) {
2129 			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2130 			return 0;
2131 		}
2132 
2133 		dwc3_stop_active_transfer(dep, true, true);
2134 
2135 		list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2136 			dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2137 
2138 		if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2139 		    (dep->flags & DWC3_EP_DELAY_STOP)) {
2140 			dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2141 			if (protocol)
2142 				dwc->clear_stall_protocol = dep->number;
2143 
2144 			return 0;
2145 		}
2146 
2147 		dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2148 
2149 		ret = dwc3_send_clear_stall_ep_cmd(dep);
2150 		if (ret) {
2151 			dev_err(dwc->dev, "failed to clear STALL on %s\n",
2152 					dep->name);
2153 			return ret;
2154 		}
2155 
2156 		dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2157 
2158 		if ((dep->flags & DWC3_EP_DELAY_START) &&
2159 		    !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2160 			__dwc3_gadget_kick_transfer(dep);
2161 
2162 		dep->flags &= ~DWC3_EP_DELAY_START;
2163 	}
2164 
2165 	return ret;
2166 }
2167 
2168 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2169 {
2170 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2171 	struct dwc3			*dwc = dep->dwc;
2172 
2173 	unsigned long			flags;
2174 
2175 	int				ret;
2176 
2177 	spin_lock_irqsave(&dwc->lock, flags);
2178 	ret = __dwc3_gadget_ep_set_halt(dep, value, false);
2179 	spin_unlock_irqrestore(&dwc->lock, flags);
2180 
2181 	return ret;
2182 }
2183 
2184 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2185 {
2186 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2187 	struct dwc3			*dwc = dep->dwc;
2188 	unsigned long			flags;
2189 	int				ret;
2190 
2191 	spin_lock_irqsave(&dwc->lock, flags);
2192 	dep->flags |= DWC3_EP_WEDGE;
2193 
2194 	if (dep->number == 0 || dep->number == 1)
2195 		ret = __dwc3_gadget_ep0_set_halt(ep, 1);
2196 	else
2197 		ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2198 	spin_unlock_irqrestore(&dwc->lock, flags);
2199 
2200 	return ret;
2201 }
2202 
2203 /* -------------------------------------------------------------------------- */
2204 
2205 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2206 	.bLength	= USB_DT_ENDPOINT_SIZE,
2207 	.bDescriptorType = USB_DT_ENDPOINT,
2208 	.bmAttributes	= USB_ENDPOINT_XFER_CONTROL,
2209 };
2210 
2211 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2212 	.enable		= dwc3_gadget_ep0_enable,
2213 	.disable	= dwc3_gadget_ep0_disable,
2214 	.alloc_request	= dwc3_gadget_ep_alloc_request,
2215 	.free_request	= dwc3_gadget_ep_free_request,
2216 	.queue		= dwc3_gadget_ep0_queue,
2217 	.dequeue	= dwc3_gadget_ep_dequeue,
2218 	.set_halt	= dwc3_gadget_ep0_set_halt,
2219 	.set_wedge	= dwc3_gadget_ep_set_wedge,
2220 };
2221 
2222 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2223 	.enable		= dwc3_gadget_ep_enable,
2224 	.disable	= dwc3_gadget_ep_disable,
2225 	.alloc_request	= dwc3_gadget_ep_alloc_request,
2226 	.free_request	= dwc3_gadget_ep_free_request,
2227 	.queue		= dwc3_gadget_ep_queue,
2228 	.dequeue	= dwc3_gadget_ep_dequeue,
2229 	.set_halt	= dwc3_gadget_ep_set_halt,
2230 	.set_wedge	= dwc3_gadget_ep_set_wedge,
2231 };
2232 
2233 /* -------------------------------------------------------------------------- */
2234 
2235 static int dwc3_gadget_get_frame(struct usb_gadget *g)
2236 {
2237 	struct dwc3		*dwc = gadget_to_dwc(g);
2238 
2239 	return __dwc3_gadget_get_frame(dwc);
2240 }
2241 
2242 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
2243 {
2244 	int			retries;
2245 
2246 	int			ret;
2247 	u32			reg;
2248 
2249 	u8			link_state;
2250 
2251 	/*
2252 	 * According to the Databook Remote wakeup request should
2253 	 * be issued only when the device is in early suspend state.
2254 	 *
2255 	 * We can check that via USB Link State bits in DSTS register.
2256 	 */
2257 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2258 
2259 	link_state = DWC3_DSTS_USBLNKST(reg);
2260 
2261 	switch (link_state) {
2262 	case DWC3_LINK_STATE_RESET:
2263 	case DWC3_LINK_STATE_RX_DET:	/* in HS, means Early Suspend */
2264 	case DWC3_LINK_STATE_U3:	/* in HS, means SUSPEND */
2265 	case DWC3_LINK_STATE_U2:	/* in HS, means Sleep (L1) */
2266 	case DWC3_LINK_STATE_U1:
2267 	case DWC3_LINK_STATE_RESUME:
2268 		break;
2269 	default:
2270 		return -EINVAL;
2271 	}
2272 
2273 	ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2274 	if (ret < 0) {
2275 		dev_err(dwc->dev, "failed to put link in Recovery\n");
2276 		return ret;
2277 	}
2278 
2279 	/* Recent versions do this automatically */
2280 	if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2281 		/* write zeroes to Link Change Request */
2282 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2283 		reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2284 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2285 	}
2286 
2287 	/* poll until Link State changes to ON */
2288 	retries = 20000;
2289 
2290 	while (retries--) {
2291 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2292 
2293 		/* in HS, means ON */
2294 		if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2295 			break;
2296 	}
2297 
2298 	if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2299 		dev_err(dwc->dev, "failed to send remote wakeup\n");
2300 		return -EINVAL;
2301 	}
2302 
2303 	return 0;
2304 }
2305 
2306 static int dwc3_gadget_wakeup(struct usb_gadget *g)
2307 {
2308 	struct dwc3		*dwc = gadget_to_dwc(g);
2309 	unsigned long		flags;
2310 	int			ret;
2311 
2312 	spin_lock_irqsave(&dwc->lock, flags);
2313 	ret = __dwc3_gadget_wakeup(dwc);
2314 	spin_unlock_irqrestore(&dwc->lock, flags);
2315 
2316 	return ret;
2317 }
2318 
2319 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2320 		int is_selfpowered)
2321 {
2322 	struct dwc3		*dwc = gadget_to_dwc(g);
2323 	unsigned long		flags;
2324 
2325 	spin_lock_irqsave(&dwc->lock, flags);
2326 	g->is_selfpowered = !!is_selfpowered;
2327 	spin_unlock_irqrestore(&dwc->lock, flags);
2328 
2329 	return 0;
2330 }
2331 
2332 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2333 {
2334 	u32 epnum;
2335 
2336 	for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2337 		struct dwc3_ep *dep;
2338 
2339 		dep = dwc->eps[epnum];
2340 		if (!dep)
2341 			continue;
2342 
2343 		dwc3_remove_requests(dwc, dep);
2344 	}
2345 }
2346 
2347 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2348 {
2349 	enum usb_ssp_rate	ssp_rate = dwc->gadget_ssp_rate;
2350 	u32			reg;
2351 
2352 	if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2353 		ssp_rate = dwc->max_ssp_rate;
2354 
2355 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2356 	reg &= ~DWC3_DCFG_SPEED_MASK;
2357 	reg &= ~DWC3_DCFG_NUMLANES(~0);
2358 
2359 	if (ssp_rate == USB_SSP_GEN_1x2)
2360 		reg |= DWC3_DCFG_SUPERSPEED;
2361 	else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2362 		reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2363 
2364 	if (ssp_rate != USB_SSP_GEN_2x1 &&
2365 	    dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2366 		reg |= DWC3_DCFG_NUMLANES(1);
2367 
2368 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2369 }
2370 
2371 static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2372 {
2373 	enum usb_device_speed	speed;
2374 	u32			reg;
2375 
2376 	speed = dwc->gadget_max_speed;
2377 	if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2378 		speed = dwc->maximum_speed;
2379 
2380 	if (speed == USB_SPEED_SUPER_PLUS &&
2381 	    DWC3_IP_IS(DWC32)) {
2382 		__dwc3_gadget_set_ssp_rate(dwc);
2383 		return;
2384 	}
2385 
2386 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2387 	reg &= ~(DWC3_DCFG_SPEED_MASK);
2388 
2389 	/*
2390 	 * WORKAROUND: DWC3 revision < 2.20a have an issue
2391 	 * which would cause metastability state on Run/Stop
2392 	 * bit if we try to force the IP to USB2-only mode.
2393 	 *
2394 	 * Because of that, we cannot configure the IP to any
2395 	 * speed other than the SuperSpeed
2396 	 *
2397 	 * Refers to:
2398 	 *
2399 	 * STAR#9000525659: Clock Domain Crossing on DCTL in
2400 	 * USB 2.0 Mode
2401 	 */
2402 	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2403 	    !dwc->dis_metastability_quirk) {
2404 		reg |= DWC3_DCFG_SUPERSPEED;
2405 	} else {
2406 		switch (speed) {
2407 		case USB_SPEED_FULL:
2408 			reg |= DWC3_DCFG_FULLSPEED;
2409 			break;
2410 		case USB_SPEED_HIGH:
2411 			reg |= DWC3_DCFG_HIGHSPEED;
2412 			break;
2413 		case USB_SPEED_SUPER:
2414 			reg |= DWC3_DCFG_SUPERSPEED;
2415 			break;
2416 		case USB_SPEED_SUPER_PLUS:
2417 			if (DWC3_IP_IS(DWC3))
2418 				reg |= DWC3_DCFG_SUPERSPEED;
2419 			else
2420 				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2421 			break;
2422 		default:
2423 			dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2424 
2425 			if (DWC3_IP_IS(DWC3))
2426 				reg |= DWC3_DCFG_SUPERSPEED;
2427 			else
2428 				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2429 		}
2430 	}
2431 
2432 	if (DWC3_IP_IS(DWC32) &&
2433 	    speed > USB_SPEED_UNKNOWN &&
2434 	    speed < USB_SPEED_SUPER_PLUS)
2435 		reg &= ~DWC3_DCFG_NUMLANES(~0);
2436 
2437 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2438 }
2439 
2440 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
2441 {
2442 	u32			reg;
2443 	u32			timeout = 500;
2444 
2445 	if (pm_runtime_suspended(dwc->dev))
2446 		return 0;
2447 
2448 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2449 	if (is_on) {
2450 		if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2451 			reg &= ~DWC3_DCTL_TRGTULST_MASK;
2452 			reg |= DWC3_DCTL_TRGTULST_RX_DET;
2453 		}
2454 
2455 		if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2456 			reg &= ~DWC3_DCTL_KEEP_CONNECT;
2457 		reg |= DWC3_DCTL_RUN_STOP;
2458 
2459 		if (dwc->has_hibernation)
2460 			reg |= DWC3_DCTL_KEEP_CONNECT;
2461 
2462 		__dwc3_gadget_set_speed(dwc);
2463 		dwc->pullups_connected = true;
2464 	} else {
2465 		reg &= ~DWC3_DCTL_RUN_STOP;
2466 
2467 		if (dwc->has_hibernation && !suspend)
2468 			reg &= ~DWC3_DCTL_KEEP_CONNECT;
2469 
2470 		dwc->pullups_connected = false;
2471 	}
2472 
2473 	dwc3_gadget_dctl_write_safe(dwc, reg);
2474 
2475 	do {
2476 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2477 		reg &= DWC3_DSTS_DEVCTRLHLT;
2478 	} while (--timeout && !(!is_on ^ !reg));
2479 
2480 	if (!timeout)
2481 		return -ETIMEDOUT;
2482 
2483 	return 0;
2484 }
2485 
2486 static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2487 static void __dwc3_gadget_stop(struct dwc3 *dwc);
2488 static int __dwc3_gadget_start(struct dwc3 *dwc);
2489 
2490 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
2491 {
2492 	unsigned long flags;
2493 
2494 	spin_lock_irqsave(&dwc->lock, flags);
2495 	dwc->connected = false;
2496 
2497 	/*
2498 	 * Per databook, when we want to stop the gadget, if a control transfer
2499 	 * is still in process, complete it and get the core into setup phase.
2500 	 */
2501 	if (dwc->ep0state != EP0_SETUP_PHASE) {
2502 		int ret;
2503 
2504 		reinit_completion(&dwc->ep0_in_setup);
2505 
2506 		spin_unlock_irqrestore(&dwc->lock, flags);
2507 		ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2508 				msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2509 		spin_lock_irqsave(&dwc->lock, flags);
2510 		if (ret == 0)
2511 			dev_warn(dwc->dev, "timed out waiting for SETUP phase\n");
2512 	}
2513 
2514 	/*
2515 	 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
2516 	 * Section 4.1.8 Table 4-7, it states that for a device-initiated
2517 	 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2518 	 * command for any active transfers" before clearing the RunStop
2519 	 * bit.
2520 	 */
2521 	dwc3_stop_active_transfers(dwc);
2522 	__dwc3_gadget_stop(dwc);
2523 	spin_unlock_irqrestore(&dwc->lock, flags);
2524 
2525 	/*
2526 	 * Note: if the GEVNTCOUNT indicates events in the event buffer, the
2527 	 * driver needs to acknowledge them before the controller can halt.
2528 	 * Simply let the interrupt handler acknowledges and handle the
2529 	 * remaining event generated by the controller while polling for
2530 	 * DSTS.DEVCTLHLT.
2531 	 */
2532 	return dwc3_gadget_run_stop(dwc, false, false);
2533 }
2534 
2535 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2536 {
2537 	struct dwc3		*dwc = gadget_to_dwc(g);
2538 	int			ret;
2539 
2540 	is_on = !!is_on;
2541 
2542 	if (dwc->pullups_connected == is_on)
2543 		return 0;
2544 
2545 	dwc->softconnect = is_on;
2546 
2547 	/*
2548 	 * Avoid issuing a runtime resume if the device is already in the
2549 	 * suspended state during gadget disconnect.  DWC3 gadget was already
2550 	 * halted/stopped during runtime suspend.
2551 	 */
2552 	if (!is_on) {
2553 		pm_runtime_barrier(dwc->dev);
2554 		if (pm_runtime_suspended(dwc->dev))
2555 			return 0;
2556 	}
2557 
2558 	/*
2559 	 * Check the return value for successful resume, or error.  For a
2560 	 * successful resume, the DWC3 runtime PM resume routine will handle
2561 	 * the run stop sequence, so avoid duplicate operations here.
2562 	 */
2563 	ret = pm_runtime_get_sync(dwc->dev);
2564 	if (!ret || ret < 0) {
2565 		pm_runtime_put(dwc->dev);
2566 		return 0;
2567 	}
2568 
2569 	if (!is_on) {
2570 		ret = dwc3_gadget_soft_disconnect(dwc);
2571 	} else {
2572 		/*
2573 		 * In the Synopsys DWC_usb31 1.90a programming guide section
2574 		 * 4.1.9, it specifies that for a reconnect after a
2575 		 * device-initiated disconnect requires a core soft reset
2576 		 * (DCTL.CSftRst) before enabling the run/stop bit.
2577 		 */
2578 		dwc3_core_soft_reset(dwc);
2579 
2580 		dwc3_event_buffers_setup(dwc);
2581 		__dwc3_gadget_start(dwc);
2582 		ret = dwc3_gadget_run_stop(dwc, true, false);
2583 	}
2584 
2585 	pm_runtime_put(dwc->dev);
2586 
2587 	return ret;
2588 }
2589 
2590 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2591 {
2592 	u32			reg;
2593 
2594 	/* Enable all but Start and End of Frame IRQs */
2595 	reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2596 			DWC3_DEVTEN_CMDCMPLTEN |
2597 			DWC3_DEVTEN_ERRTICERREN |
2598 			DWC3_DEVTEN_WKUPEVTEN |
2599 			DWC3_DEVTEN_CONNECTDONEEN |
2600 			DWC3_DEVTEN_USBRSTEN |
2601 			DWC3_DEVTEN_DISCONNEVTEN);
2602 
2603 	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2604 		reg |= DWC3_DEVTEN_ULSTCNGEN;
2605 
2606 	/* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2607 	if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2608 		reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2609 
2610 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2611 }
2612 
2613 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2614 {
2615 	/* mask all interrupts */
2616 	dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2617 }
2618 
2619 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2620 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2621 
2622 /**
2623  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2624  * @dwc: pointer to our context structure
2625  *
2626  * The following looks like complex but it's actually very simple. In order to
2627  * calculate the number of packets we can burst at once on OUT transfers, we're
2628  * gonna use RxFIFO size.
2629  *
2630  * To calculate RxFIFO size we need two numbers:
2631  * MDWIDTH = size, in bits, of the internal memory bus
2632  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2633  *
2634  * Given these two numbers, the formula is simple:
2635  *
2636  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2637  *
2638  * 24 bytes is for 3x SETUP packets
2639  * 16 bytes is a clock domain crossing tolerance
2640  *
2641  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2642  */
2643 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2644 {
2645 	u32 ram2_depth;
2646 	u32 mdwidth;
2647 	u32 nump;
2648 	u32 reg;
2649 
2650 	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2651 	mdwidth = dwc3_mdwidth(dwc);
2652 
2653 	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2654 	nump = min_t(u32, nump, 16);
2655 
2656 	/* update NumP */
2657 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2658 	reg &= ~DWC3_DCFG_NUMP_MASK;
2659 	reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2660 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2661 }
2662 
2663 static int __dwc3_gadget_start(struct dwc3 *dwc)
2664 {
2665 	struct dwc3_ep		*dep;
2666 	int			ret = 0;
2667 	u32			reg;
2668 
2669 	/*
2670 	 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2671 	 * the core supports IMOD, disable it.
2672 	 */
2673 	if (dwc->imod_interval) {
2674 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2675 		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2676 	} else if (dwc3_has_imod(dwc)) {
2677 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2678 	}
2679 
2680 	/*
2681 	 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2682 	 * field instead of letting dwc3 itself calculate that automatically.
2683 	 *
2684 	 * This way, we maximize the chances that we'll be able to get several
2685 	 * bursts of data without going through any sort of endpoint throttling.
2686 	 */
2687 	reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2688 	if (DWC3_IP_IS(DWC3))
2689 		reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2690 	else
2691 		reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2692 
2693 	dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2694 
2695 	dwc3_gadget_setup_nump(dwc);
2696 
2697 	/*
2698 	 * Currently the controller handles single stream only. So, Ignore
2699 	 * Packet Pending bit for stream selection and don't search for another
2700 	 * stream if the host sends Data Packet with PP=0 (for OUT direction) or
2701 	 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2702 	 * the stream performance.
2703 	 */
2704 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2705 	reg |= DWC3_DCFG_IGNSTRMPP;
2706 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2707 
2708 	/* Enable MST by default if the device is capable of MST */
2709 	if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2710 		reg = dwc3_readl(dwc->regs, DWC3_DCFG1);
2711 		reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2712 		dwc3_writel(dwc->regs, DWC3_DCFG1, reg);
2713 	}
2714 
2715 	/* Start with SuperSpeed Default */
2716 	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2717 
2718 	dep = dwc->eps[0];
2719 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2720 	if (ret) {
2721 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2722 		goto err0;
2723 	}
2724 
2725 	dep = dwc->eps[1];
2726 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2727 	if (ret) {
2728 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2729 		goto err1;
2730 	}
2731 
2732 	/* begin to receive SETUP packets */
2733 	dwc->ep0state = EP0_SETUP_PHASE;
2734 	dwc->ep0_bounced = false;
2735 	dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2736 	dwc->delayed_status = false;
2737 	dwc3_ep0_out_start(dwc);
2738 
2739 	dwc3_gadget_enable_irq(dwc);
2740 
2741 	return 0;
2742 
2743 err1:
2744 	__dwc3_gadget_ep_disable(dwc->eps[0]);
2745 
2746 err0:
2747 	return ret;
2748 }
2749 
2750 static int dwc3_gadget_start(struct usb_gadget *g,
2751 		struct usb_gadget_driver *driver)
2752 {
2753 	struct dwc3		*dwc = gadget_to_dwc(g);
2754 	unsigned long		flags;
2755 	int			ret;
2756 	int			irq;
2757 
2758 	irq = dwc->irq_gadget;
2759 	ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2760 			IRQF_SHARED, "dwc3", dwc->ev_buf);
2761 	if (ret) {
2762 		dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2763 				irq, ret);
2764 		return ret;
2765 	}
2766 
2767 	spin_lock_irqsave(&dwc->lock, flags);
2768 	dwc->gadget_driver	= driver;
2769 	spin_unlock_irqrestore(&dwc->lock, flags);
2770 
2771 	return 0;
2772 }
2773 
2774 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2775 {
2776 	dwc3_gadget_disable_irq(dwc);
2777 	__dwc3_gadget_ep_disable(dwc->eps[0]);
2778 	__dwc3_gadget_ep_disable(dwc->eps[1]);
2779 }
2780 
2781 static int dwc3_gadget_stop(struct usb_gadget *g)
2782 {
2783 	struct dwc3		*dwc = gadget_to_dwc(g);
2784 	unsigned long		flags;
2785 
2786 	spin_lock_irqsave(&dwc->lock, flags);
2787 	dwc->gadget_driver	= NULL;
2788 	dwc->max_cfg_eps = 0;
2789 	spin_unlock_irqrestore(&dwc->lock, flags);
2790 
2791 	free_irq(dwc->irq_gadget, dwc->ev_buf);
2792 
2793 	return 0;
2794 }
2795 
2796 static void dwc3_gadget_config_params(struct usb_gadget *g,
2797 				      struct usb_dcd_config_params *params)
2798 {
2799 	struct dwc3		*dwc = gadget_to_dwc(g);
2800 
2801 	params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
2802 	params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
2803 
2804 	/* Recommended BESL */
2805 	if (!dwc->dis_enblslpm_quirk) {
2806 		/*
2807 		 * If the recommended BESL baseline is 0 or if the BESL deep is
2808 		 * less than 2, Microsoft's Windows 10 host usb stack will issue
2809 		 * a usb reset immediately after it receives the extended BOS
2810 		 * descriptor and the enumeration will fail. To maintain
2811 		 * compatibility with the Windows' usb stack, let's set the
2812 		 * recommended BESL baseline to 1 and clamp the BESL deep to be
2813 		 * within 2 to 15.
2814 		 */
2815 		params->besl_baseline = 1;
2816 		if (dwc->is_utmi_l1_suspend)
2817 			params->besl_deep =
2818 				clamp_t(u8, dwc->hird_threshold, 2, 15);
2819 	}
2820 
2821 	/* U1 Device exit Latency */
2822 	if (dwc->dis_u1_entry_quirk)
2823 		params->bU1devExitLat = 0;
2824 	else
2825 		params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
2826 
2827 	/* U2 Device exit Latency */
2828 	if (dwc->dis_u2_entry_quirk)
2829 		params->bU2DevExitLat = 0;
2830 	else
2831 		params->bU2DevExitLat =
2832 				cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
2833 }
2834 
2835 static void dwc3_gadget_set_speed(struct usb_gadget *g,
2836 				  enum usb_device_speed speed)
2837 {
2838 	struct dwc3		*dwc = gadget_to_dwc(g);
2839 	unsigned long		flags;
2840 
2841 	spin_lock_irqsave(&dwc->lock, flags);
2842 	dwc->gadget_max_speed = speed;
2843 	spin_unlock_irqrestore(&dwc->lock, flags);
2844 }
2845 
2846 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
2847 				     enum usb_ssp_rate rate)
2848 {
2849 	struct dwc3		*dwc = gadget_to_dwc(g);
2850 	unsigned long		flags;
2851 
2852 	spin_lock_irqsave(&dwc->lock, flags);
2853 	dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
2854 	dwc->gadget_ssp_rate = rate;
2855 	spin_unlock_irqrestore(&dwc->lock, flags);
2856 }
2857 
2858 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
2859 {
2860 	struct dwc3		*dwc = gadget_to_dwc(g);
2861 	union power_supply_propval	val = {0};
2862 	int				ret;
2863 
2864 	if (dwc->usb2_phy)
2865 		return usb_phy_set_power(dwc->usb2_phy, mA);
2866 
2867 	if (!dwc->usb_psy)
2868 		return -EOPNOTSUPP;
2869 
2870 	val.intval = 1000 * mA;
2871 	ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
2872 
2873 	return ret;
2874 }
2875 
2876 /**
2877  * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
2878  * @g: pointer to the USB gadget
2879  *
2880  * Used to record the maximum number of endpoints being used in a USB composite
2881  * device. (across all configurations)  This is to be used in the calculation
2882  * of the TXFIFO sizes when resizing internal memory for individual endpoints.
2883  * It will help ensured that the resizing logic reserves enough space for at
2884  * least one max packet.
2885  */
2886 static int dwc3_gadget_check_config(struct usb_gadget *g)
2887 {
2888 	struct dwc3 *dwc = gadget_to_dwc(g);
2889 	struct usb_ep *ep;
2890 	int fifo_size = 0;
2891 	int ram1_depth;
2892 	int ep_num = 0;
2893 
2894 	if (!dwc->do_fifo_resize)
2895 		return 0;
2896 
2897 	list_for_each_entry(ep, &g->ep_list, ep_list) {
2898 		/* Only interested in the IN endpoints */
2899 		if (ep->claimed && (ep->address & USB_DIR_IN))
2900 			ep_num++;
2901 	}
2902 
2903 	if (ep_num <= dwc->max_cfg_eps)
2904 		return 0;
2905 
2906 	/* Update the max number of eps in the composition */
2907 	dwc->max_cfg_eps = ep_num;
2908 
2909 	fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
2910 	/* Based on the equation, increment by one for every ep */
2911 	fifo_size += dwc->max_cfg_eps;
2912 
2913 	/* Check if we can fit a single fifo per endpoint */
2914 	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
2915 	if (fifo_size > ram1_depth)
2916 		return -ENOMEM;
2917 
2918 	return 0;
2919 }
2920 
2921 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
2922 {
2923 	struct dwc3		*dwc = gadget_to_dwc(g);
2924 	unsigned long		flags;
2925 
2926 	spin_lock_irqsave(&dwc->lock, flags);
2927 	dwc->async_callbacks = enable;
2928 	spin_unlock_irqrestore(&dwc->lock, flags);
2929 }
2930 
2931 static const struct usb_gadget_ops dwc3_gadget_ops = {
2932 	.get_frame		= dwc3_gadget_get_frame,
2933 	.wakeup			= dwc3_gadget_wakeup,
2934 	.set_selfpowered	= dwc3_gadget_set_selfpowered,
2935 	.pullup			= dwc3_gadget_pullup,
2936 	.udc_start		= dwc3_gadget_start,
2937 	.udc_stop		= dwc3_gadget_stop,
2938 	.udc_set_speed		= dwc3_gadget_set_speed,
2939 	.udc_set_ssp_rate	= dwc3_gadget_set_ssp_rate,
2940 	.get_config_params	= dwc3_gadget_config_params,
2941 	.vbus_draw		= dwc3_gadget_vbus_draw,
2942 	.check_config		= dwc3_gadget_check_config,
2943 	.udc_async_callbacks	= dwc3_gadget_async_callbacks,
2944 };
2945 
2946 /* -------------------------------------------------------------------------- */
2947 
2948 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2949 {
2950 	struct dwc3 *dwc = dep->dwc;
2951 
2952 	usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2953 	dep->endpoint.maxburst = 1;
2954 	dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2955 	if (!dep->direction)
2956 		dwc->gadget->ep0 = &dep->endpoint;
2957 
2958 	dep->endpoint.caps.type_control = true;
2959 
2960 	return 0;
2961 }
2962 
2963 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2964 {
2965 	struct dwc3 *dwc = dep->dwc;
2966 	u32 mdwidth;
2967 	int size;
2968 	int maxpacket;
2969 
2970 	mdwidth = dwc3_mdwidth(dwc);
2971 
2972 	/* MDWIDTH is represented in bits, we need it in bytes */
2973 	mdwidth /= 8;
2974 
2975 	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2976 	if (DWC3_IP_IS(DWC3))
2977 		size = DWC3_GTXFIFOSIZ_TXFDEP(size);
2978 	else
2979 		size = DWC31_GTXFIFOSIZ_TXFDEP(size);
2980 
2981 	/*
2982 	 * maxpacket size is determined as part of the following, after assuming
2983 	 * a mult value of one maxpacket:
2984 	 * DWC3 revision 280A and prior:
2985 	 * fifo_size = mult * (max_packet / mdwidth) + 1;
2986 	 * maxpacket = mdwidth * (fifo_size - 1);
2987 	 *
2988 	 * DWC3 revision 290A and onwards:
2989 	 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
2990 	 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
2991 	 */
2992 	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
2993 		maxpacket = mdwidth * (size - 1);
2994 	else
2995 		maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
2996 
2997 	/* Functionally, space for one max packet is sufficient */
2998 	size = min_t(int, maxpacket, 1024);
2999 	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3000 
3001 	dep->endpoint.max_streams = 16;
3002 	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3003 	list_add_tail(&dep->endpoint.ep_list,
3004 			&dwc->gadget->ep_list);
3005 	dep->endpoint.caps.type_iso = true;
3006 	dep->endpoint.caps.type_bulk = true;
3007 	dep->endpoint.caps.type_int = true;
3008 
3009 	return dwc3_alloc_trb_pool(dep);
3010 }
3011 
3012 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3013 {
3014 	struct dwc3 *dwc = dep->dwc;
3015 	u32 mdwidth;
3016 	int size;
3017 
3018 	mdwidth = dwc3_mdwidth(dwc);
3019 
3020 	/* MDWIDTH is represented in bits, convert to bytes */
3021 	mdwidth /= 8;
3022 
3023 	/* All OUT endpoints share a single RxFIFO space */
3024 	size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
3025 	if (DWC3_IP_IS(DWC3))
3026 		size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3027 	else
3028 		size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3029 
3030 	/* FIFO depth is in MDWDITH bytes */
3031 	size *= mdwidth;
3032 
3033 	/*
3034 	 * To meet performance requirement, a minimum recommended RxFIFO size
3035 	 * is defined as follow:
3036 	 * RxFIFO size >= (3 x MaxPacketSize) +
3037 	 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3038 	 *
3039 	 * Then calculate the max packet limit as below.
3040 	 */
3041 	size -= (3 * 8) + 16;
3042 	if (size < 0)
3043 		size = 0;
3044 	else
3045 		size /= 3;
3046 
3047 	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3048 	dep->endpoint.max_streams = 16;
3049 	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3050 	list_add_tail(&dep->endpoint.ep_list,
3051 			&dwc->gadget->ep_list);
3052 	dep->endpoint.caps.type_iso = true;
3053 	dep->endpoint.caps.type_bulk = true;
3054 	dep->endpoint.caps.type_int = true;
3055 
3056 	return dwc3_alloc_trb_pool(dep);
3057 }
3058 
3059 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3060 {
3061 	struct dwc3_ep			*dep;
3062 	bool				direction = epnum & 1;
3063 	int				ret;
3064 	u8				num = epnum >> 1;
3065 
3066 	dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3067 	if (!dep)
3068 		return -ENOMEM;
3069 
3070 	dep->dwc = dwc;
3071 	dep->number = epnum;
3072 	dep->direction = direction;
3073 	dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3074 	dwc->eps[epnum] = dep;
3075 	dep->combo_num = 0;
3076 	dep->start_cmd_status = 0;
3077 
3078 	snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
3079 			direction ? "in" : "out");
3080 
3081 	dep->endpoint.name = dep->name;
3082 
3083 	if (!(dep->number > 1)) {
3084 		dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3085 		dep->endpoint.comp_desc = NULL;
3086 	}
3087 
3088 	if (num == 0)
3089 		ret = dwc3_gadget_init_control_endpoint(dep);
3090 	else if (direction)
3091 		ret = dwc3_gadget_init_in_endpoint(dep);
3092 	else
3093 		ret = dwc3_gadget_init_out_endpoint(dep);
3094 
3095 	if (ret)
3096 		return ret;
3097 
3098 	dep->endpoint.caps.dir_in = direction;
3099 	dep->endpoint.caps.dir_out = !direction;
3100 
3101 	INIT_LIST_HEAD(&dep->pending_list);
3102 	INIT_LIST_HEAD(&dep->started_list);
3103 	INIT_LIST_HEAD(&dep->cancelled_list);
3104 
3105 	dwc3_debugfs_create_endpoint_dir(dep);
3106 
3107 	return 0;
3108 }
3109 
3110 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3111 {
3112 	u8				epnum;
3113 
3114 	INIT_LIST_HEAD(&dwc->gadget->ep_list);
3115 
3116 	for (epnum = 0; epnum < total; epnum++) {
3117 		int			ret;
3118 
3119 		ret = dwc3_gadget_init_endpoint(dwc, epnum);
3120 		if (ret)
3121 			return ret;
3122 	}
3123 
3124 	return 0;
3125 }
3126 
3127 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3128 {
3129 	struct dwc3_ep			*dep;
3130 	u8				epnum;
3131 
3132 	for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3133 		dep = dwc->eps[epnum];
3134 		if (!dep)
3135 			continue;
3136 		/*
3137 		 * Physical endpoints 0 and 1 are special; they form the
3138 		 * bi-directional USB endpoint 0.
3139 		 *
3140 		 * For those two physical endpoints, we don't allocate a TRB
3141 		 * pool nor do we add them the endpoints list. Due to that, we
3142 		 * shouldn't do these two operations otherwise we would end up
3143 		 * with all sorts of bugs when removing dwc3.ko.
3144 		 */
3145 		if (epnum != 0 && epnum != 1) {
3146 			dwc3_free_trb_pool(dep);
3147 			list_del(&dep->endpoint.ep_list);
3148 		}
3149 
3150 		debugfs_remove_recursive(debugfs_lookup(dep->name,
3151 				debugfs_lookup(dev_name(dep->dwc->dev),
3152 					       usb_debug_root)));
3153 		kfree(dep);
3154 	}
3155 }
3156 
3157 /* -------------------------------------------------------------------------- */
3158 
3159 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3160 		struct dwc3_request *req, struct dwc3_trb *trb,
3161 		const struct dwc3_event_depevt *event, int status, int chain)
3162 {
3163 	unsigned int		count;
3164 
3165 	dwc3_ep_inc_deq(dep);
3166 
3167 	trace_dwc3_complete_trb(dep, trb);
3168 	req->num_trbs--;
3169 
3170 	/*
3171 	 * If we're in the middle of series of chained TRBs and we
3172 	 * receive a short transfer along the way, DWC3 will skip
3173 	 * through all TRBs including the last TRB in the chain (the
3174 	 * where CHN bit is zero. DWC3 will also avoid clearing HWO
3175 	 * bit and SW has to do it manually.
3176 	 *
3177 	 * We're going to do that here to avoid problems of HW trying
3178 	 * to use bogus TRBs for transfers.
3179 	 */
3180 	if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3181 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3182 
3183 	/*
3184 	 * For isochronous transfers, the first TRB in a service interval must
3185 	 * have the Isoc-First type. Track and report its interval frame number.
3186 	 */
3187 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3188 	    (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3189 		unsigned int frame_number;
3190 
3191 		frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3192 		frame_number &= ~(dep->interval - 1);
3193 		req->request.frame_number = frame_number;
3194 	}
3195 
3196 	/*
3197 	 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3198 	 * this TRB points to the bounce buffer address, it's a MPS alignment
3199 	 * TRB. Don't add it to req->remaining calculation.
3200 	 */
3201 	if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3202 	    trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3203 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3204 		return 1;
3205 	}
3206 
3207 	count = trb->size & DWC3_TRB_SIZE_MASK;
3208 	req->remaining += count;
3209 
3210 	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3211 		return 1;
3212 
3213 	if (event->status & DEPEVT_STATUS_SHORT && !chain)
3214 		return 1;
3215 
3216 	if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3217 	    (trb->ctrl & DWC3_TRB_CTRL_LST))
3218 		return 1;
3219 
3220 	return 0;
3221 }
3222 
3223 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3224 		struct dwc3_request *req, const struct dwc3_event_depevt *event,
3225 		int status)
3226 {
3227 	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3228 	struct scatterlist *sg = req->sg;
3229 	struct scatterlist *s;
3230 	unsigned int num_queued = req->num_queued_sgs;
3231 	unsigned int i;
3232 	int ret = 0;
3233 
3234 	for_each_sg(sg, s, num_queued, i) {
3235 		trb = &dep->trb_pool[dep->trb_dequeue];
3236 
3237 		req->sg = sg_next(s);
3238 		req->num_queued_sgs--;
3239 
3240 		ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3241 				trb, event, status, true);
3242 		if (ret)
3243 			break;
3244 	}
3245 
3246 	return ret;
3247 }
3248 
3249 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3250 		struct dwc3_request *req, const struct dwc3_event_depevt *event,
3251 		int status)
3252 {
3253 	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3254 
3255 	return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3256 			event, status, false);
3257 }
3258 
3259 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3260 {
3261 	return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3262 }
3263 
3264 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3265 		const struct dwc3_event_depevt *event,
3266 		struct dwc3_request *req, int status)
3267 {
3268 	int request_status;
3269 	int ret;
3270 
3271 	if (req->request.num_mapped_sgs)
3272 		ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3273 				status);
3274 	else
3275 		ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3276 				status);
3277 
3278 	req->request.actual = req->request.length - req->remaining;
3279 
3280 	if (!dwc3_gadget_ep_request_completed(req))
3281 		goto out;
3282 
3283 	if (req->needs_extra_trb) {
3284 		ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3285 				status);
3286 		req->needs_extra_trb = false;
3287 	}
3288 
3289 	/*
3290 	 * The event status only reflects the status of the TRB with IOC set.
3291 	 * For the requests that don't set interrupt on completion, the driver
3292 	 * needs to check and return the status of the completed TRBs associated
3293 	 * with the request. Use the status of the last TRB of the request.
3294 	 */
3295 	if (req->request.no_interrupt) {
3296 		struct dwc3_trb *trb;
3297 
3298 		trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
3299 		switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3300 		case DWC3_TRBSTS_MISSED_ISOC:
3301 			/* Isoc endpoint only */
3302 			request_status = -EXDEV;
3303 			break;
3304 		case DWC3_TRB_STS_XFER_IN_PROG:
3305 			/* Applicable when End Transfer with ForceRM=0 */
3306 		case DWC3_TRBSTS_SETUP_PENDING:
3307 			/* Control endpoint only */
3308 		case DWC3_TRBSTS_OK:
3309 		default:
3310 			request_status = 0;
3311 			break;
3312 		}
3313 	} else {
3314 		request_status = status;
3315 	}
3316 
3317 	dwc3_gadget_giveback(dep, req, request_status);
3318 
3319 out:
3320 	return ret;
3321 }
3322 
3323 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3324 		const struct dwc3_event_depevt *event, int status)
3325 {
3326 	struct dwc3_request	*req;
3327 
3328 	while (!list_empty(&dep->started_list)) {
3329 		int ret;
3330 
3331 		req = next_request(&dep->started_list);
3332 		ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3333 				req, status);
3334 		if (ret)
3335 			break;
3336 		/*
3337 		 * The endpoint is disabled, let the dwc3_remove_requests()
3338 		 * handle the cleanup.
3339 		 */
3340 		if (!dep->endpoint.desc)
3341 			break;
3342 	}
3343 }
3344 
3345 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3346 {
3347 	struct dwc3_request	*req;
3348 	struct dwc3		*dwc = dep->dwc;
3349 
3350 	if (!dep->endpoint.desc || !dwc->pullups_connected ||
3351 	    !dwc->connected)
3352 		return false;
3353 
3354 	if (!list_empty(&dep->pending_list))
3355 		return true;
3356 
3357 	/*
3358 	 * We only need to check the first entry of the started list. We can
3359 	 * assume the completed requests are removed from the started list.
3360 	 */
3361 	req = next_request(&dep->started_list);
3362 	if (!req)
3363 		return false;
3364 
3365 	return !dwc3_gadget_ep_request_completed(req);
3366 }
3367 
3368 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3369 		const struct dwc3_event_depevt *event)
3370 {
3371 	dep->frame_number = event->parameters;
3372 }
3373 
3374 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3375 		const struct dwc3_event_depevt *event, int status)
3376 {
3377 	struct dwc3		*dwc = dep->dwc;
3378 	bool			no_started_trb = true;
3379 
3380 	dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3381 
3382 	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3383 		goto out;
3384 
3385 	if (!dep->endpoint.desc)
3386 		return no_started_trb;
3387 
3388 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3389 		list_empty(&dep->started_list) &&
3390 		(list_empty(&dep->pending_list) || status == -EXDEV))
3391 		dwc3_stop_active_transfer(dep, true, true);
3392 	else if (dwc3_gadget_ep_should_continue(dep))
3393 		if (__dwc3_gadget_kick_transfer(dep) == 0)
3394 			no_started_trb = false;
3395 
3396 out:
3397 	/*
3398 	 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3399 	 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
3400 	 */
3401 	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3402 		u32		reg;
3403 		int		i;
3404 
3405 		for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3406 			dep = dwc->eps[i];
3407 
3408 			if (!(dep->flags & DWC3_EP_ENABLED))
3409 				continue;
3410 
3411 			if (!list_empty(&dep->started_list))
3412 				return no_started_trb;
3413 		}
3414 
3415 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3416 		reg |= dwc->u1u2;
3417 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3418 
3419 		dwc->u1u2 = 0;
3420 	}
3421 
3422 	return no_started_trb;
3423 }
3424 
3425 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3426 		const struct dwc3_event_depevt *event)
3427 {
3428 	int status = 0;
3429 
3430 	if (!dep->endpoint.desc)
3431 		return;
3432 
3433 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3434 		dwc3_gadget_endpoint_frame_from_event(dep, event);
3435 
3436 	if (event->status & DEPEVT_STATUS_BUSERR)
3437 		status = -ECONNRESET;
3438 
3439 	if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3440 		status = -EXDEV;
3441 
3442 	dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3443 }
3444 
3445 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3446 		const struct dwc3_event_depevt *event)
3447 {
3448 	int status = 0;
3449 
3450 	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3451 
3452 	if (event->status & DEPEVT_STATUS_BUSERR)
3453 		status = -ECONNRESET;
3454 
3455 	if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3456 		dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3457 }
3458 
3459 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3460 		const struct dwc3_event_depevt *event)
3461 {
3462 	dwc3_gadget_endpoint_frame_from_event(dep, event);
3463 
3464 	/*
3465 	 * The XferNotReady event is generated only once before the endpoint
3466 	 * starts. It will be generated again when END_TRANSFER command is
3467 	 * issued. For some controller versions, the XferNotReady event may be
3468 	 * generated while the END_TRANSFER command is still in process. Ignore
3469 	 * it and wait for the next XferNotReady event after the command is
3470 	 * completed.
3471 	 */
3472 	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3473 		return;
3474 
3475 	(void) __dwc3_gadget_start_isoc(dep);
3476 }
3477 
3478 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3479 		const struct dwc3_event_depevt *event)
3480 {
3481 	u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3482 
3483 	if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3484 		return;
3485 
3486 	/*
3487 	 * The END_TRANSFER command will cause the controller to generate a
3488 	 * NoStream Event, and it's not due to the host DP NoStream rejection.
3489 	 * Ignore the next NoStream event.
3490 	 */
3491 	if (dep->stream_capable)
3492 		dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3493 
3494 	dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3495 	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3496 	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3497 
3498 	if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3499 		struct dwc3 *dwc = dep->dwc;
3500 
3501 		dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3502 		if (dwc3_send_clear_stall_ep_cmd(dep)) {
3503 			struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3504 
3505 			dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3506 			if (dwc->delayed_status)
3507 				__dwc3_gadget_ep0_set_halt(ep0, 1);
3508 			return;
3509 		}
3510 
3511 		dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3512 		if (dwc->clear_stall_protocol == dep->number)
3513 			dwc3_ep0_send_delayed_status(dwc);
3514 	}
3515 
3516 	if ((dep->flags & DWC3_EP_DELAY_START) &&
3517 	    !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3518 		__dwc3_gadget_kick_transfer(dep);
3519 
3520 	dep->flags &= ~DWC3_EP_DELAY_START;
3521 }
3522 
3523 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3524 		const struct dwc3_event_depevt *event)
3525 {
3526 	struct dwc3 *dwc = dep->dwc;
3527 
3528 	if (event->status == DEPEVT_STREAMEVT_FOUND) {
3529 		dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3530 		goto out;
3531 	}
3532 
3533 	/* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3534 	switch (event->parameters) {
3535 	case DEPEVT_STREAM_PRIME:
3536 		/*
3537 		 * If the host can properly transition the endpoint state from
3538 		 * idle to prime after a NoStream rejection, there's no need to
3539 		 * force restarting the endpoint to reinitiate the stream. To
3540 		 * simplify the check, assume the host follows the USB spec if
3541 		 * it primed the endpoint more than once.
3542 		 */
3543 		if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3544 			if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3545 				dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3546 			else
3547 				dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3548 		}
3549 
3550 		break;
3551 	case DEPEVT_STREAM_NOSTREAM:
3552 		if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3553 		    !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3554 		    (!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3555 		     !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3556 			break;
3557 
3558 		/*
3559 		 * If the host rejects a stream due to no active stream, by the
3560 		 * USB and xHCI spec, the endpoint will be put back to idle
3561 		 * state. When the host is ready (buffer added/updated), it will
3562 		 * prime the endpoint to inform the usb device controller. This
3563 		 * triggers the device controller to issue ERDY to restart the
3564 		 * stream. However, some hosts don't follow this and keep the
3565 		 * endpoint in the idle state. No prime will come despite host
3566 		 * streams are updated, and the device controller will not be
3567 		 * triggered to generate ERDY to move the next stream data. To
3568 		 * workaround this and maintain compatibility with various
3569 		 * hosts, force to reinitate the stream until the host is ready
3570 		 * instead of waiting for the host to prime the endpoint.
3571 		 */
3572 		if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3573 			unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3574 
3575 			dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3576 		} else {
3577 			dep->flags |= DWC3_EP_DELAY_START;
3578 			dwc3_stop_active_transfer(dep, true, true);
3579 			return;
3580 		}
3581 		break;
3582 	}
3583 
3584 out:
3585 	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3586 }
3587 
3588 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3589 		const struct dwc3_event_depevt *event)
3590 {
3591 	struct dwc3_ep		*dep;
3592 	u8			epnum = event->endpoint_number;
3593 
3594 	dep = dwc->eps[epnum];
3595 
3596 	if (!(dep->flags & DWC3_EP_ENABLED)) {
3597 		if (!(dep->flags & DWC3_EP_TRANSFER_STARTED))
3598 			return;
3599 
3600 		/* Handle only EPCMDCMPLT when EP disabled */
3601 		if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
3602 			return;
3603 	}
3604 
3605 	if (epnum == 0 || epnum == 1) {
3606 		dwc3_ep0_interrupt(dwc, event);
3607 		return;
3608 	}
3609 
3610 	switch (event->endpoint_event) {
3611 	case DWC3_DEPEVT_XFERINPROGRESS:
3612 		dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3613 		break;
3614 	case DWC3_DEPEVT_XFERNOTREADY:
3615 		dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3616 		break;
3617 	case DWC3_DEPEVT_EPCMDCMPLT:
3618 		dwc3_gadget_endpoint_command_complete(dep, event);
3619 		break;
3620 	case DWC3_DEPEVT_XFERCOMPLETE:
3621 		dwc3_gadget_endpoint_transfer_complete(dep, event);
3622 		break;
3623 	case DWC3_DEPEVT_STREAMEVT:
3624 		dwc3_gadget_endpoint_stream_event(dep, event);
3625 		break;
3626 	case DWC3_DEPEVT_RXTXFIFOEVT:
3627 		break;
3628 	}
3629 }
3630 
3631 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3632 {
3633 	if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3634 		spin_unlock(&dwc->lock);
3635 		dwc->gadget_driver->disconnect(dwc->gadget);
3636 		spin_lock(&dwc->lock);
3637 	}
3638 }
3639 
3640 static void dwc3_suspend_gadget(struct dwc3 *dwc)
3641 {
3642 	if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3643 		spin_unlock(&dwc->lock);
3644 		dwc->gadget_driver->suspend(dwc->gadget);
3645 		spin_lock(&dwc->lock);
3646 	}
3647 }
3648 
3649 static void dwc3_resume_gadget(struct dwc3 *dwc)
3650 {
3651 	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3652 		spin_unlock(&dwc->lock);
3653 		dwc->gadget_driver->resume(dwc->gadget);
3654 		spin_lock(&dwc->lock);
3655 	}
3656 }
3657 
3658 static void dwc3_reset_gadget(struct dwc3 *dwc)
3659 {
3660 	if (!dwc->gadget_driver)
3661 		return;
3662 
3663 	if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3664 		spin_unlock(&dwc->lock);
3665 		usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3666 		spin_lock(&dwc->lock);
3667 	}
3668 }
3669 
3670 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3671 	bool interrupt)
3672 {
3673 	struct dwc3 *dwc = dep->dwc;
3674 
3675 	/*
3676 	 * Only issue End Transfer command to the control endpoint of a started
3677 	 * Data Phase. Typically we should only do so in error cases such as
3678 	 * invalid/unexpected direction as described in the control transfer
3679 	 * flow of the programming guide.
3680 	 */
3681 	if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3682 		return;
3683 
3684 	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3685 	    (dep->flags & DWC3_EP_DELAY_STOP) ||
3686 	    (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3687 		return;
3688 
3689 	/*
3690 	 * If a Setup packet is received but yet to DMA out, the controller will
3691 	 * not process the End Transfer command of any endpoint. Polling of its
3692 	 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
3693 	 * timeout. Delay issuing the End Transfer command until the Setup TRB is
3694 	 * prepared.
3695 	 */
3696 	if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
3697 		dep->flags |= DWC3_EP_DELAY_STOP;
3698 		return;
3699 	}
3700 
3701 	/*
3702 	 * NOTICE: We are violating what the Databook says about the
3703 	 * EndTransfer command. Ideally we would _always_ wait for the
3704 	 * EndTransfer Command Completion IRQ, but that's causing too
3705 	 * much trouble synchronizing between us and gadget driver.
3706 	 *
3707 	 * We have discussed this with the IP Provider and it was
3708 	 * suggested to giveback all requests here.
3709 	 *
3710 	 * Note also that a similar handling was tested by Synopsys
3711 	 * (thanks a lot Paul) and nothing bad has come out of it.
3712 	 * In short, what we're doing is issuing EndTransfer with
3713 	 * CMDIOC bit set and delay kicking transfer until the
3714 	 * EndTransfer command had completed.
3715 	 *
3716 	 * As of IP version 3.10a of the DWC_usb3 IP, the controller
3717 	 * supports a mode to work around the above limitation. The
3718 	 * software can poll the CMDACT bit in the DEPCMD register
3719 	 * after issuing a EndTransfer command. This mode is enabled
3720 	 * by writing GUCTL2[14]. This polling is already done in the
3721 	 * dwc3_send_gadget_ep_cmd() function so if the mode is
3722 	 * enabled, the EndTransfer command will have completed upon
3723 	 * returning from this function.
3724 	 *
3725 	 * This mode is NOT available on the DWC_usb31 IP.
3726 	 */
3727 
3728 	__dwc3_stop_active_transfer(dep, force, interrupt);
3729 }
3730 
3731 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3732 {
3733 	u32 epnum;
3734 
3735 	for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3736 		struct dwc3_ep *dep;
3737 		int ret;
3738 
3739 		dep = dwc->eps[epnum];
3740 		if (!dep)
3741 			continue;
3742 
3743 		if (!(dep->flags & DWC3_EP_STALL))
3744 			continue;
3745 
3746 		dep->flags &= ~DWC3_EP_STALL;
3747 
3748 		ret = dwc3_send_clear_stall_ep_cmd(dep);
3749 		WARN_ON_ONCE(ret);
3750 	}
3751 }
3752 
3753 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3754 {
3755 	int			reg;
3756 
3757 	dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3758 
3759 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3760 	reg &= ~DWC3_DCTL_INITU1ENA;
3761 	reg &= ~DWC3_DCTL_INITU2ENA;
3762 	dwc3_gadget_dctl_write_safe(dwc, reg);
3763 
3764 	dwc3_disconnect_gadget(dwc);
3765 
3766 	dwc->gadget->speed = USB_SPEED_UNKNOWN;
3767 	dwc->setup_packet_pending = false;
3768 	usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3769 
3770 	dwc->connected = false;
3771 }
3772 
3773 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3774 {
3775 	u32			reg;
3776 
3777 	/*
3778 	 * Ideally, dwc3_reset_gadget() would trigger the function
3779 	 * drivers to stop any active transfers through ep disable.
3780 	 * However, for functions which defer ep disable, such as mass
3781 	 * storage, we will need to rely on the call to stop active
3782 	 * transfers here, and avoid allowing of request queuing.
3783 	 */
3784 	dwc->connected = false;
3785 
3786 	/*
3787 	 * WORKAROUND: DWC3 revisions <1.88a have an issue which
3788 	 * would cause a missing Disconnect Event if there's a
3789 	 * pending Setup Packet in the FIFO.
3790 	 *
3791 	 * There's no suggested workaround on the official Bug
3792 	 * report, which states that "unless the driver/application
3793 	 * is doing any special handling of a disconnect event,
3794 	 * there is no functional issue".
3795 	 *
3796 	 * Unfortunately, it turns out that we _do_ some special
3797 	 * handling of a disconnect event, namely complete all
3798 	 * pending transfers, notify gadget driver of the
3799 	 * disconnection, and so on.
3800 	 *
3801 	 * Our suggested workaround is to follow the Disconnect
3802 	 * Event steps here, instead, based on a setup_packet_pending
3803 	 * flag. Such flag gets set whenever we have a SETUP_PENDING
3804 	 * status for EP0 TRBs and gets cleared on XferComplete for the
3805 	 * same endpoint.
3806 	 *
3807 	 * Refers to:
3808 	 *
3809 	 * STAR#9000466709: RTL: Device : Disconnect event not
3810 	 * generated if setup packet pending in FIFO
3811 	 */
3812 	if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
3813 		if (dwc->setup_packet_pending)
3814 			dwc3_gadget_disconnect_interrupt(dwc);
3815 	}
3816 
3817 	dwc3_reset_gadget(dwc);
3818 
3819 	/*
3820 	 * From SNPS databook section 8.1.2, the EP0 should be in setup
3821 	 * phase. So ensure that EP0 is in setup phase by issuing a stall
3822 	 * and restart if EP0 is not in setup phase.
3823 	 */
3824 	if (dwc->ep0state != EP0_SETUP_PHASE) {
3825 		unsigned int	dir;
3826 
3827 		dir = !!dwc->ep0_expect_in;
3828 		if (dwc->ep0state == EP0_DATA_PHASE)
3829 			dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
3830 		else
3831 			dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
3832 
3833 		dwc->eps[0]->trb_enqueue = 0;
3834 		dwc->eps[1]->trb_enqueue = 0;
3835 
3836 		dwc3_ep0_stall_and_restart(dwc);
3837 	}
3838 
3839 	/*
3840 	 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
3841 	 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
3842 	 * needs to ensure that it sends "a DEPENDXFER command for any active
3843 	 * transfers."
3844 	 */
3845 	dwc3_stop_active_transfers(dwc);
3846 	dwc->connected = true;
3847 
3848 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3849 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
3850 	dwc3_gadget_dctl_write_safe(dwc, reg);
3851 	dwc->test_mode = false;
3852 	dwc3_clear_stall_all_ep(dwc);
3853 
3854 	/* Reset device address to zero */
3855 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3856 	reg &= ~(DWC3_DCFG_DEVADDR_MASK);
3857 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3858 }
3859 
3860 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
3861 {
3862 	struct dwc3_ep		*dep;
3863 	int			ret;
3864 	u32			reg;
3865 	u8			lanes = 1;
3866 	u8			speed;
3867 
3868 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
3869 	speed = reg & DWC3_DSTS_CONNECTSPD;
3870 	dwc->speed = speed;
3871 
3872 	if (DWC3_IP_IS(DWC32))
3873 		lanes = DWC3_DSTS_CONNLANES(reg) + 1;
3874 
3875 	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
3876 
3877 	/*
3878 	 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
3879 	 * each time on Connect Done.
3880 	 *
3881 	 * Currently we always use the reset value. If any platform
3882 	 * wants to set this to a different value, we need to add a
3883 	 * setting and update GCTL.RAMCLKSEL here.
3884 	 */
3885 
3886 	switch (speed) {
3887 	case DWC3_DSTS_SUPERSPEED_PLUS:
3888 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3889 		dwc->gadget->ep0->maxpacket = 512;
3890 		dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
3891 
3892 		if (lanes > 1)
3893 			dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
3894 		else
3895 			dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
3896 		break;
3897 	case DWC3_DSTS_SUPERSPEED:
3898 		/*
3899 		 * WORKAROUND: DWC3 revisions <1.90a have an issue which
3900 		 * would cause a missing USB3 Reset event.
3901 		 *
3902 		 * In such situations, we should force a USB3 Reset
3903 		 * event by calling our dwc3_gadget_reset_interrupt()
3904 		 * routine.
3905 		 *
3906 		 * Refers to:
3907 		 *
3908 		 * STAR#9000483510: RTL: SS : USB3 reset event may
3909 		 * not be generated always when the link enters poll
3910 		 */
3911 		if (DWC3_VER_IS_PRIOR(DWC3, 190A))
3912 			dwc3_gadget_reset_interrupt(dwc);
3913 
3914 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
3915 		dwc->gadget->ep0->maxpacket = 512;
3916 		dwc->gadget->speed = USB_SPEED_SUPER;
3917 
3918 		if (lanes > 1) {
3919 			dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
3920 			dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
3921 		}
3922 		break;
3923 	case DWC3_DSTS_HIGHSPEED:
3924 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3925 		dwc->gadget->ep0->maxpacket = 64;
3926 		dwc->gadget->speed = USB_SPEED_HIGH;
3927 		break;
3928 	case DWC3_DSTS_FULLSPEED:
3929 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
3930 		dwc->gadget->ep0->maxpacket = 64;
3931 		dwc->gadget->speed = USB_SPEED_FULL;
3932 		break;
3933 	}
3934 
3935 	dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
3936 
3937 	/* Enable USB2 LPM Capability */
3938 
3939 	if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
3940 	    !dwc->usb2_gadget_lpm_disable &&
3941 	    (speed != DWC3_DSTS_SUPERSPEED) &&
3942 	    (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
3943 		reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3944 		reg |= DWC3_DCFG_LPM_CAP;
3945 		dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3946 
3947 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3948 		reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
3949 
3950 		reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
3951 					    (dwc->is_utmi_l1_suspend << 4));
3952 
3953 		/*
3954 		 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
3955 		 * DCFG.LPMCap is set, core responses with an ACK and the
3956 		 * BESL value in the LPM token is less than or equal to LPM
3957 		 * NYET threshold.
3958 		 */
3959 		WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
3960 				"LPM Erratum not available on dwc3 revisions < 2.40a\n");
3961 
3962 		if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
3963 			reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
3964 
3965 		dwc3_gadget_dctl_write_safe(dwc, reg);
3966 	} else {
3967 		if (dwc->usb2_gadget_lpm_disable) {
3968 			reg = dwc3_readl(dwc->regs, DWC3_DCFG);
3969 			reg &= ~DWC3_DCFG_LPM_CAP;
3970 			dwc3_writel(dwc->regs, DWC3_DCFG, reg);
3971 		}
3972 
3973 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3974 		reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
3975 		dwc3_gadget_dctl_write_safe(dwc, reg);
3976 	}
3977 
3978 	dep = dwc->eps[0];
3979 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3980 	if (ret) {
3981 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3982 		return;
3983 	}
3984 
3985 	dep = dwc->eps[1];
3986 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
3987 	if (ret) {
3988 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
3989 		return;
3990 	}
3991 
3992 	/*
3993 	 * Configure PHY via GUSB3PIPECTLn if required.
3994 	 *
3995 	 * Update GTXFIFOSIZn
3996 	 *
3997 	 * In both cases reset values should be sufficient.
3998 	 */
3999 }
4000 
4001 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
4002 {
4003 	/*
4004 	 * TODO take core out of low power mode when that's
4005 	 * implemented.
4006 	 */
4007 
4008 	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
4009 		spin_unlock(&dwc->lock);
4010 		dwc->gadget_driver->resume(dwc->gadget);
4011 		spin_lock(&dwc->lock);
4012 	}
4013 }
4014 
4015 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
4016 		unsigned int evtinfo)
4017 {
4018 	enum dwc3_link_state	next = evtinfo & DWC3_LINK_STATE_MASK;
4019 	unsigned int		pwropt;
4020 
4021 	/*
4022 	 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
4023 	 * Hibernation mode enabled which would show up when device detects
4024 	 * host-initiated U3 exit.
4025 	 *
4026 	 * In that case, device will generate a Link State Change Interrupt
4027 	 * from U3 to RESUME which is only necessary if Hibernation is
4028 	 * configured in.
4029 	 *
4030 	 * There are no functional changes due to such spurious event and we
4031 	 * just need to ignore it.
4032 	 *
4033 	 * Refers to:
4034 	 *
4035 	 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4036 	 * operational mode
4037 	 */
4038 	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4039 	if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4040 			(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4041 		if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4042 				(next == DWC3_LINK_STATE_RESUME)) {
4043 			return;
4044 		}
4045 	}
4046 
4047 	/*
4048 	 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4049 	 * on the link partner, the USB session might do multiple entry/exit
4050 	 * of low power states before a transfer takes place.
4051 	 *
4052 	 * Due to this problem, we might experience lower throughput. The
4053 	 * suggested workaround is to disable DCTL[12:9] bits if we're
4054 	 * transitioning from U1/U2 to U0 and enable those bits again
4055 	 * after a transfer completes and there are no pending transfers
4056 	 * on any of the enabled endpoints.
4057 	 *
4058 	 * This is the first half of that workaround.
4059 	 *
4060 	 * Refers to:
4061 	 *
4062 	 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4063 	 * core send LGO_Ux entering U0
4064 	 */
4065 	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4066 		if (next == DWC3_LINK_STATE_U0) {
4067 			u32	u1u2;
4068 			u32	reg;
4069 
4070 			switch (dwc->link_state) {
4071 			case DWC3_LINK_STATE_U1:
4072 			case DWC3_LINK_STATE_U2:
4073 				reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4074 				u1u2 = reg & (DWC3_DCTL_INITU2ENA
4075 						| DWC3_DCTL_ACCEPTU2ENA
4076 						| DWC3_DCTL_INITU1ENA
4077 						| DWC3_DCTL_ACCEPTU1ENA);
4078 
4079 				if (!dwc->u1u2)
4080 					dwc->u1u2 = reg & u1u2;
4081 
4082 				reg &= ~u1u2;
4083 
4084 				dwc3_gadget_dctl_write_safe(dwc, reg);
4085 				break;
4086 			default:
4087 				/* do nothing */
4088 				break;
4089 			}
4090 		}
4091 	}
4092 
4093 	switch (next) {
4094 	case DWC3_LINK_STATE_U1:
4095 		if (dwc->speed == USB_SPEED_SUPER)
4096 			dwc3_suspend_gadget(dwc);
4097 		break;
4098 	case DWC3_LINK_STATE_U2:
4099 	case DWC3_LINK_STATE_U3:
4100 		dwc3_suspend_gadget(dwc);
4101 		break;
4102 	case DWC3_LINK_STATE_RESUME:
4103 		dwc3_resume_gadget(dwc);
4104 		break;
4105 	default:
4106 		/* do nothing */
4107 		break;
4108 	}
4109 
4110 	dwc->link_state = next;
4111 }
4112 
4113 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4114 					  unsigned int evtinfo)
4115 {
4116 	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4117 
4118 	if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
4119 		dwc3_suspend_gadget(dwc);
4120 
4121 	dwc->link_state = next;
4122 }
4123 
4124 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
4125 		unsigned int evtinfo)
4126 {
4127 	unsigned int is_ss = evtinfo & BIT(4);
4128 
4129 	/*
4130 	 * WORKAROUND: DWC3 revison 2.20a with hibernation support
4131 	 * have a known issue which can cause USB CV TD.9.23 to fail
4132 	 * randomly.
4133 	 *
4134 	 * Because of this issue, core could generate bogus hibernation
4135 	 * events which SW needs to ignore.
4136 	 *
4137 	 * Refers to:
4138 	 *
4139 	 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
4140 	 * Device Fallback from SuperSpeed
4141 	 */
4142 	if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
4143 		return;
4144 
4145 	/* enter hibernation here */
4146 }
4147 
4148 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4149 		const struct dwc3_event_devt *event)
4150 {
4151 	switch (event->type) {
4152 	case DWC3_DEVICE_EVENT_DISCONNECT:
4153 		dwc3_gadget_disconnect_interrupt(dwc);
4154 		break;
4155 	case DWC3_DEVICE_EVENT_RESET:
4156 		dwc3_gadget_reset_interrupt(dwc);
4157 		break;
4158 	case DWC3_DEVICE_EVENT_CONNECT_DONE:
4159 		dwc3_gadget_conndone_interrupt(dwc);
4160 		break;
4161 	case DWC3_DEVICE_EVENT_WAKEUP:
4162 		dwc3_gadget_wakeup_interrupt(dwc);
4163 		break;
4164 	case DWC3_DEVICE_EVENT_HIBER_REQ:
4165 		if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
4166 					"unexpected hibernation event\n"))
4167 			break;
4168 
4169 		dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
4170 		break;
4171 	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4172 		dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
4173 		break;
4174 	case DWC3_DEVICE_EVENT_SUSPEND:
4175 		/* It changed to be suspend event for version 2.30a and above */
4176 		if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) {
4177 			/*
4178 			 * Ignore suspend event until the gadget enters into
4179 			 * USB_STATE_CONFIGURED state.
4180 			 */
4181 			if (dwc->gadget->state >= USB_STATE_CONFIGURED)
4182 				dwc3_gadget_suspend_interrupt(dwc,
4183 						event->event_info);
4184 		}
4185 		break;
4186 	case DWC3_DEVICE_EVENT_SOF:
4187 	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4188 	case DWC3_DEVICE_EVENT_CMD_CMPL:
4189 	case DWC3_DEVICE_EVENT_OVERFLOW:
4190 		break;
4191 	default:
4192 		dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4193 	}
4194 }
4195 
4196 static void dwc3_process_event_entry(struct dwc3 *dwc,
4197 		const union dwc3_event *event)
4198 {
4199 	trace_dwc3_event(event->raw, dwc);
4200 
4201 	if (!event->type.is_devspec)
4202 		dwc3_endpoint_interrupt(dwc, &event->depevt);
4203 	else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4204 		dwc3_gadget_interrupt(dwc, &event->devt);
4205 	else
4206 		dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4207 }
4208 
4209 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4210 {
4211 	struct dwc3 *dwc = evt->dwc;
4212 	irqreturn_t ret = IRQ_NONE;
4213 	int left;
4214 
4215 	left = evt->count;
4216 
4217 	if (!(evt->flags & DWC3_EVENT_PENDING))
4218 		return IRQ_NONE;
4219 
4220 	while (left > 0) {
4221 		union dwc3_event event;
4222 
4223 		event.raw = *(u32 *) (evt->cache + evt->lpos);
4224 
4225 		dwc3_process_event_entry(dwc, &event);
4226 
4227 		/*
4228 		 * FIXME we wrap around correctly to the next entry as
4229 		 * almost all entries are 4 bytes in size. There is one
4230 		 * entry which has 12 bytes which is a regular entry
4231 		 * followed by 8 bytes data. ATM I don't know how
4232 		 * things are organized if we get next to the a
4233 		 * boundary so I worry about that once we try to handle
4234 		 * that.
4235 		 */
4236 		evt->lpos = (evt->lpos + 4) % evt->length;
4237 		left -= 4;
4238 	}
4239 
4240 	evt->count = 0;
4241 	ret = IRQ_HANDLED;
4242 
4243 	/* Unmask interrupt */
4244 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4245 		    DWC3_GEVNTSIZ_SIZE(evt->length));
4246 
4247 	if (dwc->imod_interval) {
4248 		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4249 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
4250 	}
4251 
4252 	/* Keep the clearing of DWC3_EVENT_PENDING at the end */
4253 	evt->flags &= ~DWC3_EVENT_PENDING;
4254 
4255 	return ret;
4256 }
4257 
4258 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4259 {
4260 	struct dwc3_event_buffer *evt = _evt;
4261 	struct dwc3 *dwc = evt->dwc;
4262 	unsigned long flags;
4263 	irqreturn_t ret = IRQ_NONE;
4264 
4265 	local_bh_disable();
4266 	spin_lock_irqsave(&dwc->lock, flags);
4267 	ret = dwc3_process_event_buf(evt);
4268 	spin_unlock_irqrestore(&dwc->lock, flags);
4269 	local_bh_enable();
4270 
4271 	return ret;
4272 }
4273 
4274 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4275 {
4276 	struct dwc3 *dwc = evt->dwc;
4277 	u32 amount;
4278 	u32 count;
4279 
4280 	if (pm_runtime_suspended(dwc->dev)) {
4281 		pm_runtime_get(dwc->dev);
4282 		disable_irq_nosync(dwc->irq_gadget);
4283 		dwc->pending_events = true;
4284 		return IRQ_HANDLED;
4285 	}
4286 
4287 	/*
4288 	 * With PCIe legacy interrupt, test shows that top-half irq handler can
4289 	 * be called again after HW interrupt deassertion. Check if bottom-half
4290 	 * irq event handler completes before caching new event to prevent
4291 	 * losing events.
4292 	 */
4293 	if (evt->flags & DWC3_EVENT_PENDING)
4294 		return IRQ_HANDLED;
4295 
4296 	count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
4297 	count &= DWC3_GEVNTCOUNT_MASK;
4298 	if (!count)
4299 		return IRQ_NONE;
4300 
4301 	evt->count = count;
4302 	evt->flags |= DWC3_EVENT_PENDING;
4303 
4304 	/* Mask interrupt */
4305 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4306 		    DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4307 
4308 	amount = min(count, evt->length - evt->lpos);
4309 	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4310 
4311 	if (amount < count)
4312 		memcpy(evt->cache, evt->buf, count - amount);
4313 
4314 	dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
4315 
4316 	return IRQ_WAKE_THREAD;
4317 }
4318 
4319 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4320 {
4321 	struct dwc3_event_buffer	*evt = _evt;
4322 
4323 	return dwc3_check_event_buf(evt);
4324 }
4325 
4326 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4327 {
4328 	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4329 	int irq;
4330 
4331 	irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
4332 	if (irq > 0)
4333 		goto out;
4334 
4335 	if (irq == -EPROBE_DEFER)
4336 		goto out;
4337 
4338 	irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
4339 	if (irq > 0)
4340 		goto out;
4341 
4342 	if (irq == -EPROBE_DEFER)
4343 		goto out;
4344 
4345 	irq = platform_get_irq(dwc3_pdev, 0);
4346 	if (irq > 0)
4347 		goto out;
4348 
4349 	if (!irq)
4350 		irq = -EINVAL;
4351 
4352 out:
4353 	return irq;
4354 }
4355 
4356 static void dwc_gadget_release(struct device *dev)
4357 {
4358 	struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4359 
4360 	kfree(gadget);
4361 }
4362 
4363 /**
4364  * dwc3_gadget_init - initializes gadget related registers
4365  * @dwc: pointer to our controller context structure
4366  *
4367  * Returns 0 on success otherwise negative errno.
4368  */
4369 int dwc3_gadget_init(struct dwc3 *dwc)
4370 {
4371 	int ret;
4372 	int irq;
4373 	struct device *dev;
4374 
4375 	irq = dwc3_gadget_get_irq(dwc);
4376 	if (irq < 0) {
4377 		ret = irq;
4378 		goto err0;
4379 	}
4380 
4381 	dwc->irq_gadget = irq;
4382 
4383 	dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
4384 					  sizeof(*dwc->ep0_trb) * 2,
4385 					  &dwc->ep0_trb_addr, GFP_KERNEL);
4386 	if (!dwc->ep0_trb) {
4387 		dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4388 		ret = -ENOMEM;
4389 		goto err0;
4390 	}
4391 
4392 	dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4393 	if (!dwc->setup_buf) {
4394 		ret = -ENOMEM;
4395 		goto err1;
4396 	}
4397 
4398 	dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
4399 			&dwc->bounce_addr, GFP_KERNEL);
4400 	if (!dwc->bounce) {
4401 		ret = -ENOMEM;
4402 		goto err2;
4403 	}
4404 
4405 	init_completion(&dwc->ep0_in_setup);
4406 	dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4407 	if (!dwc->gadget) {
4408 		ret = -ENOMEM;
4409 		goto err3;
4410 	}
4411 
4412 
4413 	usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
4414 	dev				= &dwc->gadget->dev;
4415 	dev->platform_data		= dwc;
4416 	dwc->gadget->ops		= &dwc3_gadget_ops;
4417 	dwc->gadget->speed		= USB_SPEED_UNKNOWN;
4418 	dwc->gadget->ssp_rate		= USB_SSP_GEN_UNKNOWN;
4419 	dwc->gadget->sg_supported	= true;
4420 	dwc->gadget->name		= "dwc3-gadget";
4421 	dwc->gadget->lpm_capable	= !dwc->usb2_gadget_lpm_disable;
4422 
4423 	/*
4424 	 * FIXME We might be setting max_speed to <SUPER, however versions
4425 	 * <2.20a of dwc3 have an issue with metastability (documented
4426 	 * elsewhere in this driver) which tells us we can't set max speed to
4427 	 * anything lower than SUPER.
4428 	 *
4429 	 * Because gadget.max_speed is only used by composite.c and function
4430 	 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
4431 	 * to happen so we avoid sending SuperSpeed Capability descriptor
4432 	 * together with our BOS descriptor as that could confuse host into
4433 	 * thinking we can handle super speed.
4434 	 *
4435 	 * Note that, in fact, we won't even support GetBOS requests when speed
4436 	 * is less than super speed because we don't have means, yet, to tell
4437 	 * composite.c that we are USB 2.0 + LPM ECN.
4438 	 */
4439 	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4440 	    !dwc->dis_metastability_quirk)
4441 		dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4442 				dwc->revision);
4443 
4444 	dwc->gadget->max_speed		= dwc->maximum_speed;
4445 	dwc->gadget->max_ssp_rate	= dwc->max_ssp_rate;
4446 
4447 	/*
4448 	 * REVISIT: Here we should clear all pending IRQs to be
4449 	 * sure we're starting from a well known location.
4450 	 */
4451 
4452 	ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
4453 	if (ret)
4454 		goto err4;
4455 
4456 	ret = usb_add_gadget(dwc->gadget);
4457 	if (ret) {
4458 		dev_err(dwc->dev, "failed to add gadget\n");
4459 		goto err5;
4460 	}
4461 
4462 	if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4463 		dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
4464 	else
4465 		dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
4466 
4467 	return 0;
4468 
4469 err5:
4470 	dwc3_gadget_free_endpoints(dwc);
4471 err4:
4472 	usb_put_gadget(dwc->gadget);
4473 	dwc->gadget = NULL;
4474 err3:
4475 	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4476 			dwc->bounce_addr);
4477 
4478 err2:
4479 	kfree(dwc->setup_buf);
4480 
4481 err1:
4482 	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4483 			dwc->ep0_trb, dwc->ep0_trb_addr);
4484 
4485 err0:
4486 	return ret;
4487 }
4488 
4489 /* -------------------------------------------------------------------------- */
4490 
4491 void dwc3_gadget_exit(struct dwc3 *dwc)
4492 {
4493 	if (!dwc->gadget)
4494 		return;
4495 
4496 	usb_del_gadget(dwc->gadget);
4497 	dwc3_gadget_free_endpoints(dwc);
4498 	usb_put_gadget(dwc->gadget);
4499 	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4500 			  dwc->bounce_addr);
4501 	kfree(dwc->setup_buf);
4502 	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4503 			  dwc->ep0_trb, dwc->ep0_trb_addr);
4504 }
4505 
4506 int dwc3_gadget_suspend(struct dwc3 *dwc)
4507 {
4508 	if (!dwc->gadget_driver)
4509 		return 0;
4510 
4511 	dwc3_gadget_run_stop(dwc, false, false);
4512 	dwc3_disconnect_gadget(dwc);
4513 	__dwc3_gadget_stop(dwc);
4514 
4515 	return 0;
4516 }
4517 
4518 int dwc3_gadget_resume(struct dwc3 *dwc)
4519 {
4520 	int			ret;
4521 
4522 	if (!dwc->gadget_driver || !dwc->softconnect)
4523 		return 0;
4524 
4525 	ret = __dwc3_gadget_start(dwc);
4526 	if (ret < 0)
4527 		goto err0;
4528 
4529 	ret = dwc3_gadget_run_stop(dwc, true, false);
4530 	if (ret < 0)
4531 		goto err1;
4532 
4533 	return 0;
4534 
4535 err1:
4536 	__dwc3_gadget_stop(dwc);
4537 
4538 err0:
4539 	return ret;
4540 }
4541 
4542 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
4543 {
4544 	if (dwc->pending_events) {
4545 		dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
4546 		dwc->pending_events = false;
4547 		enable_irq(dwc->irq_gadget);
4548 	}
4549 }
4550