xref: /openbmc/linux/drivers/usb/dwc3/gadget.c (revision c3d3501c)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4  *
5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6  *
7  * Authors: Felipe Balbi <balbi@ti.com>,
8  *	    Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9  */
10 
11 #include <linux/kernel.h>
12 #include <linux/delay.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <linux/platform_device.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/list.h>
20 #include <linux/dma-mapping.h>
21 
22 #include <linux/usb/ch9.h>
23 #include <linux/usb/gadget.h>
24 
25 #include "debug.h"
26 #include "core.h"
27 #include "gadget.h"
28 #include "io.h"
29 
30 #define DWC3_ALIGN_FRAME(d, n)	(((d)->frame_number + ((d)->interval * (n))) \
31 					& ~((d)->interval - 1))
32 
33 /**
34  * dwc3_gadget_set_test_mode - enables usb2 test modes
35  * @dwc: pointer to our context structure
36  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
37  *
38  * Caller should take care of locking. This function will return 0 on
39  * success or -EINVAL if wrong Test Selector is passed.
40  */
dwc3_gadget_set_test_mode(struct dwc3 * dwc,int mode)41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42 {
43 	u32		reg;
44 
45 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47 
48 	switch (mode) {
49 	case USB_TEST_J:
50 	case USB_TEST_K:
51 	case USB_TEST_SE0_NAK:
52 	case USB_TEST_PACKET:
53 	case USB_TEST_FORCE_ENABLE:
54 		reg |= mode << 1;
55 		break;
56 	default:
57 		return -EINVAL;
58 	}
59 
60 	dwc3_gadget_dctl_write_safe(dwc, reg);
61 
62 	return 0;
63 }
64 
65 /**
66  * dwc3_gadget_get_link_state - gets current state of usb link
67  * @dwc: pointer to our context structure
68  *
69  * Caller should take care of locking. This function will
70  * return the link state on success (>= 0) or -ETIMEDOUT.
71  */
dwc3_gadget_get_link_state(struct dwc3 * dwc)72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73 {
74 	u32		reg;
75 
76 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77 
78 	return DWC3_DSTS_USBLNKST(reg);
79 }
80 
81 /**
82  * dwc3_gadget_set_link_state - sets usb link to a particular state
83  * @dwc: pointer to our context structure
84  * @state: the state to put link into
85  *
86  * Caller should take care of locking. This function will
87  * return 0 on success or -ETIMEDOUT.
88  */
dwc3_gadget_set_link_state(struct dwc3 * dwc,enum dwc3_link_state state)89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90 {
91 	int		retries = 10000;
92 	u32		reg;
93 
94 	/*
95 	 * Wait until device controller is ready. Only applies to 1.94a and
96 	 * later RTL.
97 	 */
98 	if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99 		while (--retries) {
100 			reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 			if (reg & DWC3_DSTS_DCNRD)
102 				udelay(5);
103 			else
104 				break;
105 		}
106 
107 		if (retries <= 0)
108 			return -ETIMEDOUT;
109 	}
110 
111 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113 
114 	/* set no action before sending new link state change */
115 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116 
117 	/* set requested state */
118 	reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
120 
121 	/*
122 	 * The following code is racy when called from dwc3_gadget_wakeup,
123 	 * and is not needed, at least on newer versions
124 	 */
125 	if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126 		return 0;
127 
128 	/* wait for a change in DSTS */
129 	retries = 10000;
130 	while (--retries) {
131 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
132 
133 		if (DWC3_DSTS_USBLNKST(reg) == state)
134 			return 0;
135 
136 		udelay(5);
137 	}
138 
139 	return -ETIMEDOUT;
140 }
141 
dwc3_ep0_reset_state(struct dwc3 * dwc)142 static void dwc3_ep0_reset_state(struct dwc3 *dwc)
143 {
144 	unsigned int	dir;
145 
146 	if (dwc->ep0state != EP0_SETUP_PHASE) {
147 		dir = !!dwc->ep0_expect_in;
148 		if (dwc->ep0state == EP0_DATA_PHASE)
149 			dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
150 		else
151 			dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
152 
153 		dwc->eps[0]->trb_enqueue = 0;
154 		dwc->eps[1]->trb_enqueue = 0;
155 
156 		dwc3_ep0_stall_and_restart(dwc);
157 	}
158 }
159 
160 /**
161  * dwc3_ep_inc_trb - increment a trb index.
162  * @index: Pointer to the TRB index to increment.
163  *
164  * The index should never point to the link TRB. After incrementing,
165  * if it is point to the link TRB, wrap around to the beginning. The
166  * link TRB is always at the last TRB entry.
167  */
dwc3_ep_inc_trb(u8 * index)168 static void dwc3_ep_inc_trb(u8 *index)
169 {
170 	(*index)++;
171 	if (*index == (DWC3_TRB_NUM - 1))
172 		*index = 0;
173 }
174 
175 /**
176  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
177  * @dep: The endpoint whose enqueue pointer we're incrementing
178  */
dwc3_ep_inc_enq(struct dwc3_ep * dep)179 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
180 {
181 	dwc3_ep_inc_trb(&dep->trb_enqueue);
182 }
183 
184 /**
185  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
186  * @dep: The endpoint whose enqueue pointer we're incrementing
187  */
dwc3_ep_inc_deq(struct dwc3_ep * dep)188 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
189 {
190 	dwc3_ep_inc_trb(&dep->trb_dequeue);
191 }
192 
dwc3_gadget_del_and_unmap_request(struct dwc3_ep * dep,struct dwc3_request * req,int status)193 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
194 		struct dwc3_request *req, int status)
195 {
196 	struct dwc3			*dwc = dep->dwc;
197 
198 	list_del(&req->list);
199 	req->remaining = 0;
200 	req->needs_extra_trb = false;
201 	req->num_trbs = 0;
202 
203 	if (req->request.status == -EINPROGRESS)
204 		req->request.status = status;
205 
206 	if (req->trb)
207 		usb_gadget_unmap_request_by_dev(dwc->sysdev,
208 				&req->request, req->direction);
209 
210 	req->trb = NULL;
211 	trace_dwc3_gadget_giveback(req);
212 
213 	if (dep->number > 1)
214 		pm_runtime_put(dwc->dev);
215 }
216 
217 /**
218  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
219  * @dep: The endpoint to whom the request belongs to
220  * @req: The request we're giving back
221  * @status: completion code for the request
222  *
223  * Must be called with controller's lock held and interrupts disabled. This
224  * function will unmap @req and call its ->complete() callback to notify upper
225  * layers that it has completed.
226  */
dwc3_gadget_giveback(struct dwc3_ep * dep,struct dwc3_request * req,int status)227 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
228 		int status)
229 {
230 	struct dwc3			*dwc = dep->dwc;
231 
232 	dwc3_gadget_del_and_unmap_request(dep, req, status);
233 	req->status = DWC3_REQUEST_STATUS_COMPLETED;
234 
235 	spin_unlock(&dwc->lock);
236 	usb_gadget_giveback_request(&dep->endpoint, &req->request);
237 	spin_lock(&dwc->lock);
238 }
239 
240 /**
241  * dwc3_send_gadget_generic_command - issue a generic command for the controller
242  * @dwc: pointer to the controller context
243  * @cmd: the command to be issued
244  * @param: command parameter
245  *
246  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
247  * and wait for its completion.
248  */
dwc3_send_gadget_generic_command(struct dwc3 * dwc,unsigned int cmd,u32 param)249 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
250 		u32 param)
251 {
252 	u32		timeout = 500;
253 	int		status = 0;
254 	int		ret = 0;
255 	u32		reg;
256 
257 	dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
258 	dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
259 
260 	do {
261 		reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
262 		if (!(reg & DWC3_DGCMD_CMDACT)) {
263 			status = DWC3_DGCMD_STATUS(reg);
264 			if (status)
265 				ret = -EINVAL;
266 			break;
267 		}
268 	} while (--timeout);
269 
270 	if (!timeout) {
271 		ret = -ETIMEDOUT;
272 		status = -ETIMEDOUT;
273 	}
274 
275 	trace_dwc3_gadget_generic_cmd(cmd, param, status);
276 
277 	return ret;
278 }
279 
280 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async);
281 
282 /**
283  * dwc3_send_gadget_ep_cmd - issue an endpoint command
284  * @dep: the endpoint to which the command is going to be issued
285  * @cmd: the command to be issued
286  * @params: parameters to the command
287  *
288  * Caller should handle locking. This function will issue @cmd with given
289  * @params to @dep and wait for its completion.
290  *
291  * According to the programming guide, if the link state is in L1/L2/U3,
292  * then sending the Start Transfer command may not complete. The
293  * programming guide suggested to bring the link state back to ON/U0 by
294  * performing remote wakeup prior to sending the command. However, don't
295  * initiate remote wakeup when the user/function does not send wakeup
296  * request via wakeup ops. Send the command when it's allowed.
297  *
298  * Notes:
299  * For L1 link state, issuing a command requires the clearing of
300  * GUSB2PHYCFG.SUSPENDUSB2, which turns on the signal required to complete
301  * the given command (usually within 50us). This should happen within the
302  * command timeout set by driver. No additional step is needed.
303  *
304  * For L2 or U3 link state, the gadget is in USB suspend. Care should be
305  * taken when sending Start Transfer command to ensure that it's done after
306  * USB resume.
307  */
dwc3_send_gadget_ep_cmd(struct dwc3_ep * dep,unsigned int cmd,struct dwc3_gadget_ep_cmd_params * params)308 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
309 		struct dwc3_gadget_ep_cmd_params *params)
310 {
311 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
312 	struct dwc3		*dwc = dep->dwc;
313 	u32			timeout = 5000;
314 	u32			saved_config = 0;
315 	u32			reg;
316 
317 	int			cmd_status = 0;
318 	int			ret = -EINVAL;
319 
320 	/*
321 	 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
322 	 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
323 	 * endpoint command.
324 	 *
325 	 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
326 	 * settings. Restore them after the command is completed.
327 	 *
328 	 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
329 	 */
330 	if (dwc->gadget->speed <= USB_SPEED_HIGH ||
331 	    DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
332 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
333 		if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
334 			saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
335 			reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
336 		}
337 
338 		if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
339 			saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
340 			reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
341 		}
342 
343 		if (saved_config)
344 			dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
345 	}
346 
347 	/*
348 	 * For some commands such as Update Transfer command, DEPCMDPARn
349 	 * registers are reserved. Since the driver often sends Update Transfer
350 	 * command, don't write to DEPCMDPARn to avoid register write delays and
351 	 * improve performance.
352 	 */
353 	if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
354 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
355 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
356 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
357 	}
358 
359 	/*
360 	 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
361 	 * not relying on XferNotReady, we can make use of a special "No
362 	 * Response Update Transfer" command where we should clear both CmdAct
363 	 * and CmdIOC bits.
364 	 *
365 	 * With this, we don't need to wait for command completion and can
366 	 * straight away issue further commands to the endpoint.
367 	 *
368 	 * NOTICE: We're making an assumption that control endpoints will never
369 	 * make use of Update Transfer command. This is a safe assumption
370 	 * because we can never have more than one request at a time with
371 	 * Control Endpoints. If anybody changes that assumption, this chunk
372 	 * needs to be updated accordingly.
373 	 */
374 	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
375 			!usb_endpoint_xfer_isoc(desc))
376 		cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
377 	else
378 		cmd |= DWC3_DEPCMD_CMDACT;
379 
380 	dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
381 
382 	if (!(cmd & DWC3_DEPCMD_CMDACT) ||
383 		(DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
384 		!(cmd & DWC3_DEPCMD_CMDIOC))) {
385 		ret = 0;
386 		goto skip_status;
387 	}
388 
389 	do {
390 		reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
391 		if (!(reg & DWC3_DEPCMD_CMDACT)) {
392 			cmd_status = DWC3_DEPCMD_STATUS(reg);
393 
394 			switch (cmd_status) {
395 			case 0:
396 				ret = 0;
397 				break;
398 			case DEPEVT_TRANSFER_NO_RESOURCE:
399 				dev_WARN(dwc->dev, "No resource for %s\n",
400 					 dep->name);
401 				ret = -EINVAL;
402 				break;
403 			case DEPEVT_TRANSFER_BUS_EXPIRY:
404 				/*
405 				 * SW issues START TRANSFER command to
406 				 * isochronous ep with future frame interval. If
407 				 * future interval time has already passed when
408 				 * core receives the command, it will respond
409 				 * with an error status of 'Bus Expiry'.
410 				 *
411 				 * Instead of always returning -EINVAL, let's
412 				 * give a hint to the gadget driver that this is
413 				 * the case by returning -EAGAIN.
414 				 */
415 				ret = -EAGAIN;
416 				break;
417 			default:
418 				dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
419 			}
420 
421 			break;
422 		}
423 	} while (--timeout);
424 
425 	if (timeout == 0) {
426 		ret = -ETIMEDOUT;
427 		cmd_status = -ETIMEDOUT;
428 	}
429 
430 skip_status:
431 	trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
432 
433 	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
434 		if (ret == 0)
435 			dep->flags |= DWC3_EP_TRANSFER_STARTED;
436 
437 		if (ret != -ETIMEDOUT)
438 			dwc3_gadget_ep_get_transfer_index(dep);
439 	}
440 
441 	if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
442 	    !(cmd & DWC3_DEPCMD_CMDIOC))
443 		mdelay(1);
444 
445 	if (saved_config) {
446 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
447 		reg |= saved_config;
448 		dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
449 	}
450 
451 	return ret;
452 }
453 
dwc3_send_clear_stall_ep_cmd(struct dwc3_ep * dep)454 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
455 {
456 	struct dwc3 *dwc = dep->dwc;
457 	struct dwc3_gadget_ep_cmd_params params;
458 	u32 cmd = DWC3_DEPCMD_CLEARSTALL;
459 
460 	/*
461 	 * As of core revision 2.60a the recommended programming model
462 	 * is to set the ClearPendIN bit when issuing a Clear Stall EP
463 	 * command for IN endpoints. This is to prevent an issue where
464 	 * some (non-compliant) hosts may not send ACK TPs for pending
465 	 * IN transfers due to a mishandled error condition. Synopsys
466 	 * STAR 9000614252.
467 	 */
468 	if (dep->direction &&
469 	    !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
470 	    (dwc->gadget->speed >= USB_SPEED_SUPER))
471 		cmd |= DWC3_DEPCMD_CLEARPENDIN;
472 
473 	memset(&params, 0, sizeof(params));
474 
475 	return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
476 }
477 
dwc3_trb_dma_offset(struct dwc3_ep * dep,struct dwc3_trb * trb)478 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
479 		struct dwc3_trb *trb)
480 {
481 	u32		offset = (char *) trb - (char *) dep->trb_pool;
482 
483 	return dep->trb_pool_dma + offset;
484 }
485 
dwc3_alloc_trb_pool(struct dwc3_ep * dep)486 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
487 {
488 	struct dwc3		*dwc = dep->dwc;
489 
490 	if (dep->trb_pool)
491 		return 0;
492 
493 	dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
494 			sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
495 			&dep->trb_pool_dma, GFP_KERNEL);
496 	if (!dep->trb_pool) {
497 		dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
498 				dep->name);
499 		return -ENOMEM;
500 	}
501 
502 	return 0;
503 }
504 
dwc3_free_trb_pool(struct dwc3_ep * dep)505 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
506 {
507 	struct dwc3		*dwc = dep->dwc;
508 
509 	dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
510 			dep->trb_pool, dep->trb_pool_dma);
511 
512 	dep->trb_pool = NULL;
513 	dep->trb_pool_dma = 0;
514 }
515 
dwc3_gadget_set_xfer_resource(struct dwc3_ep * dep)516 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
517 {
518 	struct dwc3_gadget_ep_cmd_params params;
519 
520 	memset(&params, 0x00, sizeof(params));
521 
522 	params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
523 
524 	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
525 			&params);
526 }
527 
528 /**
529  * dwc3_gadget_start_config - configure ep resources
530  * @dep: endpoint that is being enabled
531  *
532  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
533  * completion, it will set Transfer Resource for all available endpoints.
534  *
535  * The assignment of transfer resources cannot perfectly follow the data book
536  * due to the fact that the controller driver does not have all knowledge of the
537  * configuration in advance. It is given this information piecemeal by the
538  * composite gadget framework after every SET_CONFIGURATION and
539  * SET_INTERFACE. Trying to follow the databook programming model in this
540  * scenario can cause errors. For two reasons:
541  *
542  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
543  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
544  * incorrect in the scenario of multiple interfaces.
545  *
546  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
547  * endpoint on alt setting (8.1.6).
548  *
549  * The following simplified method is used instead:
550  *
551  * All hardware endpoints can be assigned a transfer resource and this setting
552  * will stay persistent until either a core reset or hibernation. So whenever we
553  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
554  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
555  * guaranteed that there are as many transfer resources as endpoints.
556  *
557  * This function is called for each endpoint when it is being enabled but is
558  * triggered only when called for EP0-out, which always happens first, and which
559  * should only happen in one of the above conditions.
560  */
dwc3_gadget_start_config(struct dwc3_ep * dep)561 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
562 {
563 	struct dwc3_gadget_ep_cmd_params params;
564 	struct dwc3		*dwc;
565 	u32			cmd;
566 	int			i;
567 	int			ret;
568 
569 	if (dep->number)
570 		return 0;
571 
572 	memset(&params, 0x00, sizeof(params));
573 	cmd = DWC3_DEPCMD_DEPSTARTCFG;
574 	dwc = dep->dwc;
575 
576 	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
577 	if (ret)
578 		return ret;
579 
580 	for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
581 		struct dwc3_ep *dep = dwc->eps[i];
582 
583 		if (!dep)
584 			continue;
585 
586 		ret = dwc3_gadget_set_xfer_resource(dep);
587 		if (ret)
588 			return ret;
589 	}
590 
591 	return 0;
592 }
593 
dwc3_gadget_set_ep_config(struct dwc3_ep * dep,unsigned int action)594 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
595 {
596 	const struct usb_ss_ep_comp_descriptor *comp_desc;
597 	const struct usb_endpoint_descriptor *desc;
598 	struct dwc3_gadget_ep_cmd_params params;
599 	struct dwc3 *dwc = dep->dwc;
600 
601 	comp_desc = dep->endpoint.comp_desc;
602 	desc = dep->endpoint.desc;
603 
604 	memset(&params, 0x00, sizeof(params));
605 
606 	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
607 		| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
608 
609 	/* Burst size is only needed in SuperSpeed mode */
610 	if (dwc->gadget->speed >= USB_SPEED_SUPER) {
611 		u32 burst = dep->endpoint.maxburst;
612 
613 		params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
614 	}
615 
616 	params.param0 |= action;
617 	if (action == DWC3_DEPCFG_ACTION_RESTORE)
618 		params.param2 |= dep->saved_state;
619 
620 	if (usb_endpoint_xfer_control(desc))
621 		params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
622 
623 	if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
624 		params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
625 
626 	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
627 		params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
628 			| DWC3_DEPCFG_XFER_COMPLETE_EN
629 			| DWC3_DEPCFG_STREAM_EVENT_EN;
630 		dep->stream_capable = true;
631 	}
632 
633 	if (!usb_endpoint_xfer_control(desc))
634 		params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
635 
636 	/*
637 	 * We are doing 1:1 mapping for endpoints, meaning
638 	 * Physical Endpoints 2 maps to Logical Endpoint 2 and
639 	 * so on. We consider the direction bit as part of the physical
640 	 * endpoint number. So USB endpoint 0x81 is 0x03.
641 	 */
642 	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
643 
644 	/*
645 	 * We must use the lower 16 TX FIFOs even though
646 	 * HW might have more
647 	 */
648 	if (dep->direction)
649 		params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
650 
651 	if (desc->bInterval) {
652 		u8 bInterval_m1;
653 
654 		/*
655 		 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
656 		 *
657 		 * NOTE: The programming guide incorrectly stated bInterval_m1
658 		 * must be set to 0 when operating in fullspeed. Internally the
659 		 * controller does not have this limitation. See DWC_usb3x
660 		 * programming guide section 3.2.2.1.
661 		 */
662 		bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
663 
664 		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
665 		    dwc->gadget->speed == USB_SPEED_FULL)
666 			dep->interval = desc->bInterval;
667 		else
668 			dep->interval = 1 << (desc->bInterval - 1);
669 
670 		params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
671 	}
672 
673 	return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
674 }
675 
676 /**
677  * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
678  * @dwc: pointer to the DWC3 context
679  * @mult: multiplier to be used when calculating the fifo_size
680  *
681  * Calculates the size value based on the equation below:
682  *
683  * DWC3 revision 280A and prior:
684  * fifo_size = mult * (max_packet / mdwidth) + 1;
685  *
686  * DWC3 revision 290A and onwards:
687  * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
688  *
689  * The max packet size is set to 1024, as the txfifo requirements mainly apply
690  * to super speed USB use cases.  However, it is safe to overestimate the fifo
691  * allocations for other scenarios, i.e. high speed USB.
692  */
dwc3_gadget_calc_tx_fifo_size(struct dwc3 * dwc,int mult)693 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
694 {
695 	int max_packet = 1024;
696 	int fifo_size;
697 	int mdwidth;
698 
699 	mdwidth = dwc3_mdwidth(dwc);
700 
701 	/* MDWIDTH is represented in bits, we need it in bytes */
702 	mdwidth >>= 3;
703 
704 	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
705 		fifo_size = mult * (max_packet / mdwidth) + 1;
706 	else
707 		fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
708 	return fifo_size;
709 }
710 
711 /**
712  * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
713  * @dwc: pointer to the DWC3 context
714  *
715  * Iterates through all the endpoint registers and clears the previous txfifo
716  * allocations.
717  */
dwc3_gadget_clear_tx_fifos(struct dwc3 * dwc)718 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
719 {
720 	struct dwc3_ep *dep;
721 	int fifo_depth;
722 	int size;
723 	int num;
724 
725 	if (!dwc->do_fifo_resize)
726 		return;
727 
728 	/* Read ep0IN related TXFIFO size */
729 	dep = dwc->eps[1];
730 	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
731 	if (DWC3_IP_IS(DWC3))
732 		fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
733 	else
734 		fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
735 
736 	dwc->last_fifo_depth = fifo_depth;
737 	/* Clear existing TXFIFO for all IN eps except ep0 */
738 	for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
739 	     num += 2) {
740 		dep = dwc->eps[num];
741 		/* Don't change TXFRAMNUM on usb31 version */
742 		size = DWC3_IP_IS(DWC3) ? 0 :
743 			dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
744 				   DWC31_GTXFIFOSIZ_TXFRAMNUM;
745 
746 		dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
747 		dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
748 	}
749 	dwc->num_ep_resized = 0;
750 }
751 
752 /*
753  * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
754  * @dwc: pointer to our context structure
755  *
756  * This function will a best effort FIFO allocation in order
757  * to improve FIFO usage and throughput, while still allowing
758  * us to enable as many endpoints as possible.
759  *
760  * Keep in mind that this operation will be highly dependent
761  * on the configured size for RAM1 - which contains TxFifo -,
762  * the amount of endpoints enabled on coreConsultant tool, and
763  * the width of the Master Bus.
764  *
765  * In general, FIFO depths are represented with the following equation:
766  *
767  * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
768  *
769  * In conjunction with dwc3_gadget_check_config(), this resizing logic will
770  * ensure that all endpoints will have enough internal memory for one max
771  * packet per endpoint.
772  */
dwc3_gadget_resize_tx_fifos(struct dwc3_ep * dep)773 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
774 {
775 	struct dwc3 *dwc = dep->dwc;
776 	int fifo_0_start;
777 	int ram1_depth;
778 	int fifo_size;
779 	int min_depth;
780 	int num_in_ep;
781 	int remaining;
782 	int num_fifos = 1;
783 	int fifo;
784 	int tmp;
785 
786 	if (!dwc->do_fifo_resize)
787 		return 0;
788 
789 	/* resize IN endpoints except ep0 */
790 	if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
791 		return 0;
792 
793 	/* bail if already resized */
794 	if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
795 		return 0;
796 
797 	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
798 
799 	if ((dep->endpoint.maxburst > 1 &&
800 	     usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||
801 	    usb_endpoint_xfer_isoc(dep->endpoint.desc))
802 		num_fifos = 3;
803 
804 	if (dep->endpoint.maxburst > 6 &&
805 	    (usb_endpoint_xfer_bulk(dep->endpoint.desc) ||
806 	     usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
807 		num_fifos = dwc->tx_fifo_resize_max_num;
808 
809 	/* FIFO size for a single buffer */
810 	fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
811 
812 	/* Calculate the number of remaining EPs w/o any FIFO */
813 	num_in_ep = dwc->max_cfg_eps;
814 	num_in_ep -= dwc->num_ep_resized;
815 
816 	/* Reserve at least one FIFO for the number of IN EPs */
817 	min_depth = num_in_ep * (fifo + 1);
818 	remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
819 	remaining = max_t(int, 0, remaining);
820 	/*
821 	 * We've already reserved 1 FIFO per EP, so check what we can fit in
822 	 * addition to it.  If there is not enough remaining space, allocate
823 	 * all the remaining space to the EP.
824 	 */
825 	fifo_size = (num_fifos - 1) * fifo;
826 	if (remaining < fifo_size)
827 		fifo_size = remaining;
828 
829 	fifo_size += fifo;
830 	/* Last increment according to the TX FIFO size equation */
831 	fifo_size++;
832 
833 	/* Check if TXFIFOs start at non-zero addr */
834 	tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
835 	fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
836 
837 	fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
838 	if (DWC3_IP_IS(DWC3))
839 		dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
840 	else
841 		dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
842 
843 	/* Check fifo size allocation doesn't exceed available RAM size. */
844 	if (dwc->last_fifo_depth >= ram1_depth) {
845 		dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
846 			dwc->last_fifo_depth, ram1_depth,
847 			dep->endpoint.name, fifo_size);
848 		if (DWC3_IP_IS(DWC3))
849 			fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
850 		else
851 			fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
852 
853 		dwc->last_fifo_depth -= fifo_size;
854 		return -ENOMEM;
855 	}
856 
857 	dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
858 	dep->flags |= DWC3_EP_TXFIFO_RESIZED;
859 	dwc->num_ep_resized++;
860 
861 	return 0;
862 }
863 
864 /**
865  * __dwc3_gadget_ep_enable - initializes a hw endpoint
866  * @dep: endpoint to be initialized
867  * @action: one of INIT, MODIFY or RESTORE
868  *
869  * Caller should take care of locking. Execute all necessary commands to
870  * initialize a HW endpoint so it can be used by a gadget driver.
871  */
__dwc3_gadget_ep_enable(struct dwc3_ep * dep,unsigned int action)872 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
873 {
874 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
875 	struct dwc3		*dwc = dep->dwc;
876 
877 	u32			reg;
878 	int			ret;
879 
880 	if (!(dep->flags & DWC3_EP_ENABLED)) {
881 		ret = dwc3_gadget_resize_tx_fifos(dep);
882 		if (ret)
883 			return ret;
884 
885 		ret = dwc3_gadget_start_config(dep);
886 		if (ret)
887 			return ret;
888 	}
889 
890 	ret = dwc3_gadget_set_ep_config(dep, action);
891 	if (ret)
892 		return ret;
893 
894 	if (!(dep->flags & DWC3_EP_ENABLED)) {
895 		struct dwc3_trb	*trb_st_hw;
896 		struct dwc3_trb	*trb_link;
897 
898 		dep->type = usb_endpoint_type(desc);
899 		dep->flags |= DWC3_EP_ENABLED;
900 
901 		reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
902 		reg |= DWC3_DALEPENA_EP(dep->number);
903 		dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
904 
905 		dep->trb_dequeue = 0;
906 		dep->trb_enqueue = 0;
907 
908 		if (usb_endpoint_xfer_control(desc))
909 			goto out;
910 
911 		/* Initialize the TRB ring */
912 		memset(dep->trb_pool, 0,
913 		       sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
914 
915 		/* Link TRB. The HWO bit is never reset */
916 		trb_st_hw = &dep->trb_pool[0];
917 
918 		trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
919 		trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
920 		trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
921 		trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
922 		trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
923 	}
924 
925 	/*
926 	 * Issue StartTransfer here with no-op TRB so we can always rely on No
927 	 * Response Update Transfer command.
928 	 */
929 	if (usb_endpoint_xfer_bulk(desc) ||
930 			usb_endpoint_xfer_int(desc)) {
931 		struct dwc3_gadget_ep_cmd_params params;
932 		struct dwc3_trb	*trb;
933 		dma_addr_t trb_dma;
934 		u32 cmd;
935 
936 		memset(&params, 0, sizeof(params));
937 		trb = &dep->trb_pool[0];
938 		trb_dma = dwc3_trb_dma_offset(dep, trb);
939 
940 		params.param0 = upper_32_bits(trb_dma);
941 		params.param1 = lower_32_bits(trb_dma);
942 
943 		cmd = DWC3_DEPCMD_STARTTRANSFER;
944 
945 		ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
946 		if (ret < 0)
947 			return ret;
948 
949 		if (dep->stream_capable) {
950 			/*
951 			 * For streams, at start, there maybe a race where the
952 			 * host primes the endpoint before the function driver
953 			 * queues a request to initiate a stream. In that case,
954 			 * the controller will not see the prime to generate the
955 			 * ERDY and start stream. To workaround this, issue a
956 			 * no-op TRB as normal, but end it immediately. As a
957 			 * result, when the function driver queues the request,
958 			 * the next START_TRANSFER command will cause the
959 			 * controller to generate an ERDY to initiate the
960 			 * stream.
961 			 */
962 			dwc3_stop_active_transfer(dep, true, true);
963 
964 			/*
965 			 * All stream eps will reinitiate stream on NoStream
966 			 * rejection until we can determine that the host can
967 			 * prime after the first transfer.
968 			 *
969 			 * However, if the controller is capable of
970 			 * TXF_FLUSH_BYPASS, then IN direction endpoints will
971 			 * automatically restart the stream without the driver
972 			 * initiation.
973 			 */
974 			if (!dep->direction ||
975 			    !(dwc->hwparams.hwparams9 &
976 			      DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
977 				dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
978 		}
979 	}
980 
981 out:
982 	trace_dwc3_gadget_ep_enable(dep);
983 
984 	return 0;
985 }
986 
dwc3_remove_requests(struct dwc3 * dwc,struct dwc3_ep * dep,int status)987 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
988 {
989 	struct dwc3_request		*req;
990 
991 	dwc3_stop_active_transfer(dep, true, false);
992 
993 	/* If endxfer is delayed, avoid unmapping requests */
994 	if (dep->flags & DWC3_EP_DELAY_STOP)
995 		return;
996 
997 	/* - giveback all requests to gadget driver */
998 	while (!list_empty(&dep->started_list)) {
999 		req = next_request(&dep->started_list);
1000 
1001 		dwc3_gadget_giveback(dep, req, status);
1002 	}
1003 
1004 	while (!list_empty(&dep->pending_list)) {
1005 		req = next_request(&dep->pending_list);
1006 
1007 		dwc3_gadget_giveback(dep, req, status);
1008 	}
1009 
1010 	while (!list_empty(&dep->cancelled_list)) {
1011 		req = next_request(&dep->cancelled_list);
1012 
1013 		dwc3_gadget_giveback(dep, req, status);
1014 	}
1015 }
1016 
1017 /**
1018  * __dwc3_gadget_ep_disable - disables a hw endpoint
1019  * @dep: the endpoint to disable
1020  *
1021  * This function undoes what __dwc3_gadget_ep_enable did and also removes
1022  * requests which are currently being processed by the hardware and those which
1023  * are not yet scheduled.
1024  *
1025  * Caller should take care of locking.
1026  */
__dwc3_gadget_ep_disable(struct dwc3_ep * dep)1027 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1028 {
1029 	struct dwc3		*dwc = dep->dwc;
1030 	u32			reg;
1031 	u32			mask;
1032 
1033 	trace_dwc3_gadget_ep_disable(dep);
1034 
1035 	/* make sure HW endpoint isn't stalled */
1036 	if (dep->flags & DWC3_EP_STALL)
1037 		__dwc3_gadget_ep_set_halt(dep, 0, false);
1038 
1039 	reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
1040 	reg &= ~DWC3_DALEPENA_EP(dep->number);
1041 	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
1042 
1043 	dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
1044 
1045 	dep->stream_capable = false;
1046 	dep->type = 0;
1047 	mask = DWC3_EP_TXFIFO_RESIZED;
1048 	/*
1049 	 * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
1050 	 * set.  Do not clear DEP flags, so that the end transfer command will
1051 	 * be reattempted during the next SETUP stage.
1052 	 */
1053 	if (dep->flags & DWC3_EP_DELAY_STOP)
1054 		mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED);
1055 	dep->flags &= mask;
1056 
1057 	/* Clear out the ep descriptors for non-ep0 */
1058 	if (dep->number > 1) {
1059 		dep->endpoint.comp_desc = NULL;
1060 		dep->endpoint.desc = NULL;
1061 	}
1062 
1063 	return 0;
1064 }
1065 
1066 /* -------------------------------------------------------------------------- */
1067 
dwc3_gadget_ep0_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)1068 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1069 		const struct usb_endpoint_descriptor *desc)
1070 {
1071 	return -EINVAL;
1072 }
1073 
dwc3_gadget_ep0_disable(struct usb_ep * ep)1074 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1075 {
1076 	return -EINVAL;
1077 }
1078 
1079 /* -------------------------------------------------------------------------- */
1080 
dwc3_gadget_ep_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)1081 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1082 		const struct usb_endpoint_descriptor *desc)
1083 {
1084 	struct dwc3_ep			*dep;
1085 	struct dwc3			*dwc;
1086 	unsigned long			flags;
1087 	int				ret;
1088 
1089 	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1090 		pr_debug("dwc3: invalid parameters\n");
1091 		return -EINVAL;
1092 	}
1093 
1094 	if (!desc->wMaxPacketSize) {
1095 		pr_debug("dwc3: missing wMaxPacketSize\n");
1096 		return -EINVAL;
1097 	}
1098 
1099 	dep = to_dwc3_ep(ep);
1100 	dwc = dep->dwc;
1101 
1102 	if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1103 					"%s is already enabled\n",
1104 					dep->name))
1105 		return 0;
1106 
1107 	spin_lock_irqsave(&dwc->lock, flags);
1108 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1109 	spin_unlock_irqrestore(&dwc->lock, flags);
1110 
1111 	return ret;
1112 }
1113 
dwc3_gadget_ep_disable(struct usb_ep * ep)1114 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1115 {
1116 	struct dwc3_ep			*dep;
1117 	struct dwc3			*dwc;
1118 	unsigned long			flags;
1119 	int				ret;
1120 
1121 	if (!ep) {
1122 		pr_debug("dwc3: invalid parameters\n");
1123 		return -EINVAL;
1124 	}
1125 
1126 	dep = to_dwc3_ep(ep);
1127 	dwc = dep->dwc;
1128 
1129 	if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1130 					"%s is already disabled\n",
1131 					dep->name))
1132 		return 0;
1133 
1134 	spin_lock_irqsave(&dwc->lock, flags);
1135 	ret = __dwc3_gadget_ep_disable(dep);
1136 	spin_unlock_irqrestore(&dwc->lock, flags);
1137 
1138 	return ret;
1139 }
1140 
dwc3_gadget_ep_alloc_request(struct usb_ep * ep,gfp_t gfp_flags)1141 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1142 		gfp_t gfp_flags)
1143 {
1144 	struct dwc3_request		*req;
1145 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1146 
1147 	req = kzalloc(sizeof(*req), gfp_flags);
1148 	if (!req)
1149 		return NULL;
1150 
1151 	req->direction	= dep->direction;
1152 	req->epnum	= dep->number;
1153 	req->dep	= dep;
1154 	req->status	= DWC3_REQUEST_STATUS_UNKNOWN;
1155 
1156 	trace_dwc3_alloc_request(req);
1157 
1158 	return &req->request;
1159 }
1160 
dwc3_gadget_ep_free_request(struct usb_ep * ep,struct usb_request * request)1161 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1162 		struct usb_request *request)
1163 {
1164 	struct dwc3_request		*req = to_dwc3_request(request);
1165 
1166 	trace_dwc3_free_request(req);
1167 	kfree(req);
1168 }
1169 
1170 /**
1171  * dwc3_ep_prev_trb - returns the previous TRB in the ring
1172  * @dep: The endpoint with the TRB ring
1173  * @index: The index of the current TRB in the ring
1174  *
1175  * Returns the TRB prior to the one pointed to by the index. If the
1176  * index is 0, we will wrap backwards, skip the link TRB, and return
1177  * the one just before that.
1178  */
dwc3_ep_prev_trb(struct dwc3_ep * dep,u8 index)1179 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1180 {
1181 	u8 tmp = index;
1182 
1183 	if (!tmp)
1184 		tmp = DWC3_TRB_NUM - 1;
1185 
1186 	return &dep->trb_pool[tmp - 1];
1187 }
1188 
dwc3_calc_trbs_left(struct dwc3_ep * dep)1189 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1190 {
1191 	u8			trbs_left;
1192 
1193 	/*
1194 	 * If the enqueue & dequeue are equal then the TRB ring is either full
1195 	 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1196 	 * pending to be processed by the driver.
1197 	 */
1198 	if (dep->trb_enqueue == dep->trb_dequeue) {
1199 		/*
1200 		 * If there is any request remained in the started_list at
1201 		 * this point, that means there is no TRB available.
1202 		 */
1203 		if (!list_empty(&dep->started_list))
1204 			return 0;
1205 
1206 		return DWC3_TRB_NUM - 1;
1207 	}
1208 
1209 	trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1210 	trbs_left &= (DWC3_TRB_NUM - 1);
1211 
1212 	if (dep->trb_dequeue < dep->trb_enqueue)
1213 		trbs_left--;
1214 
1215 	return trbs_left;
1216 }
1217 
1218 /**
1219  * dwc3_prepare_one_trb - setup one TRB from one request
1220  * @dep: endpoint for which this request is prepared
1221  * @req: dwc3_request pointer
1222  * @trb_length: buffer size of the TRB
1223  * @chain: should this TRB be chained to the next?
1224  * @node: only for isochronous endpoints. First TRB needs different type.
1225  * @use_bounce_buffer: set to use bounce buffer
1226  * @must_interrupt: set to interrupt on TRB completion
1227  */
dwc3_prepare_one_trb(struct dwc3_ep * dep,struct dwc3_request * req,unsigned int trb_length,unsigned int chain,unsigned int node,bool use_bounce_buffer,bool must_interrupt)1228 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1229 		struct dwc3_request *req, unsigned int trb_length,
1230 		unsigned int chain, unsigned int node, bool use_bounce_buffer,
1231 		bool must_interrupt)
1232 {
1233 	struct dwc3_trb		*trb;
1234 	dma_addr_t		dma;
1235 	unsigned int		stream_id = req->request.stream_id;
1236 	unsigned int		short_not_ok = req->request.short_not_ok;
1237 	unsigned int		no_interrupt = req->request.no_interrupt;
1238 	unsigned int		is_last = req->request.is_last;
1239 	struct dwc3		*dwc = dep->dwc;
1240 	struct usb_gadget	*gadget = dwc->gadget;
1241 	enum usb_device_speed	speed = gadget->speed;
1242 
1243 	if (use_bounce_buffer)
1244 		dma = dep->dwc->bounce_addr;
1245 	else if (req->request.num_sgs > 0)
1246 		dma = sg_dma_address(req->start_sg);
1247 	else
1248 		dma = req->request.dma;
1249 
1250 	trb = &dep->trb_pool[dep->trb_enqueue];
1251 
1252 	if (!req->trb) {
1253 		dwc3_gadget_move_started_request(req);
1254 		req->trb = trb;
1255 		req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1256 	}
1257 
1258 	req->num_trbs++;
1259 
1260 	trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1261 	trb->bpl = lower_32_bits(dma);
1262 	trb->bph = upper_32_bits(dma);
1263 
1264 	switch (usb_endpoint_type(dep->endpoint.desc)) {
1265 	case USB_ENDPOINT_XFER_CONTROL:
1266 		trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1267 		break;
1268 
1269 	case USB_ENDPOINT_XFER_ISOC:
1270 		if (!node) {
1271 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1272 
1273 			/*
1274 			 * USB Specification 2.0 Section 5.9.2 states that: "If
1275 			 * there is only a single transaction in the microframe,
1276 			 * only a DATA0 data packet PID is used.  If there are
1277 			 * two transactions per microframe, DATA1 is used for
1278 			 * the first transaction data packet and DATA0 is used
1279 			 * for the second transaction data packet.  If there are
1280 			 * three transactions per microframe, DATA2 is used for
1281 			 * the first transaction data packet, DATA1 is used for
1282 			 * the second, and DATA0 is used for the third."
1283 			 *
1284 			 * IOW, we should satisfy the following cases:
1285 			 *
1286 			 * 1) length <= maxpacket
1287 			 *	- DATA0
1288 			 *
1289 			 * 2) maxpacket < length <= (2 * maxpacket)
1290 			 *	- DATA1, DATA0
1291 			 *
1292 			 * 3) (2 * maxpacket) < length <= (3 * maxpacket)
1293 			 *	- DATA2, DATA1, DATA0
1294 			 */
1295 			if (speed == USB_SPEED_HIGH) {
1296 				struct usb_ep *ep = &dep->endpoint;
1297 				unsigned int mult = 2;
1298 				unsigned int maxp = usb_endpoint_maxp(ep->desc);
1299 
1300 				if (req->request.length <= (2 * maxp))
1301 					mult--;
1302 
1303 				if (req->request.length <= maxp)
1304 					mult--;
1305 
1306 				trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1307 			}
1308 		} else {
1309 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1310 		}
1311 
1312 		if (!no_interrupt && !chain)
1313 			trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1314 		break;
1315 
1316 	case USB_ENDPOINT_XFER_BULK:
1317 	case USB_ENDPOINT_XFER_INT:
1318 		trb->ctrl = DWC3_TRBCTL_NORMAL;
1319 		break;
1320 	default:
1321 		/*
1322 		 * This is only possible with faulty memory because we
1323 		 * checked it already :)
1324 		 */
1325 		dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1326 				usb_endpoint_type(dep->endpoint.desc));
1327 	}
1328 
1329 	/*
1330 	 * Enable Continue on Short Packet
1331 	 * when endpoint is not a stream capable
1332 	 */
1333 	if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1334 		if (!dep->stream_capable)
1335 			trb->ctrl |= DWC3_TRB_CTRL_CSP;
1336 
1337 		if (short_not_ok)
1338 			trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1339 	}
1340 
1341 	/* All TRBs setup for MST must set CSP=1 when LST=0 */
1342 	if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1343 		trb->ctrl |= DWC3_TRB_CTRL_CSP;
1344 
1345 	if ((!no_interrupt && !chain) || must_interrupt)
1346 		trb->ctrl |= DWC3_TRB_CTRL_IOC;
1347 
1348 	if (chain)
1349 		trb->ctrl |= DWC3_TRB_CTRL_CHN;
1350 	else if (dep->stream_capable && is_last &&
1351 		 !DWC3_MST_CAPABLE(&dwc->hwparams))
1352 		trb->ctrl |= DWC3_TRB_CTRL_LST;
1353 
1354 	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1355 		trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1356 
1357 	/*
1358 	 * As per data book 4.2.3.2TRB Control Bit Rules section
1359 	 *
1360 	 * The controller autonomously checks the HWO field of a TRB to determine if the
1361 	 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1362 	 * is valid before setting the HWO field to '1'. In most systems, this means that
1363 	 * software must update the fourth DWORD of a TRB last.
1364 	 *
1365 	 * However there is a possibility of CPU re-ordering here which can cause
1366 	 * controller to observe the HWO bit set prematurely.
1367 	 * Add a write memory barrier to prevent CPU re-ordering.
1368 	 */
1369 	wmb();
1370 	trb->ctrl |= DWC3_TRB_CTRL_HWO;
1371 
1372 	dwc3_ep_inc_enq(dep);
1373 
1374 	trace_dwc3_prepare_trb(dep, trb);
1375 }
1376 
dwc3_needs_extra_trb(struct dwc3_ep * dep,struct dwc3_request * req)1377 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1378 {
1379 	unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1380 	unsigned int rem = req->request.length % maxp;
1381 
1382 	if ((req->request.length && req->request.zero && !rem &&
1383 			!usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1384 			(!req->direction && rem))
1385 		return true;
1386 
1387 	return false;
1388 }
1389 
1390 /**
1391  * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1392  * @dep: The endpoint that the request belongs to
1393  * @req: The request to prepare
1394  * @entry_length: The last SG entry size
1395  * @node: Indicates whether this is not the first entry (for isoc only)
1396  *
1397  * Return the number of TRBs prepared.
1398  */
dwc3_prepare_last_sg(struct dwc3_ep * dep,struct dwc3_request * req,unsigned int entry_length,unsigned int node)1399 static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1400 		struct dwc3_request *req, unsigned int entry_length,
1401 		unsigned int node)
1402 {
1403 	unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1404 	unsigned int rem = req->request.length % maxp;
1405 	unsigned int num_trbs = 1;
1406 
1407 	if (dwc3_needs_extra_trb(dep, req))
1408 		num_trbs++;
1409 
1410 	if (dwc3_calc_trbs_left(dep) < num_trbs)
1411 		return 0;
1412 
1413 	req->needs_extra_trb = num_trbs > 1;
1414 
1415 	/* Prepare a normal TRB */
1416 	if (req->direction || req->request.length)
1417 		dwc3_prepare_one_trb(dep, req, entry_length,
1418 				req->needs_extra_trb, node, false, false);
1419 
1420 	/* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1421 	if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1422 		dwc3_prepare_one_trb(dep, req,
1423 				req->direction ? 0 : maxp - rem,
1424 				false, 1, true, false);
1425 
1426 	return num_trbs;
1427 }
1428 
dwc3_prepare_trbs_sg(struct dwc3_ep * dep,struct dwc3_request * req)1429 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1430 		struct dwc3_request *req)
1431 {
1432 	struct scatterlist *sg = req->start_sg;
1433 	struct scatterlist *s;
1434 	int		i;
1435 	unsigned int length = req->request.length;
1436 	unsigned int remaining = req->request.num_mapped_sgs
1437 		- req->num_queued_sgs;
1438 	unsigned int num_trbs = req->num_trbs;
1439 	bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1440 
1441 	/*
1442 	 * If we resume preparing the request, then get the remaining length of
1443 	 * the request and resume where we left off.
1444 	 */
1445 	for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1446 		length -= sg_dma_len(s);
1447 
1448 	for_each_sg(sg, s, remaining, i) {
1449 		unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1450 		unsigned int trb_length;
1451 		bool must_interrupt = false;
1452 		bool last_sg = false;
1453 
1454 		trb_length = min_t(unsigned int, length, sg_dma_len(s));
1455 
1456 		length -= trb_length;
1457 
1458 		/*
1459 		 * IOMMU driver is coalescing the list of sgs which shares a
1460 		 * page boundary into one and giving it to USB driver. With
1461 		 * this the number of sgs mapped is not equal to the number of
1462 		 * sgs passed. So mark the chain bit to false if it isthe last
1463 		 * mapped sg.
1464 		 */
1465 		if ((i == remaining - 1) || !length)
1466 			last_sg = true;
1467 
1468 		if (!num_trbs_left)
1469 			break;
1470 
1471 		if (last_sg) {
1472 			if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1473 				break;
1474 		} else {
1475 			/*
1476 			 * Look ahead to check if we have enough TRBs for the
1477 			 * next SG entry. If not, set interrupt on this TRB to
1478 			 * resume preparing the next SG entry when more TRBs are
1479 			 * free.
1480 			 */
1481 			if (num_trbs_left == 1 || (needs_extra_trb &&
1482 					num_trbs_left <= 2 &&
1483 					sg_dma_len(sg_next(s)) >= length)) {
1484 				struct dwc3_request *r;
1485 
1486 				/* Check if previous requests already set IOC */
1487 				list_for_each_entry(r, &dep->started_list, list) {
1488 					if (r != req && !r->request.no_interrupt)
1489 						break;
1490 
1491 					if (r == req)
1492 						must_interrupt = true;
1493 				}
1494 			}
1495 
1496 			dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1497 					must_interrupt);
1498 		}
1499 
1500 		/*
1501 		 * There can be a situation where all sgs in sglist are not
1502 		 * queued because of insufficient trb number. To handle this
1503 		 * case, update start_sg to next sg to be queued, so that
1504 		 * we have free trbs we can continue queuing from where we
1505 		 * previously stopped
1506 		 */
1507 		if (!last_sg)
1508 			req->start_sg = sg_next(s);
1509 
1510 		req->num_queued_sgs++;
1511 		req->num_pending_sgs--;
1512 
1513 		/*
1514 		 * The number of pending SG entries may not correspond to the
1515 		 * number of mapped SG entries. If all the data are queued, then
1516 		 * don't include unused SG entries.
1517 		 */
1518 		if (length == 0) {
1519 			req->num_pending_sgs = 0;
1520 			break;
1521 		}
1522 
1523 		if (must_interrupt)
1524 			break;
1525 	}
1526 
1527 	return req->num_trbs - num_trbs;
1528 }
1529 
dwc3_prepare_trbs_linear(struct dwc3_ep * dep,struct dwc3_request * req)1530 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1531 		struct dwc3_request *req)
1532 {
1533 	return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1534 }
1535 
1536 /*
1537  * dwc3_prepare_trbs - setup TRBs from requests
1538  * @dep: endpoint for which requests are being prepared
1539  *
1540  * The function goes through the requests list and sets up TRBs for the
1541  * transfers. The function returns once there are no more TRBs available or
1542  * it runs out of requests.
1543  *
1544  * Returns the number of TRBs prepared or negative errno.
1545  */
dwc3_prepare_trbs(struct dwc3_ep * dep)1546 static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1547 {
1548 	struct dwc3_request	*req, *n;
1549 	int			ret = 0;
1550 
1551 	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1552 
1553 	/*
1554 	 * We can get in a situation where there's a request in the started list
1555 	 * but there weren't enough TRBs to fully kick it in the first time
1556 	 * around, so it has been waiting for more TRBs to be freed up.
1557 	 *
1558 	 * In that case, we should check if we have a request with pending_sgs
1559 	 * in the started list and prepare TRBs for that request first,
1560 	 * otherwise we will prepare TRBs completely out of order and that will
1561 	 * break things.
1562 	 */
1563 	list_for_each_entry(req, &dep->started_list, list) {
1564 		if (req->num_pending_sgs > 0) {
1565 			ret = dwc3_prepare_trbs_sg(dep, req);
1566 			if (!ret || req->num_pending_sgs)
1567 				return ret;
1568 		}
1569 
1570 		if (!dwc3_calc_trbs_left(dep))
1571 			return ret;
1572 
1573 		/*
1574 		 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1575 		 * burst capability may try to read and use TRBs beyond the
1576 		 * active transfer instead of stopping.
1577 		 */
1578 		if (dep->stream_capable && req->request.is_last &&
1579 		    !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1580 			return ret;
1581 	}
1582 
1583 	list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1584 		struct dwc3	*dwc = dep->dwc;
1585 
1586 		ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1587 						    dep->direction);
1588 		if (ret)
1589 			return ret;
1590 
1591 		req->sg			= req->request.sg;
1592 		req->start_sg		= req->sg;
1593 		req->num_queued_sgs	= 0;
1594 		req->num_pending_sgs	= req->request.num_mapped_sgs;
1595 
1596 		if (req->num_pending_sgs > 0) {
1597 			ret = dwc3_prepare_trbs_sg(dep, req);
1598 			if (req->num_pending_sgs)
1599 				return ret;
1600 		} else {
1601 			ret = dwc3_prepare_trbs_linear(dep, req);
1602 		}
1603 
1604 		if (!ret || !dwc3_calc_trbs_left(dep))
1605 			return ret;
1606 
1607 		/*
1608 		 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1609 		 * burst capability may try to read and use TRBs beyond the
1610 		 * active transfer instead of stopping.
1611 		 */
1612 		if (dep->stream_capable && req->request.is_last &&
1613 		    !DWC3_MST_CAPABLE(&dwc->hwparams))
1614 			return ret;
1615 	}
1616 
1617 	return ret;
1618 }
1619 
1620 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1621 
__dwc3_gadget_kick_transfer(struct dwc3_ep * dep)1622 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1623 {
1624 	struct dwc3_gadget_ep_cmd_params params;
1625 	struct dwc3_request		*req;
1626 	int				starting;
1627 	int				ret;
1628 	u32				cmd;
1629 
1630 	/*
1631 	 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1632 	 * This happens when we need to stop and restart a transfer such as in
1633 	 * the case of reinitiating a stream or retrying an isoc transfer.
1634 	 */
1635 	ret = dwc3_prepare_trbs(dep);
1636 	if (ret < 0)
1637 		return ret;
1638 
1639 	starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1640 
1641 	/*
1642 	 * If there's no new TRB prepared and we don't need to restart a
1643 	 * transfer, there's no need to update the transfer.
1644 	 */
1645 	if (!ret && !starting)
1646 		return ret;
1647 
1648 	req = next_request(&dep->started_list);
1649 	if (!req) {
1650 		dep->flags |= DWC3_EP_PENDING_REQUEST;
1651 		return 0;
1652 	}
1653 
1654 	memset(&params, 0, sizeof(params));
1655 
1656 	if (starting) {
1657 		params.param0 = upper_32_bits(req->trb_dma);
1658 		params.param1 = lower_32_bits(req->trb_dma);
1659 		cmd = DWC3_DEPCMD_STARTTRANSFER;
1660 
1661 		if (dep->stream_capable)
1662 			cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1663 
1664 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1665 			cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1666 	} else {
1667 		cmd = DWC3_DEPCMD_UPDATETRANSFER |
1668 			DWC3_DEPCMD_PARAM(dep->resource_index);
1669 	}
1670 
1671 	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1672 	if (ret < 0) {
1673 		struct dwc3_request *tmp;
1674 
1675 		if (ret == -EAGAIN)
1676 			return ret;
1677 
1678 		dwc3_stop_active_transfer(dep, true, true);
1679 
1680 		list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1681 			dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1682 
1683 		/* If ep isn't started, then there's no end transfer pending */
1684 		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1685 			dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1686 
1687 		return ret;
1688 	}
1689 
1690 	if (dep->stream_capable && req->request.is_last &&
1691 	    !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1692 		dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1693 
1694 	return 0;
1695 }
1696 
__dwc3_gadget_get_frame(struct dwc3 * dwc)1697 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1698 {
1699 	u32			reg;
1700 
1701 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1702 	return DWC3_DSTS_SOFFN(reg);
1703 }
1704 
1705 /**
1706  * __dwc3_stop_active_transfer - stop the current active transfer
1707  * @dep: isoc endpoint
1708  * @force: set forcerm bit in the command
1709  * @interrupt: command complete interrupt after End Transfer command
1710  *
1711  * When setting force, the ForceRM bit will be set. In that case
1712  * the controller won't update the TRB progress on command
1713  * completion. It also won't clear the HWO bit in the TRB.
1714  * The command will also not complete immediately in that case.
1715  */
__dwc3_stop_active_transfer(struct dwc3_ep * dep,bool force,bool interrupt)1716 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1717 {
1718 	struct dwc3_gadget_ep_cmd_params params;
1719 	u32 cmd;
1720 	int ret;
1721 
1722 	cmd = DWC3_DEPCMD_ENDTRANSFER;
1723 	cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1724 	cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1725 	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1726 	memset(&params, 0, sizeof(params));
1727 	ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1728 	/*
1729 	 * If the End Transfer command was timed out while the device is
1730 	 * not in SETUP phase, it's possible that an incoming Setup packet
1731 	 * may prevent the command's completion. Let's retry when the
1732 	 * ep0state returns to EP0_SETUP_PHASE.
1733 	 */
1734 	if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
1735 		dep->flags |= DWC3_EP_DELAY_STOP;
1736 		return 0;
1737 	}
1738 	WARN_ON_ONCE(ret);
1739 	dep->resource_index = 0;
1740 
1741 	if (!interrupt)
1742 		dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1743 	else if (!ret)
1744 		dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1745 
1746 	dep->flags &= ~DWC3_EP_DELAY_STOP;
1747 	return ret;
1748 }
1749 
1750 /**
1751  * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1752  * @dep: isoc endpoint
1753  *
1754  * This function tests for the correct combination of BIT[15:14] from the 16-bit
1755  * microframe number reported by the XferNotReady event for the future frame
1756  * number to start the isoc transfer.
1757  *
1758  * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1759  * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1760  * XferNotReady event are invalid. The driver uses this number to schedule the
1761  * isochronous transfer and passes it to the START TRANSFER command. Because
1762  * this number is invalid, the command may fail. If BIT[15:14] matches the
1763  * internal 16-bit microframe, the START TRANSFER command will pass and the
1764  * transfer will start at the scheduled time, if it is off by 1, the command
1765  * will still pass, but the transfer will start 2 seconds in the future. For all
1766  * other conditions, the START TRANSFER command will fail with bus-expiry.
1767  *
1768  * In order to workaround this issue, we can test for the correct combination of
1769  * BIT[15:14] by sending START TRANSFER commands with different values of
1770  * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1771  * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1772  * As the result, within the 4 possible combinations for BIT[15:14], there will
1773  * be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1774  * command status will result in a 2-second delay start. The smaller BIT[15:14]
1775  * value is the correct combination.
1776  *
1777  * Since there are only 4 outcomes and the results are ordered, we can simply
1778  * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1779  * deduce the smaller successful combination.
1780  *
1781  * Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1782  * of BIT[15:14]. The correct combination is as follow:
1783  *
1784  * if test0 fails and test1 passes, BIT[15:14] is 'b01
1785  * if test0 fails and test1 fails, BIT[15:14] is 'b10
1786  * if test0 passes and test1 fails, BIT[15:14] is 'b11
1787  * if test0 passes and test1 passes, BIT[15:14] is 'b00
1788  *
1789  * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1790  * endpoints.
1791  */
dwc3_gadget_start_isoc_quirk(struct dwc3_ep * dep)1792 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1793 {
1794 	int cmd_status = 0;
1795 	bool test0;
1796 	bool test1;
1797 
1798 	while (dep->combo_num < 2) {
1799 		struct dwc3_gadget_ep_cmd_params params;
1800 		u32 test_frame_number;
1801 		u32 cmd;
1802 
1803 		/*
1804 		 * Check if we can start isoc transfer on the next interval or
1805 		 * 4 uframes in the future with BIT[15:14] as dep->combo_num
1806 		 */
1807 		test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1808 		test_frame_number |= dep->combo_num << 14;
1809 		test_frame_number += max_t(u32, 4, dep->interval);
1810 
1811 		params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1812 		params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1813 
1814 		cmd = DWC3_DEPCMD_STARTTRANSFER;
1815 		cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1816 		cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1817 
1818 		/* Redo if some other failure beside bus-expiry is received */
1819 		if (cmd_status && cmd_status != -EAGAIN) {
1820 			dep->start_cmd_status = 0;
1821 			dep->combo_num = 0;
1822 			return 0;
1823 		}
1824 
1825 		/* Store the first test status */
1826 		if (dep->combo_num == 0)
1827 			dep->start_cmd_status = cmd_status;
1828 
1829 		dep->combo_num++;
1830 
1831 		/*
1832 		 * End the transfer if the START_TRANSFER command is successful
1833 		 * to wait for the next XferNotReady to test the command again
1834 		 */
1835 		if (cmd_status == 0) {
1836 			dwc3_stop_active_transfer(dep, true, true);
1837 			return 0;
1838 		}
1839 	}
1840 
1841 	/* test0 and test1 are both completed at this point */
1842 	test0 = (dep->start_cmd_status == 0);
1843 	test1 = (cmd_status == 0);
1844 
1845 	if (!test0 && test1)
1846 		dep->combo_num = 1;
1847 	else if (!test0 && !test1)
1848 		dep->combo_num = 2;
1849 	else if (test0 && !test1)
1850 		dep->combo_num = 3;
1851 	else if (test0 && test1)
1852 		dep->combo_num = 0;
1853 
1854 	dep->frame_number &= DWC3_FRNUMBER_MASK;
1855 	dep->frame_number |= dep->combo_num << 14;
1856 	dep->frame_number += max_t(u32, 4, dep->interval);
1857 
1858 	/* Reinitialize test variables */
1859 	dep->start_cmd_status = 0;
1860 	dep->combo_num = 0;
1861 
1862 	return __dwc3_gadget_kick_transfer(dep);
1863 }
1864 
__dwc3_gadget_start_isoc(struct dwc3_ep * dep)1865 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1866 {
1867 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1868 	struct dwc3 *dwc = dep->dwc;
1869 	int ret;
1870 	int i;
1871 
1872 	if (list_empty(&dep->pending_list) &&
1873 	    list_empty(&dep->started_list)) {
1874 		dep->flags |= DWC3_EP_PENDING_REQUEST;
1875 		return -EAGAIN;
1876 	}
1877 
1878 	if (!dwc->dis_start_transfer_quirk &&
1879 	    (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1880 	     DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1881 		if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1882 			return dwc3_gadget_start_isoc_quirk(dep);
1883 	}
1884 
1885 	if (desc->bInterval <= 14 &&
1886 	    dwc->gadget->speed >= USB_SPEED_HIGH) {
1887 		u32 frame = __dwc3_gadget_get_frame(dwc);
1888 		bool rollover = frame <
1889 				(dep->frame_number & DWC3_FRNUMBER_MASK);
1890 
1891 		/*
1892 		 * frame_number is set from XferNotReady and may be already
1893 		 * out of date. DSTS only provides the lower 14 bit of the
1894 		 * current frame number. So add the upper two bits of
1895 		 * frame_number and handle a possible rollover.
1896 		 * This will provide the correct frame_number unless more than
1897 		 * rollover has happened since XferNotReady.
1898 		 */
1899 
1900 		dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1901 				     frame;
1902 		if (rollover)
1903 			dep->frame_number += BIT(14);
1904 	}
1905 
1906 	for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1907 		int future_interval = i + 1;
1908 
1909 		/* Give the controller at least 500us to schedule transfers */
1910 		if (desc->bInterval < 3)
1911 			future_interval += 3 - desc->bInterval;
1912 
1913 		dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1914 
1915 		ret = __dwc3_gadget_kick_transfer(dep);
1916 		if (ret != -EAGAIN)
1917 			break;
1918 	}
1919 
1920 	/*
1921 	 * After a number of unsuccessful start attempts due to bus-expiry
1922 	 * status, issue END_TRANSFER command and retry on the next XferNotReady
1923 	 * event.
1924 	 */
1925 	if (ret == -EAGAIN)
1926 		ret = __dwc3_stop_active_transfer(dep, false, true);
1927 
1928 	return ret;
1929 }
1930 
__dwc3_gadget_ep_queue(struct dwc3_ep * dep,struct dwc3_request * req)1931 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1932 {
1933 	struct dwc3		*dwc = dep->dwc;
1934 
1935 	if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1936 		dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1937 				dep->name);
1938 		return -ESHUTDOWN;
1939 	}
1940 
1941 	if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1942 				&req->request, req->dep->name))
1943 		return -EINVAL;
1944 
1945 	if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1946 				"%s: request %pK already in flight\n",
1947 				dep->name, &req->request))
1948 		return -EINVAL;
1949 
1950 	pm_runtime_get(dwc->dev);
1951 
1952 	req->request.actual	= 0;
1953 	req->request.status	= -EINPROGRESS;
1954 
1955 	trace_dwc3_ep_queue(req);
1956 
1957 	list_add_tail(&req->list, &dep->pending_list);
1958 	req->status = DWC3_REQUEST_STATUS_QUEUED;
1959 
1960 	if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1961 		return 0;
1962 
1963 	/*
1964 	 * Start the transfer only after the END_TRANSFER is completed
1965 	 * and endpoint STALL is cleared.
1966 	 */
1967 	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1968 	    (dep->flags & DWC3_EP_WEDGE) ||
1969 	    (dep->flags & DWC3_EP_DELAY_STOP) ||
1970 	    (dep->flags & DWC3_EP_STALL)) {
1971 		dep->flags |= DWC3_EP_DELAY_START;
1972 		return 0;
1973 	}
1974 
1975 	/*
1976 	 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1977 	 * wait for a XferNotReady event so we will know what's the current
1978 	 * (micro-)frame number.
1979 	 *
1980 	 * Without this trick, we are very, very likely gonna get Bus Expiry
1981 	 * errors which will force us issue EndTransfer command.
1982 	 */
1983 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1984 		if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1985 			if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1986 				return __dwc3_gadget_start_isoc(dep);
1987 
1988 			return 0;
1989 		}
1990 	}
1991 
1992 	__dwc3_gadget_kick_transfer(dep);
1993 
1994 	return 0;
1995 }
1996 
dwc3_gadget_ep_queue(struct usb_ep * ep,struct usb_request * request,gfp_t gfp_flags)1997 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1998 	gfp_t gfp_flags)
1999 {
2000 	struct dwc3_request		*req = to_dwc3_request(request);
2001 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2002 	struct dwc3			*dwc = dep->dwc;
2003 
2004 	unsigned long			flags;
2005 
2006 	int				ret;
2007 
2008 	spin_lock_irqsave(&dwc->lock, flags);
2009 	ret = __dwc3_gadget_ep_queue(dep, req);
2010 	spin_unlock_irqrestore(&dwc->lock, flags);
2011 
2012 	return ret;
2013 }
2014 
dwc3_gadget_ep_skip_trbs(struct dwc3_ep * dep,struct dwc3_request * req)2015 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
2016 {
2017 	int i;
2018 
2019 	/* If req->trb is not set, then the request has not started */
2020 	if (!req->trb)
2021 		return;
2022 
2023 	/*
2024 	 * If request was already started, this means we had to
2025 	 * stop the transfer. With that we also need to ignore
2026 	 * all TRBs used by the request, however TRBs can only
2027 	 * be modified after completion of END_TRANSFER
2028 	 * command. So what we do here is that we wait for
2029 	 * END_TRANSFER completion and only after that, we jump
2030 	 * over TRBs by clearing HWO and incrementing dequeue
2031 	 * pointer.
2032 	 */
2033 	for (i = 0; i < req->num_trbs; i++) {
2034 		struct dwc3_trb *trb;
2035 
2036 		trb = &dep->trb_pool[dep->trb_dequeue];
2037 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2038 		dwc3_ep_inc_deq(dep);
2039 	}
2040 
2041 	req->num_trbs = 0;
2042 }
2043 
dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep * dep)2044 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2045 {
2046 	struct dwc3_request		*req;
2047 	struct dwc3			*dwc = dep->dwc;
2048 
2049 	while (!list_empty(&dep->cancelled_list)) {
2050 		req = next_request(&dep->cancelled_list);
2051 		dwc3_gadget_ep_skip_trbs(dep, req);
2052 		switch (req->status) {
2053 		case DWC3_REQUEST_STATUS_DISCONNECTED:
2054 			dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
2055 			break;
2056 		case DWC3_REQUEST_STATUS_DEQUEUED:
2057 			dwc3_gadget_giveback(dep, req, -ECONNRESET);
2058 			break;
2059 		case DWC3_REQUEST_STATUS_STALLED:
2060 			dwc3_gadget_giveback(dep, req, -EPIPE);
2061 			break;
2062 		default:
2063 			dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2064 			dwc3_gadget_giveback(dep, req, -ECONNRESET);
2065 			break;
2066 		}
2067 		/*
2068 		 * The endpoint is disabled, let the dwc3_remove_requests()
2069 		 * handle the cleanup.
2070 		 */
2071 		if (!dep->endpoint.desc)
2072 			break;
2073 	}
2074 }
2075 
dwc3_gadget_ep_dequeue(struct usb_ep * ep,struct usb_request * request)2076 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2077 		struct usb_request *request)
2078 {
2079 	struct dwc3_request		*req = to_dwc3_request(request);
2080 	struct dwc3_request		*r = NULL;
2081 
2082 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2083 	struct dwc3			*dwc = dep->dwc;
2084 
2085 	unsigned long			flags;
2086 	int				ret = 0;
2087 
2088 	trace_dwc3_ep_dequeue(req);
2089 
2090 	spin_lock_irqsave(&dwc->lock, flags);
2091 
2092 	list_for_each_entry(r, &dep->cancelled_list, list) {
2093 		if (r == req)
2094 			goto out;
2095 	}
2096 
2097 	list_for_each_entry(r, &dep->pending_list, list) {
2098 		if (r == req) {
2099 			/*
2100 			 * Explicitly check for EP0/1 as dequeue for those
2101 			 * EPs need to be handled differently.  Control EP
2102 			 * only deals with one USB req, and giveback will
2103 			 * occur during dwc3_ep0_stall_and_restart().  EP0
2104 			 * requests are never added to started_list.
2105 			 */
2106 			if (dep->number > 1)
2107 				dwc3_gadget_giveback(dep, req, -ECONNRESET);
2108 			else
2109 				dwc3_ep0_reset_state(dwc);
2110 			goto out;
2111 		}
2112 	}
2113 
2114 	list_for_each_entry(r, &dep->started_list, list) {
2115 		if (r == req) {
2116 			struct dwc3_request *t;
2117 
2118 			/* wait until it is processed */
2119 			dwc3_stop_active_transfer(dep, true, true);
2120 
2121 			/*
2122 			 * Remove any started request if the transfer is
2123 			 * cancelled.
2124 			 */
2125 			list_for_each_entry_safe(r, t, &dep->started_list, list)
2126 				dwc3_gadget_move_cancelled_request(r,
2127 						DWC3_REQUEST_STATUS_DEQUEUED);
2128 
2129 			dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2130 
2131 			goto out;
2132 		}
2133 	}
2134 
2135 	dev_err(dwc->dev, "request %pK was not queued to %s\n",
2136 		request, ep->name);
2137 	ret = -EINVAL;
2138 out:
2139 	spin_unlock_irqrestore(&dwc->lock, flags);
2140 
2141 	return ret;
2142 }
2143 
__dwc3_gadget_ep_set_halt(struct dwc3_ep * dep,int value,int protocol)2144 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2145 {
2146 	struct dwc3_gadget_ep_cmd_params	params;
2147 	struct dwc3				*dwc = dep->dwc;
2148 	struct dwc3_request			*req;
2149 	struct dwc3_request			*tmp;
2150 	int					ret;
2151 
2152 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2153 		dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2154 		return -EINVAL;
2155 	}
2156 
2157 	memset(&params, 0x00, sizeof(params));
2158 
2159 	if (value) {
2160 		struct dwc3_trb *trb;
2161 
2162 		unsigned int transfer_in_flight;
2163 		unsigned int started;
2164 
2165 		if (dep->number > 1)
2166 			trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
2167 		else
2168 			trb = &dwc->ep0_trb[dep->trb_enqueue];
2169 
2170 		transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2171 		started = !list_empty(&dep->started_list);
2172 
2173 		if (!protocol && ((dep->direction && transfer_in_flight) ||
2174 				(!dep->direction && started))) {
2175 			return -EAGAIN;
2176 		}
2177 
2178 		ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2179 				&params);
2180 		if (ret)
2181 			dev_err(dwc->dev, "failed to set STALL on %s\n",
2182 					dep->name);
2183 		else
2184 			dep->flags |= DWC3_EP_STALL;
2185 	} else {
2186 		/*
2187 		 * Don't issue CLEAR_STALL command to control endpoints. The
2188 		 * controller automatically clears the STALL when it receives
2189 		 * the SETUP token.
2190 		 */
2191 		if (dep->number <= 1) {
2192 			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2193 			return 0;
2194 		}
2195 
2196 		dwc3_stop_active_transfer(dep, true, true);
2197 
2198 		list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2199 			dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2200 
2201 		if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2202 		    (dep->flags & DWC3_EP_DELAY_STOP)) {
2203 			dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2204 			if (protocol)
2205 				dwc->clear_stall_protocol = dep->number;
2206 
2207 			return 0;
2208 		}
2209 
2210 		dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2211 
2212 		ret = dwc3_send_clear_stall_ep_cmd(dep);
2213 		if (ret) {
2214 			dev_err(dwc->dev, "failed to clear STALL on %s\n",
2215 					dep->name);
2216 			return ret;
2217 		}
2218 
2219 		dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2220 
2221 		if ((dep->flags & DWC3_EP_DELAY_START) &&
2222 		    !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2223 			__dwc3_gadget_kick_transfer(dep);
2224 
2225 		dep->flags &= ~DWC3_EP_DELAY_START;
2226 	}
2227 
2228 	return ret;
2229 }
2230 
dwc3_gadget_ep_set_halt(struct usb_ep * ep,int value)2231 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2232 {
2233 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2234 	struct dwc3			*dwc = dep->dwc;
2235 
2236 	unsigned long			flags;
2237 
2238 	int				ret;
2239 
2240 	spin_lock_irqsave(&dwc->lock, flags);
2241 	ret = __dwc3_gadget_ep_set_halt(dep, value, false);
2242 	spin_unlock_irqrestore(&dwc->lock, flags);
2243 
2244 	return ret;
2245 }
2246 
dwc3_gadget_ep_set_wedge(struct usb_ep * ep)2247 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2248 {
2249 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
2250 	struct dwc3			*dwc = dep->dwc;
2251 	unsigned long			flags;
2252 	int				ret;
2253 
2254 	spin_lock_irqsave(&dwc->lock, flags);
2255 	dep->flags |= DWC3_EP_WEDGE;
2256 
2257 	if (dep->number == 0 || dep->number == 1)
2258 		ret = __dwc3_gadget_ep0_set_halt(ep, 1);
2259 	else
2260 		ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2261 	spin_unlock_irqrestore(&dwc->lock, flags);
2262 
2263 	return ret;
2264 }
2265 
2266 /* -------------------------------------------------------------------------- */
2267 
2268 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2269 	.bLength	= USB_DT_ENDPOINT_SIZE,
2270 	.bDescriptorType = USB_DT_ENDPOINT,
2271 	.bmAttributes	= USB_ENDPOINT_XFER_CONTROL,
2272 };
2273 
2274 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2275 	.enable		= dwc3_gadget_ep0_enable,
2276 	.disable	= dwc3_gadget_ep0_disable,
2277 	.alloc_request	= dwc3_gadget_ep_alloc_request,
2278 	.free_request	= dwc3_gadget_ep_free_request,
2279 	.queue		= dwc3_gadget_ep0_queue,
2280 	.dequeue	= dwc3_gadget_ep_dequeue,
2281 	.set_halt	= dwc3_gadget_ep0_set_halt,
2282 	.set_wedge	= dwc3_gadget_ep_set_wedge,
2283 };
2284 
2285 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2286 	.enable		= dwc3_gadget_ep_enable,
2287 	.disable	= dwc3_gadget_ep_disable,
2288 	.alloc_request	= dwc3_gadget_ep_alloc_request,
2289 	.free_request	= dwc3_gadget_ep_free_request,
2290 	.queue		= dwc3_gadget_ep_queue,
2291 	.dequeue	= dwc3_gadget_ep_dequeue,
2292 	.set_halt	= dwc3_gadget_ep_set_halt,
2293 	.set_wedge	= dwc3_gadget_ep_set_wedge,
2294 };
2295 
2296 /* -------------------------------------------------------------------------- */
2297 
dwc3_gadget_enable_linksts_evts(struct dwc3 * dwc,bool set)2298 static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set)
2299 {
2300 	u32 reg;
2301 
2302 	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2303 		return;
2304 
2305 	reg = dwc3_readl(dwc->regs, DWC3_DEVTEN);
2306 	if (set)
2307 		reg |= DWC3_DEVTEN_ULSTCNGEN;
2308 	else
2309 		reg &= ~DWC3_DEVTEN_ULSTCNGEN;
2310 
2311 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2312 }
2313 
dwc3_gadget_get_frame(struct usb_gadget * g)2314 static int dwc3_gadget_get_frame(struct usb_gadget *g)
2315 {
2316 	struct dwc3		*dwc = gadget_to_dwc(g);
2317 
2318 	return __dwc3_gadget_get_frame(dwc);
2319 }
2320 
__dwc3_gadget_wakeup(struct dwc3 * dwc,bool async)2321 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async)
2322 {
2323 	int			retries;
2324 
2325 	int			ret;
2326 	u32			reg;
2327 
2328 	u8			link_state;
2329 
2330 	/*
2331 	 * According to the Databook Remote wakeup request should
2332 	 * be issued only when the device is in early suspend state.
2333 	 *
2334 	 * We can check that via USB Link State bits in DSTS register.
2335 	 */
2336 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2337 
2338 	link_state = DWC3_DSTS_USBLNKST(reg);
2339 
2340 	switch (link_state) {
2341 	case DWC3_LINK_STATE_RESET:
2342 	case DWC3_LINK_STATE_RX_DET:	/* in HS, means Early Suspend */
2343 	case DWC3_LINK_STATE_U3:	/* in HS, means SUSPEND */
2344 	case DWC3_LINK_STATE_U2:	/* in HS, means Sleep (L1) */
2345 	case DWC3_LINK_STATE_U1:
2346 	case DWC3_LINK_STATE_RESUME:
2347 		break;
2348 	default:
2349 		return -EINVAL;
2350 	}
2351 
2352 	if (async)
2353 		dwc3_gadget_enable_linksts_evts(dwc, true);
2354 
2355 	ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2356 	if (ret < 0) {
2357 		dev_err(dwc->dev, "failed to put link in Recovery\n");
2358 		dwc3_gadget_enable_linksts_evts(dwc, false);
2359 		return ret;
2360 	}
2361 
2362 	/* Recent versions do this automatically */
2363 	if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2364 		/* write zeroes to Link Change Request */
2365 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2366 		reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2367 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2368 	}
2369 
2370 	/*
2371 	 * Since link status change events are enabled we will receive
2372 	 * an U0 event when wakeup is successful. So bail out.
2373 	 */
2374 	if (async)
2375 		return 0;
2376 
2377 	/* poll until Link State changes to ON */
2378 	retries = 20000;
2379 
2380 	while (retries--) {
2381 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2382 
2383 		/* in HS, means ON */
2384 		if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2385 			break;
2386 	}
2387 
2388 	if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2389 		dev_err(dwc->dev, "failed to send remote wakeup\n");
2390 		return -EINVAL;
2391 	}
2392 
2393 	return 0;
2394 }
2395 
dwc3_gadget_wakeup(struct usb_gadget * g)2396 static int dwc3_gadget_wakeup(struct usb_gadget *g)
2397 {
2398 	struct dwc3		*dwc = gadget_to_dwc(g);
2399 	unsigned long		flags;
2400 	int			ret;
2401 
2402 	if (!dwc->wakeup_configured) {
2403 		dev_err(dwc->dev, "remote wakeup not configured\n");
2404 		return -EINVAL;
2405 	}
2406 
2407 	spin_lock_irqsave(&dwc->lock, flags);
2408 	if (!dwc->gadget->wakeup_armed) {
2409 		dev_err(dwc->dev, "not armed for remote wakeup\n");
2410 		spin_unlock_irqrestore(&dwc->lock, flags);
2411 		return -EINVAL;
2412 	}
2413 	ret = __dwc3_gadget_wakeup(dwc, true);
2414 
2415 	spin_unlock_irqrestore(&dwc->lock, flags);
2416 
2417 	return ret;
2418 }
2419 
2420 static void dwc3_resume_gadget(struct dwc3 *dwc);
2421 
dwc3_gadget_func_wakeup(struct usb_gadget * g,int intf_id)2422 static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id)
2423 {
2424 	struct  dwc3		*dwc = gadget_to_dwc(g);
2425 	unsigned long		flags;
2426 	int			ret;
2427 	int			link_state;
2428 
2429 	if (!dwc->wakeup_configured) {
2430 		dev_err(dwc->dev, "remote wakeup not configured\n");
2431 		return -EINVAL;
2432 	}
2433 
2434 	spin_lock_irqsave(&dwc->lock, flags);
2435 	/*
2436 	 * If the link is in U3, signal for remote wakeup and wait for the
2437 	 * link to transition to U0 before sending device notification.
2438 	 */
2439 	link_state = dwc3_gadget_get_link_state(dwc);
2440 	if (link_state == DWC3_LINK_STATE_U3) {
2441 		ret = __dwc3_gadget_wakeup(dwc, false);
2442 		if (ret) {
2443 			spin_unlock_irqrestore(&dwc->lock, flags);
2444 			return -EINVAL;
2445 		}
2446 		dwc3_resume_gadget(dwc);
2447 		dwc->suspended = false;
2448 		dwc->link_state = DWC3_LINK_STATE_U0;
2449 	}
2450 
2451 	ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
2452 					       DWC3_DGCMDPAR_DN_FUNC_WAKE |
2453 					       DWC3_DGCMDPAR_INTF_SEL(intf_id));
2454 	if (ret)
2455 		dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);
2456 
2457 	spin_unlock_irqrestore(&dwc->lock, flags);
2458 
2459 	return ret;
2460 }
2461 
dwc3_gadget_set_remote_wakeup(struct usb_gadget * g,int set)2462 static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set)
2463 {
2464 	struct dwc3		*dwc = gadget_to_dwc(g);
2465 	unsigned long		flags;
2466 
2467 	spin_lock_irqsave(&dwc->lock, flags);
2468 	dwc->wakeup_configured = !!set;
2469 	spin_unlock_irqrestore(&dwc->lock, flags);
2470 
2471 	return 0;
2472 }
2473 
dwc3_gadget_set_selfpowered(struct usb_gadget * g,int is_selfpowered)2474 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2475 		int is_selfpowered)
2476 {
2477 	struct dwc3		*dwc = gadget_to_dwc(g);
2478 	unsigned long		flags;
2479 
2480 	spin_lock_irqsave(&dwc->lock, flags);
2481 	g->is_selfpowered = !!is_selfpowered;
2482 	spin_unlock_irqrestore(&dwc->lock, flags);
2483 
2484 	return 0;
2485 }
2486 
dwc3_stop_active_transfers(struct dwc3 * dwc)2487 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2488 {
2489 	u32 epnum;
2490 
2491 	for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2492 		struct dwc3_ep *dep;
2493 
2494 		dep = dwc->eps[epnum];
2495 		if (!dep)
2496 			continue;
2497 
2498 		dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
2499 	}
2500 }
2501 
__dwc3_gadget_set_ssp_rate(struct dwc3 * dwc)2502 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2503 {
2504 	enum usb_ssp_rate	ssp_rate = dwc->gadget_ssp_rate;
2505 	u32			reg;
2506 
2507 	if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2508 		ssp_rate = dwc->max_ssp_rate;
2509 
2510 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2511 	reg &= ~DWC3_DCFG_SPEED_MASK;
2512 	reg &= ~DWC3_DCFG_NUMLANES(~0);
2513 
2514 	if (ssp_rate == USB_SSP_GEN_1x2)
2515 		reg |= DWC3_DCFG_SUPERSPEED;
2516 	else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2517 		reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2518 
2519 	if (ssp_rate != USB_SSP_GEN_2x1 &&
2520 	    dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2521 		reg |= DWC3_DCFG_NUMLANES(1);
2522 
2523 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2524 }
2525 
__dwc3_gadget_set_speed(struct dwc3 * dwc)2526 static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2527 {
2528 	enum usb_device_speed	speed;
2529 	u32			reg;
2530 
2531 	speed = dwc->gadget_max_speed;
2532 	if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2533 		speed = dwc->maximum_speed;
2534 
2535 	if (speed == USB_SPEED_SUPER_PLUS &&
2536 	    DWC3_IP_IS(DWC32)) {
2537 		__dwc3_gadget_set_ssp_rate(dwc);
2538 		return;
2539 	}
2540 
2541 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2542 	reg &= ~(DWC3_DCFG_SPEED_MASK);
2543 
2544 	/*
2545 	 * WORKAROUND: DWC3 revision < 2.20a have an issue
2546 	 * which would cause metastability state on Run/Stop
2547 	 * bit if we try to force the IP to USB2-only mode.
2548 	 *
2549 	 * Because of that, we cannot configure the IP to any
2550 	 * speed other than the SuperSpeed
2551 	 *
2552 	 * Refers to:
2553 	 *
2554 	 * STAR#9000525659: Clock Domain Crossing on DCTL in
2555 	 * USB 2.0 Mode
2556 	 */
2557 	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2558 	    !dwc->dis_metastability_quirk) {
2559 		reg |= DWC3_DCFG_SUPERSPEED;
2560 	} else {
2561 		switch (speed) {
2562 		case USB_SPEED_FULL:
2563 			reg |= DWC3_DCFG_FULLSPEED;
2564 			break;
2565 		case USB_SPEED_HIGH:
2566 			reg |= DWC3_DCFG_HIGHSPEED;
2567 			break;
2568 		case USB_SPEED_SUPER:
2569 			reg |= DWC3_DCFG_SUPERSPEED;
2570 			break;
2571 		case USB_SPEED_SUPER_PLUS:
2572 			if (DWC3_IP_IS(DWC3))
2573 				reg |= DWC3_DCFG_SUPERSPEED;
2574 			else
2575 				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2576 			break;
2577 		default:
2578 			dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2579 
2580 			if (DWC3_IP_IS(DWC3))
2581 				reg |= DWC3_DCFG_SUPERSPEED;
2582 			else
2583 				reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2584 		}
2585 	}
2586 
2587 	if (DWC3_IP_IS(DWC32) &&
2588 	    speed > USB_SPEED_UNKNOWN &&
2589 	    speed < USB_SPEED_SUPER_PLUS)
2590 		reg &= ~DWC3_DCFG_NUMLANES(~0);
2591 
2592 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2593 }
2594 
dwc3_gadget_run_stop(struct dwc3 * dwc,int is_on)2595 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
2596 {
2597 	u32			reg;
2598 	u32			timeout = 2000;
2599 
2600 	if (pm_runtime_suspended(dwc->dev))
2601 		return 0;
2602 
2603 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2604 	if (is_on) {
2605 		if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2606 			reg &= ~DWC3_DCTL_TRGTULST_MASK;
2607 			reg |= DWC3_DCTL_TRGTULST_RX_DET;
2608 		}
2609 
2610 		if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2611 			reg &= ~DWC3_DCTL_KEEP_CONNECT;
2612 		reg |= DWC3_DCTL_RUN_STOP;
2613 
2614 		__dwc3_gadget_set_speed(dwc);
2615 		dwc->pullups_connected = true;
2616 	} else {
2617 		reg &= ~DWC3_DCTL_RUN_STOP;
2618 
2619 		dwc->pullups_connected = false;
2620 	}
2621 
2622 	dwc3_gadget_dctl_write_safe(dwc, reg);
2623 
2624 	do {
2625 		usleep_range(1000, 2000);
2626 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2627 		reg &= DWC3_DSTS_DEVCTRLHLT;
2628 	} while (--timeout && !(!is_on ^ !reg));
2629 
2630 	if (!timeout)
2631 		return -ETIMEDOUT;
2632 
2633 	return 0;
2634 }
2635 
2636 static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2637 static void __dwc3_gadget_stop(struct dwc3 *dwc);
2638 static int __dwc3_gadget_start(struct dwc3 *dwc);
2639 
dwc3_gadget_soft_disconnect(struct dwc3 * dwc)2640 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
2641 {
2642 	unsigned long flags;
2643 	int ret;
2644 
2645 	spin_lock_irqsave(&dwc->lock, flags);
2646 	if (!dwc->pullups_connected) {
2647 		spin_unlock_irqrestore(&dwc->lock, flags);
2648 		return 0;
2649 	}
2650 
2651 	dwc->connected = false;
2652 
2653 	/*
2654 	 * Attempt to end pending SETUP status phase, and not wait for the
2655 	 * function to do so.
2656 	 */
2657 	if (dwc->delayed_status)
2658 		dwc3_ep0_send_delayed_status(dwc);
2659 
2660 	/*
2661 	 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
2662 	 * Section 4.1.8 Table 4-7, it states that for a device-initiated
2663 	 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2664 	 * command for any active transfers" before clearing the RunStop
2665 	 * bit.
2666 	 */
2667 	dwc3_stop_active_transfers(dwc);
2668 	spin_unlock_irqrestore(&dwc->lock, flags);
2669 
2670 	/*
2671 	 * Per databook, when we want to stop the gadget, if a control transfer
2672 	 * is still in process, complete it and get the core into setup phase.
2673 	 * In case the host is unresponsive to a SETUP transaction, forcefully
2674 	 * stall the transfer, and move back to the SETUP phase, so that any
2675 	 * pending endxfers can be executed.
2676 	 */
2677 	if (dwc->ep0state != EP0_SETUP_PHASE) {
2678 		reinit_completion(&dwc->ep0_in_setup);
2679 
2680 		ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2681 				msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2682 		if (ret == 0) {
2683 			dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
2684 			spin_lock_irqsave(&dwc->lock, flags);
2685 			dwc3_ep0_reset_state(dwc);
2686 			spin_unlock_irqrestore(&dwc->lock, flags);
2687 		}
2688 	}
2689 
2690 	/*
2691 	 * Note: if the GEVNTCOUNT indicates events in the event buffer, the
2692 	 * driver needs to acknowledge them before the controller can halt.
2693 	 * Simply let the interrupt handler acknowledges and handle the
2694 	 * remaining event generated by the controller while polling for
2695 	 * DSTS.DEVCTLHLT.
2696 	 */
2697 	ret = dwc3_gadget_run_stop(dwc, false);
2698 
2699 	/*
2700 	 * Stop the gadget after controller is halted, so that if needed, the
2701 	 * events to update EP0 state can still occur while the run/stop
2702 	 * routine polls for the halted state.  DEVTEN is cleared as part of
2703 	 * gadget stop.
2704 	 */
2705 	spin_lock_irqsave(&dwc->lock, flags);
2706 	__dwc3_gadget_stop(dwc);
2707 	spin_unlock_irqrestore(&dwc->lock, flags);
2708 
2709 	return ret;
2710 }
2711 
dwc3_gadget_soft_connect(struct dwc3 * dwc)2712 static int dwc3_gadget_soft_connect(struct dwc3 *dwc)
2713 {
2714 	int ret;
2715 
2716 	/*
2717 	 * In the Synopsys DWC_usb31 1.90a programming guide section
2718 	 * 4.1.9, it specifies that for a reconnect after a
2719 	 * device-initiated disconnect requires a core soft reset
2720 	 * (DCTL.CSftRst) before enabling the run/stop bit.
2721 	 */
2722 	ret = dwc3_core_soft_reset(dwc);
2723 	if (ret)
2724 		return ret;
2725 
2726 	dwc3_event_buffers_setup(dwc);
2727 	__dwc3_gadget_start(dwc);
2728 	return dwc3_gadget_run_stop(dwc, true);
2729 }
2730 
dwc3_gadget_pullup(struct usb_gadget * g,int is_on)2731 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2732 {
2733 	struct dwc3		*dwc = gadget_to_dwc(g);
2734 	int			ret;
2735 
2736 	is_on = !!is_on;
2737 
2738 	dwc->softconnect = is_on;
2739 
2740 	/*
2741 	 * Avoid issuing a runtime resume if the device is already in the
2742 	 * suspended state during gadget disconnect.  DWC3 gadget was already
2743 	 * halted/stopped during runtime suspend.
2744 	 */
2745 	if (!is_on) {
2746 		pm_runtime_barrier(dwc->dev);
2747 		if (pm_runtime_suspended(dwc->dev))
2748 			return 0;
2749 	}
2750 
2751 	/*
2752 	 * Check the return value for successful resume, or error.  For a
2753 	 * successful resume, the DWC3 runtime PM resume routine will handle
2754 	 * the run stop sequence, so avoid duplicate operations here.
2755 	 */
2756 	ret = pm_runtime_get_sync(dwc->dev);
2757 	if (!ret || ret < 0) {
2758 		pm_runtime_put(dwc->dev);
2759 		if (ret < 0)
2760 			pm_runtime_set_suspended(dwc->dev);
2761 		return ret;
2762 	}
2763 
2764 	if (dwc->pullups_connected == is_on) {
2765 		pm_runtime_put(dwc->dev);
2766 		return 0;
2767 	}
2768 
2769 	synchronize_irq(dwc->irq_gadget);
2770 
2771 	if (!is_on)
2772 		ret = dwc3_gadget_soft_disconnect(dwc);
2773 	else
2774 		ret = dwc3_gadget_soft_connect(dwc);
2775 
2776 	pm_runtime_put(dwc->dev);
2777 
2778 	return ret;
2779 }
2780 
dwc3_gadget_enable_irq(struct dwc3 * dwc)2781 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2782 {
2783 	u32			reg;
2784 
2785 	/* Enable all but Start and End of Frame IRQs */
2786 	reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2787 			DWC3_DEVTEN_CMDCMPLTEN |
2788 			DWC3_DEVTEN_ERRTICERREN |
2789 			DWC3_DEVTEN_WKUPEVTEN |
2790 			DWC3_DEVTEN_CONNECTDONEEN |
2791 			DWC3_DEVTEN_USBRSTEN |
2792 			DWC3_DEVTEN_DISCONNEVTEN);
2793 
2794 	if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2795 		reg |= DWC3_DEVTEN_ULSTCNGEN;
2796 
2797 	/* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2798 	if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2799 		reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2800 
2801 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2802 }
2803 
dwc3_gadget_disable_irq(struct dwc3 * dwc)2804 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2805 {
2806 	/* mask all interrupts */
2807 	dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2808 }
2809 
2810 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2811 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2812 
2813 /**
2814  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2815  * @dwc: pointer to our context structure
2816  *
2817  * The following looks like complex but it's actually very simple. In order to
2818  * calculate the number of packets we can burst at once on OUT transfers, we're
2819  * gonna use RxFIFO size.
2820  *
2821  * To calculate RxFIFO size we need two numbers:
2822  * MDWIDTH = size, in bits, of the internal memory bus
2823  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2824  *
2825  * Given these two numbers, the formula is simple:
2826  *
2827  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2828  *
2829  * 24 bytes is for 3x SETUP packets
2830  * 16 bytes is a clock domain crossing tolerance
2831  *
2832  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2833  */
dwc3_gadget_setup_nump(struct dwc3 * dwc)2834 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2835 {
2836 	u32 ram2_depth;
2837 	u32 mdwidth;
2838 	u32 nump;
2839 	u32 reg;
2840 
2841 	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2842 	mdwidth = dwc3_mdwidth(dwc);
2843 
2844 	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2845 	nump = min_t(u32, nump, 16);
2846 
2847 	/* update NumP */
2848 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2849 	reg &= ~DWC3_DCFG_NUMP_MASK;
2850 	reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2851 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2852 }
2853 
__dwc3_gadget_start(struct dwc3 * dwc)2854 static int __dwc3_gadget_start(struct dwc3 *dwc)
2855 {
2856 	struct dwc3_ep		*dep;
2857 	int			ret = 0;
2858 	u32			reg;
2859 
2860 	/*
2861 	 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2862 	 * the core supports IMOD, disable it.
2863 	 */
2864 	if (dwc->imod_interval) {
2865 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2866 		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2867 	} else if (dwc3_has_imod(dwc)) {
2868 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2869 	}
2870 
2871 	/*
2872 	 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2873 	 * field instead of letting dwc3 itself calculate that automatically.
2874 	 *
2875 	 * This way, we maximize the chances that we'll be able to get several
2876 	 * bursts of data without going through any sort of endpoint throttling.
2877 	 */
2878 	reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2879 	if (DWC3_IP_IS(DWC3))
2880 		reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2881 	else
2882 		reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2883 
2884 	dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2885 
2886 	dwc3_gadget_setup_nump(dwc);
2887 
2888 	/*
2889 	 * Currently the controller handles single stream only. So, Ignore
2890 	 * Packet Pending bit for stream selection and don't search for another
2891 	 * stream if the host sends Data Packet with PP=0 (for OUT direction) or
2892 	 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2893 	 * the stream performance.
2894 	 */
2895 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2896 	reg |= DWC3_DCFG_IGNSTRMPP;
2897 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2898 
2899 	/* Enable MST by default if the device is capable of MST */
2900 	if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2901 		reg = dwc3_readl(dwc->regs, DWC3_DCFG1);
2902 		reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2903 		dwc3_writel(dwc->regs, DWC3_DCFG1, reg);
2904 	}
2905 
2906 	/* Start with SuperSpeed Default */
2907 	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2908 
2909 	dep = dwc->eps[0];
2910 	dep->flags = 0;
2911 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2912 	if (ret) {
2913 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2914 		goto err0;
2915 	}
2916 
2917 	dep = dwc->eps[1];
2918 	dep->flags = 0;
2919 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2920 	if (ret) {
2921 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2922 		goto err1;
2923 	}
2924 
2925 	/* begin to receive SETUP packets */
2926 	dwc->ep0state = EP0_SETUP_PHASE;
2927 	dwc->ep0_bounced = false;
2928 	dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2929 	dwc->delayed_status = false;
2930 	dwc3_ep0_out_start(dwc);
2931 
2932 	dwc3_gadget_enable_irq(dwc);
2933 	dwc3_enable_susphy(dwc, true);
2934 
2935 	return 0;
2936 
2937 err1:
2938 	__dwc3_gadget_ep_disable(dwc->eps[0]);
2939 
2940 err0:
2941 	return ret;
2942 }
2943 
dwc3_gadget_start(struct usb_gadget * g,struct usb_gadget_driver * driver)2944 static int dwc3_gadget_start(struct usb_gadget *g,
2945 		struct usb_gadget_driver *driver)
2946 {
2947 	struct dwc3		*dwc = gadget_to_dwc(g);
2948 	unsigned long		flags;
2949 	int			ret;
2950 	int			irq;
2951 
2952 	irq = dwc->irq_gadget;
2953 	ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2954 			IRQF_SHARED, "dwc3", dwc->ev_buf);
2955 	if (ret) {
2956 		dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2957 				irq, ret);
2958 		return ret;
2959 	}
2960 
2961 	spin_lock_irqsave(&dwc->lock, flags);
2962 	dwc->gadget_driver	= driver;
2963 	spin_unlock_irqrestore(&dwc->lock, flags);
2964 
2965 	if (dwc->sys_wakeup)
2966 		device_wakeup_enable(dwc->sysdev);
2967 
2968 	return 0;
2969 }
2970 
__dwc3_gadget_stop(struct dwc3 * dwc)2971 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2972 {
2973 	dwc3_gadget_disable_irq(dwc);
2974 	__dwc3_gadget_ep_disable(dwc->eps[0]);
2975 	__dwc3_gadget_ep_disable(dwc->eps[1]);
2976 }
2977 
dwc3_gadget_stop(struct usb_gadget * g)2978 static int dwc3_gadget_stop(struct usb_gadget *g)
2979 {
2980 	struct dwc3		*dwc = gadget_to_dwc(g);
2981 	unsigned long		flags;
2982 
2983 	if (dwc->sys_wakeup)
2984 		device_wakeup_disable(dwc->sysdev);
2985 
2986 	spin_lock_irqsave(&dwc->lock, flags);
2987 	dwc->gadget_driver	= NULL;
2988 	dwc->max_cfg_eps = 0;
2989 	spin_unlock_irqrestore(&dwc->lock, flags);
2990 
2991 	free_irq(dwc->irq_gadget, dwc->ev_buf);
2992 
2993 	return 0;
2994 }
2995 
dwc3_gadget_config_params(struct usb_gadget * g,struct usb_dcd_config_params * params)2996 static void dwc3_gadget_config_params(struct usb_gadget *g,
2997 				      struct usb_dcd_config_params *params)
2998 {
2999 	struct dwc3		*dwc = gadget_to_dwc(g);
3000 
3001 	params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
3002 	params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
3003 
3004 	/* Recommended BESL */
3005 	if (!dwc->dis_enblslpm_quirk) {
3006 		/*
3007 		 * If the recommended BESL baseline is 0 or if the BESL deep is
3008 		 * less than 2, Microsoft's Windows 10 host usb stack will issue
3009 		 * a usb reset immediately after it receives the extended BOS
3010 		 * descriptor and the enumeration will fail. To maintain
3011 		 * compatibility with the Windows' usb stack, let's set the
3012 		 * recommended BESL baseline to 1 and clamp the BESL deep to be
3013 		 * within 2 to 15.
3014 		 */
3015 		params->besl_baseline = 1;
3016 		if (dwc->is_utmi_l1_suspend)
3017 			params->besl_deep =
3018 				clamp_t(u8, dwc->hird_threshold, 2, 15);
3019 	}
3020 
3021 	/* U1 Device exit Latency */
3022 	if (dwc->dis_u1_entry_quirk)
3023 		params->bU1devExitLat = 0;
3024 	else
3025 		params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
3026 
3027 	/* U2 Device exit Latency */
3028 	if (dwc->dis_u2_entry_quirk)
3029 		params->bU2DevExitLat = 0;
3030 	else
3031 		params->bU2DevExitLat =
3032 				cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
3033 }
3034 
dwc3_gadget_set_speed(struct usb_gadget * g,enum usb_device_speed speed)3035 static void dwc3_gadget_set_speed(struct usb_gadget *g,
3036 				  enum usb_device_speed speed)
3037 {
3038 	struct dwc3		*dwc = gadget_to_dwc(g);
3039 	unsigned long		flags;
3040 
3041 	spin_lock_irqsave(&dwc->lock, flags);
3042 	dwc->gadget_max_speed = speed;
3043 	spin_unlock_irqrestore(&dwc->lock, flags);
3044 }
3045 
dwc3_gadget_set_ssp_rate(struct usb_gadget * g,enum usb_ssp_rate rate)3046 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
3047 				     enum usb_ssp_rate rate)
3048 {
3049 	struct dwc3		*dwc = gadget_to_dwc(g);
3050 	unsigned long		flags;
3051 
3052 	spin_lock_irqsave(&dwc->lock, flags);
3053 	dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
3054 	dwc->gadget_ssp_rate = rate;
3055 	spin_unlock_irqrestore(&dwc->lock, flags);
3056 }
3057 
dwc3_gadget_vbus_draw(struct usb_gadget * g,unsigned int mA)3058 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
3059 {
3060 	struct dwc3		*dwc = gadget_to_dwc(g);
3061 	union power_supply_propval	val = {0};
3062 	int				ret;
3063 
3064 	if (dwc->usb2_phy)
3065 		return usb_phy_set_power(dwc->usb2_phy, mA);
3066 
3067 	if (!dwc->usb_psy)
3068 		return -EOPNOTSUPP;
3069 
3070 	val.intval = 1000 * mA;
3071 	ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
3072 
3073 	return ret;
3074 }
3075 
3076 /**
3077  * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3078  * @g: pointer to the USB gadget
3079  *
3080  * Used to record the maximum number of endpoints being used in a USB composite
3081  * device. (across all configurations)  This is to be used in the calculation
3082  * of the TXFIFO sizes when resizing internal memory for individual endpoints.
3083  * It will help ensured that the resizing logic reserves enough space for at
3084  * least one max packet.
3085  */
dwc3_gadget_check_config(struct usb_gadget * g)3086 static int dwc3_gadget_check_config(struct usb_gadget *g)
3087 {
3088 	struct dwc3 *dwc = gadget_to_dwc(g);
3089 	struct usb_ep *ep;
3090 	int fifo_size = 0;
3091 	int ram1_depth;
3092 	int ep_num = 0;
3093 
3094 	if (!dwc->do_fifo_resize)
3095 		return 0;
3096 
3097 	list_for_each_entry(ep, &g->ep_list, ep_list) {
3098 		/* Only interested in the IN endpoints */
3099 		if (ep->claimed && (ep->address & USB_DIR_IN))
3100 			ep_num++;
3101 	}
3102 
3103 	if (ep_num <= dwc->max_cfg_eps)
3104 		return 0;
3105 
3106 	/* Update the max number of eps in the composition */
3107 	dwc->max_cfg_eps = ep_num;
3108 
3109 	fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
3110 	/* Based on the equation, increment by one for every ep */
3111 	fifo_size += dwc->max_cfg_eps;
3112 
3113 	/* Check if we can fit a single fifo per endpoint */
3114 	ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
3115 	if (fifo_size > ram1_depth)
3116 		return -ENOMEM;
3117 
3118 	return 0;
3119 }
3120 
dwc3_gadget_async_callbacks(struct usb_gadget * g,bool enable)3121 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
3122 {
3123 	struct dwc3		*dwc = gadget_to_dwc(g);
3124 	unsigned long		flags;
3125 
3126 	spin_lock_irqsave(&dwc->lock, flags);
3127 	dwc->async_callbacks = enable;
3128 	spin_unlock_irqrestore(&dwc->lock, flags);
3129 }
3130 
3131 static const struct usb_gadget_ops dwc3_gadget_ops = {
3132 	.get_frame		= dwc3_gadget_get_frame,
3133 	.wakeup			= dwc3_gadget_wakeup,
3134 	.func_wakeup		= dwc3_gadget_func_wakeup,
3135 	.set_remote_wakeup	= dwc3_gadget_set_remote_wakeup,
3136 	.set_selfpowered	= dwc3_gadget_set_selfpowered,
3137 	.pullup			= dwc3_gadget_pullup,
3138 	.udc_start		= dwc3_gadget_start,
3139 	.udc_stop		= dwc3_gadget_stop,
3140 	.udc_set_speed		= dwc3_gadget_set_speed,
3141 	.udc_set_ssp_rate	= dwc3_gadget_set_ssp_rate,
3142 	.get_config_params	= dwc3_gadget_config_params,
3143 	.vbus_draw		= dwc3_gadget_vbus_draw,
3144 	.check_config		= dwc3_gadget_check_config,
3145 	.udc_async_callbacks	= dwc3_gadget_async_callbacks,
3146 };
3147 
3148 /* -------------------------------------------------------------------------- */
3149 
dwc3_gadget_init_control_endpoint(struct dwc3_ep * dep)3150 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
3151 {
3152 	struct dwc3 *dwc = dep->dwc;
3153 
3154 	usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
3155 	dep->endpoint.maxburst = 1;
3156 	dep->endpoint.ops = &dwc3_gadget_ep0_ops;
3157 	if (!dep->direction)
3158 		dwc->gadget->ep0 = &dep->endpoint;
3159 
3160 	dep->endpoint.caps.type_control = true;
3161 
3162 	return 0;
3163 }
3164 
dwc3_gadget_init_in_endpoint(struct dwc3_ep * dep)3165 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
3166 {
3167 	struct dwc3 *dwc = dep->dwc;
3168 	u32 mdwidth;
3169 	int size;
3170 	int maxpacket;
3171 
3172 	mdwidth = dwc3_mdwidth(dwc);
3173 
3174 	/* MDWIDTH is represented in bits, we need it in bytes */
3175 	mdwidth /= 8;
3176 
3177 	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
3178 	if (DWC3_IP_IS(DWC3))
3179 		size = DWC3_GTXFIFOSIZ_TXFDEP(size);
3180 	else
3181 		size = DWC31_GTXFIFOSIZ_TXFDEP(size);
3182 
3183 	/*
3184 	 * maxpacket size is determined as part of the following, after assuming
3185 	 * a mult value of one maxpacket:
3186 	 * DWC3 revision 280A and prior:
3187 	 * fifo_size = mult * (max_packet / mdwidth) + 1;
3188 	 * maxpacket = mdwidth * (fifo_size - 1);
3189 	 *
3190 	 * DWC3 revision 290A and onwards:
3191 	 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
3192 	 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
3193 	 */
3194 	if (DWC3_VER_IS_PRIOR(DWC3, 290A))
3195 		maxpacket = mdwidth * (size - 1);
3196 	else
3197 		maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
3198 
3199 	/* Functionally, space for one max packet is sufficient */
3200 	size = min_t(int, maxpacket, 1024);
3201 	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3202 
3203 	dep->endpoint.max_streams = 16;
3204 	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3205 	list_add_tail(&dep->endpoint.ep_list,
3206 			&dwc->gadget->ep_list);
3207 	dep->endpoint.caps.type_iso = true;
3208 	dep->endpoint.caps.type_bulk = true;
3209 	dep->endpoint.caps.type_int = true;
3210 
3211 	return dwc3_alloc_trb_pool(dep);
3212 }
3213 
dwc3_gadget_init_out_endpoint(struct dwc3_ep * dep)3214 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3215 {
3216 	struct dwc3 *dwc = dep->dwc;
3217 	u32 mdwidth;
3218 	int size;
3219 
3220 	mdwidth = dwc3_mdwidth(dwc);
3221 
3222 	/* MDWIDTH is represented in bits, convert to bytes */
3223 	mdwidth /= 8;
3224 
3225 	/* All OUT endpoints share a single RxFIFO space */
3226 	size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
3227 	if (DWC3_IP_IS(DWC3))
3228 		size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3229 	else
3230 		size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3231 
3232 	/* FIFO depth is in MDWDITH bytes */
3233 	size *= mdwidth;
3234 
3235 	/*
3236 	 * To meet performance requirement, a minimum recommended RxFIFO size
3237 	 * is defined as follow:
3238 	 * RxFIFO size >= (3 x MaxPacketSize) +
3239 	 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3240 	 *
3241 	 * Then calculate the max packet limit as below.
3242 	 */
3243 	size -= (3 * 8) + 16;
3244 	if (size < 0)
3245 		size = 0;
3246 	else
3247 		size /= 3;
3248 
3249 	usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3250 	dep->endpoint.max_streams = 16;
3251 	dep->endpoint.ops = &dwc3_gadget_ep_ops;
3252 	list_add_tail(&dep->endpoint.ep_list,
3253 			&dwc->gadget->ep_list);
3254 	dep->endpoint.caps.type_iso = true;
3255 	dep->endpoint.caps.type_bulk = true;
3256 	dep->endpoint.caps.type_int = true;
3257 
3258 	return dwc3_alloc_trb_pool(dep);
3259 }
3260 
dwc3_gadget_init_endpoint(struct dwc3 * dwc,u8 epnum)3261 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3262 {
3263 	struct dwc3_ep			*dep;
3264 	bool				direction = epnum & 1;
3265 	int				ret;
3266 	u8				num = epnum >> 1;
3267 
3268 	dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3269 	if (!dep)
3270 		return -ENOMEM;
3271 
3272 	dep->dwc = dwc;
3273 	dep->number = epnum;
3274 	dep->direction = direction;
3275 	dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3276 	dwc->eps[epnum] = dep;
3277 	dep->combo_num = 0;
3278 	dep->start_cmd_status = 0;
3279 
3280 	snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
3281 			direction ? "in" : "out");
3282 
3283 	dep->endpoint.name = dep->name;
3284 
3285 	if (!(dep->number > 1)) {
3286 		dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3287 		dep->endpoint.comp_desc = NULL;
3288 	}
3289 
3290 	if (num == 0)
3291 		ret = dwc3_gadget_init_control_endpoint(dep);
3292 	else if (direction)
3293 		ret = dwc3_gadget_init_in_endpoint(dep);
3294 	else
3295 		ret = dwc3_gadget_init_out_endpoint(dep);
3296 
3297 	if (ret)
3298 		return ret;
3299 
3300 	dep->endpoint.caps.dir_in = direction;
3301 	dep->endpoint.caps.dir_out = !direction;
3302 
3303 	INIT_LIST_HEAD(&dep->pending_list);
3304 	INIT_LIST_HEAD(&dep->started_list);
3305 	INIT_LIST_HEAD(&dep->cancelled_list);
3306 
3307 	dwc3_debugfs_create_endpoint_dir(dep);
3308 
3309 	return 0;
3310 }
3311 
dwc3_gadget_init_endpoints(struct dwc3 * dwc,u8 total)3312 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3313 {
3314 	u8				epnum;
3315 
3316 	INIT_LIST_HEAD(&dwc->gadget->ep_list);
3317 
3318 	for (epnum = 0; epnum < total; epnum++) {
3319 		int			ret;
3320 
3321 		ret = dwc3_gadget_init_endpoint(dwc, epnum);
3322 		if (ret)
3323 			return ret;
3324 	}
3325 
3326 	return 0;
3327 }
3328 
dwc3_gadget_free_endpoints(struct dwc3 * dwc)3329 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3330 {
3331 	struct dwc3_ep			*dep;
3332 	u8				epnum;
3333 
3334 	for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3335 		dep = dwc->eps[epnum];
3336 		if (!dep)
3337 			continue;
3338 		/*
3339 		 * Physical endpoints 0 and 1 are special; they form the
3340 		 * bi-directional USB endpoint 0.
3341 		 *
3342 		 * For those two physical endpoints, we don't allocate a TRB
3343 		 * pool nor do we add them the endpoints list. Due to that, we
3344 		 * shouldn't do these two operations otherwise we would end up
3345 		 * with all sorts of bugs when removing dwc3.ko.
3346 		 */
3347 		if (epnum != 0 && epnum != 1) {
3348 			dwc3_free_trb_pool(dep);
3349 			list_del(&dep->endpoint.ep_list);
3350 		}
3351 
3352 		dwc3_debugfs_remove_endpoint_dir(dep);
3353 		kfree(dep);
3354 	}
3355 }
3356 
3357 /* -------------------------------------------------------------------------- */
3358 
dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep * dep,struct dwc3_request * req,struct dwc3_trb * trb,const struct dwc3_event_depevt * event,int status,int chain)3359 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3360 		struct dwc3_request *req, struct dwc3_trb *trb,
3361 		const struct dwc3_event_depevt *event, int status, int chain)
3362 {
3363 	unsigned int		count;
3364 
3365 	dwc3_ep_inc_deq(dep);
3366 
3367 	trace_dwc3_complete_trb(dep, trb);
3368 	req->num_trbs--;
3369 
3370 	/*
3371 	 * If we're in the middle of series of chained TRBs and we
3372 	 * receive a short transfer along the way, DWC3 will skip
3373 	 * through all TRBs including the last TRB in the chain (the
3374 	 * where CHN bit is zero. DWC3 will also avoid clearing HWO
3375 	 * bit and SW has to do it manually.
3376 	 *
3377 	 * We're going to do that here to avoid problems of HW trying
3378 	 * to use bogus TRBs for transfers.
3379 	 */
3380 	if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3381 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3382 
3383 	/*
3384 	 * For isochronous transfers, the first TRB in a service interval must
3385 	 * have the Isoc-First type. Track and report its interval frame number.
3386 	 */
3387 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3388 	    (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3389 		unsigned int frame_number;
3390 
3391 		frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3392 		frame_number &= ~(dep->interval - 1);
3393 		req->request.frame_number = frame_number;
3394 	}
3395 
3396 	/*
3397 	 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3398 	 * this TRB points to the bounce buffer address, it's a MPS alignment
3399 	 * TRB. Don't add it to req->remaining calculation.
3400 	 */
3401 	if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3402 	    trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3403 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3404 		return 1;
3405 	}
3406 
3407 	count = trb->size & DWC3_TRB_SIZE_MASK;
3408 	req->remaining += count;
3409 
3410 	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3411 		return 1;
3412 
3413 	if (event->status & DEPEVT_STATUS_SHORT && !chain)
3414 		return 1;
3415 
3416 	if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
3417 	    DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
3418 		return 1;
3419 
3420 	if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3421 	    (trb->ctrl & DWC3_TRB_CTRL_LST))
3422 		return 1;
3423 
3424 	return 0;
3425 }
3426 
dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep * dep,struct dwc3_request * req,const struct dwc3_event_depevt * event,int status)3427 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3428 		struct dwc3_request *req, const struct dwc3_event_depevt *event,
3429 		int status)
3430 {
3431 	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3432 	struct scatterlist *sg = req->sg;
3433 	struct scatterlist *s;
3434 	unsigned int num_queued = req->num_queued_sgs;
3435 	unsigned int i;
3436 	int ret = 0;
3437 
3438 	for_each_sg(sg, s, num_queued, i) {
3439 		trb = &dep->trb_pool[dep->trb_dequeue];
3440 
3441 		req->sg = sg_next(s);
3442 		req->num_queued_sgs--;
3443 
3444 		ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3445 				trb, event, status, true);
3446 		if (ret)
3447 			break;
3448 	}
3449 
3450 	return ret;
3451 }
3452 
dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep * dep,struct dwc3_request * req,const struct dwc3_event_depevt * event,int status)3453 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3454 		struct dwc3_request *req, const struct dwc3_event_depevt *event,
3455 		int status)
3456 {
3457 	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3458 
3459 	return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3460 			event, status, false);
3461 }
3462 
dwc3_gadget_ep_request_completed(struct dwc3_request * req)3463 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3464 {
3465 	return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3466 }
3467 
dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,struct dwc3_request * req,int status)3468 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3469 		const struct dwc3_event_depevt *event,
3470 		struct dwc3_request *req, int status)
3471 {
3472 	int request_status;
3473 	int ret;
3474 
3475 	if (req->request.num_mapped_sgs)
3476 		ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3477 				status);
3478 	else
3479 		ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3480 				status);
3481 
3482 	req->request.actual = req->request.length - req->remaining;
3483 
3484 	if (!dwc3_gadget_ep_request_completed(req))
3485 		goto out;
3486 
3487 	if (req->needs_extra_trb) {
3488 		ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3489 				status);
3490 		req->needs_extra_trb = false;
3491 	}
3492 
3493 	/*
3494 	 * The event status only reflects the status of the TRB with IOC set.
3495 	 * For the requests that don't set interrupt on completion, the driver
3496 	 * needs to check and return the status of the completed TRBs associated
3497 	 * with the request. Use the status of the last TRB of the request.
3498 	 */
3499 	if (req->request.no_interrupt) {
3500 		struct dwc3_trb *trb;
3501 
3502 		trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
3503 		switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3504 		case DWC3_TRBSTS_MISSED_ISOC:
3505 			/* Isoc endpoint only */
3506 			request_status = -EXDEV;
3507 			break;
3508 		case DWC3_TRB_STS_XFER_IN_PROG:
3509 			/* Applicable when End Transfer with ForceRM=0 */
3510 		case DWC3_TRBSTS_SETUP_PENDING:
3511 			/* Control endpoint only */
3512 		case DWC3_TRBSTS_OK:
3513 		default:
3514 			request_status = 0;
3515 			break;
3516 		}
3517 	} else {
3518 		request_status = status;
3519 	}
3520 
3521 	dwc3_gadget_giveback(dep, req, request_status);
3522 
3523 out:
3524 	return ret;
3525 }
3526 
dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,int status)3527 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3528 		const struct dwc3_event_depevt *event, int status)
3529 {
3530 	struct dwc3_request	*req;
3531 
3532 	while (!list_empty(&dep->started_list)) {
3533 		int ret;
3534 
3535 		req = next_request(&dep->started_list);
3536 		ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3537 				req, status);
3538 		if (ret)
3539 			break;
3540 		/*
3541 		 * The endpoint is disabled, let the dwc3_remove_requests()
3542 		 * handle the cleanup.
3543 		 */
3544 		if (!dep->endpoint.desc)
3545 			break;
3546 	}
3547 }
3548 
dwc3_gadget_ep_should_continue(struct dwc3_ep * dep)3549 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3550 {
3551 	struct dwc3_request	*req;
3552 	struct dwc3		*dwc = dep->dwc;
3553 
3554 	if (!dep->endpoint.desc || !dwc->pullups_connected ||
3555 	    !dwc->connected)
3556 		return false;
3557 
3558 	if (!list_empty(&dep->pending_list))
3559 		return true;
3560 
3561 	/*
3562 	 * We only need to check the first entry of the started list. We can
3563 	 * assume the completed requests are removed from the started list.
3564 	 */
3565 	req = next_request(&dep->started_list);
3566 	if (!req)
3567 		return false;
3568 
3569 	return !dwc3_gadget_ep_request_completed(req);
3570 }
3571 
dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3572 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3573 		const struct dwc3_event_depevt *event)
3574 {
3575 	dep->frame_number = event->parameters;
3576 }
3577 
dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,int status)3578 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3579 		const struct dwc3_event_depevt *event, int status)
3580 {
3581 	struct dwc3		*dwc = dep->dwc;
3582 	bool			no_started_trb = true;
3583 
3584 	dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3585 
3586 	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3587 		goto out;
3588 
3589 	if (!dep->endpoint.desc)
3590 		return no_started_trb;
3591 
3592 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3593 		list_empty(&dep->started_list) &&
3594 		(list_empty(&dep->pending_list) || status == -EXDEV))
3595 		dwc3_stop_active_transfer(dep, true, true);
3596 	else if (dwc3_gadget_ep_should_continue(dep))
3597 		if (__dwc3_gadget_kick_transfer(dep) == 0)
3598 			no_started_trb = false;
3599 
3600 out:
3601 	/*
3602 	 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3603 	 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
3604 	 */
3605 	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3606 		u32		reg;
3607 		int		i;
3608 
3609 		for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3610 			dep = dwc->eps[i];
3611 
3612 			if (!(dep->flags & DWC3_EP_ENABLED))
3613 				continue;
3614 
3615 			if (!list_empty(&dep->started_list))
3616 				return no_started_trb;
3617 		}
3618 
3619 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3620 		reg |= dwc->u1u2;
3621 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3622 
3623 		dwc->u1u2 = 0;
3624 	}
3625 
3626 	return no_started_trb;
3627 }
3628 
dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3629 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3630 		const struct dwc3_event_depevt *event)
3631 {
3632 	int status = 0;
3633 
3634 	if (!dep->endpoint.desc)
3635 		return;
3636 
3637 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3638 		dwc3_gadget_endpoint_frame_from_event(dep, event);
3639 
3640 	if (event->status & DEPEVT_STATUS_BUSERR)
3641 		status = -ECONNRESET;
3642 
3643 	if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3644 		status = -EXDEV;
3645 
3646 	dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3647 }
3648 
dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3649 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3650 		const struct dwc3_event_depevt *event)
3651 {
3652 	int status = 0;
3653 
3654 	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3655 
3656 	if (event->status & DEPEVT_STATUS_BUSERR)
3657 		status = -ECONNRESET;
3658 
3659 	if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3660 		dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3661 }
3662 
dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3663 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3664 		const struct dwc3_event_depevt *event)
3665 {
3666 	dwc3_gadget_endpoint_frame_from_event(dep, event);
3667 
3668 	/*
3669 	 * The XferNotReady event is generated only once before the endpoint
3670 	 * starts. It will be generated again when END_TRANSFER command is
3671 	 * issued. For some controller versions, the XferNotReady event may be
3672 	 * generated while the END_TRANSFER command is still in process. Ignore
3673 	 * it and wait for the next XferNotReady event after the command is
3674 	 * completed.
3675 	 */
3676 	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3677 		return;
3678 
3679 	(void) __dwc3_gadget_start_isoc(dep);
3680 }
3681 
dwc3_gadget_endpoint_command_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3682 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3683 		const struct dwc3_event_depevt *event)
3684 {
3685 	u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3686 
3687 	if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3688 		return;
3689 
3690 	/*
3691 	 * The END_TRANSFER command will cause the controller to generate a
3692 	 * NoStream Event, and it's not due to the host DP NoStream rejection.
3693 	 * Ignore the next NoStream event.
3694 	 */
3695 	if (dep->stream_capable)
3696 		dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3697 
3698 	dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3699 	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3700 	dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3701 
3702 	if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3703 		struct dwc3 *dwc = dep->dwc;
3704 
3705 		dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3706 		if (dwc3_send_clear_stall_ep_cmd(dep)) {
3707 			struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3708 
3709 			dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3710 			if (dwc->delayed_status)
3711 				__dwc3_gadget_ep0_set_halt(ep0, 1);
3712 			return;
3713 		}
3714 
3715 		dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3716 		if (dwc->clear_stall_protocol == dep->number)
3717 			dwc3_ep0_send_delayed_status(dwc);
3718 	}
3719 
3720 	if ((dep->flags & DWC3_EP_DELAY_START) &&
3721 	    !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3722 		__dwc3_gadget_kick_transfer(dep);
3723 
3724 	dep->flags &= ~DWC3_EP_DELAY_START;
3725 }
3726 
dwc3_gadget_endpoint_stream_event(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3727 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3728 		const struct dwc3_event_depevt *event)
3729 {
3730 	struct dwc3 *dwc = dep->dwc;
3731 
3732 	if (event->status == DEPEVT_STREAMEVT_FOUND) {
3733 		dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3734 		goto out;
3735 	}
3736 
3737 	/* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3738 	switch (event->parameters) {
3739 	case DEPEVT_STREAM_PRIME:
3740 		/*
3741 		 * If the host can properly transition the endpoint state from
3742 		 * idle to prime after a NoStream rejection, there's no need to
3743 		 * force restarting the endpoint to reinitiate the stream. To
3744 		 * simplify the check, assume the host follows the USB spec if
3745 		 * it primed the endpoint more than once.
3746 		 */
3747 		if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3748 			if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3749 				dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3750 			else
3751 				dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3752 		}
3753 
3754 		break;
3755 	case DEPEVT_STREAM_NOSTREAM:
3756 		if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3757 		    !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3758 		    (!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3759 		     !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3760 			break;
3761 
3762 		/*
3763 		 * If the host rejects a stream due to no active stream, by the
3764 		 * USB and xHCI spec, the endpoint will be put back to idle
3765 		 * state. When the host is ready (buffer added/updated), it will
3766 		 * prime the endpoint to inform the usb device controller. This
3767 		 * triggers the device controller to issue ERDY to restart the
3768 		 * stream. However, some hosts don't follow this and keep the
3769 		 * endpoint in the idle state. No prime will come despite host
3770 		 * streams are updated, and the device controller will not be
3771 		 * triggered to generate ERDY to move the next stream data. To
3772 		 * workaround this and maintain compatibility with various
3773 		 * hosts, force to reinitiate the stream until the host is ready
3774 		 * instead of waiting for the host to prime the endpoint.
3775 		 */
3776 		if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3777 			unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3778 
3779 			dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3780 		} else {
3781 			dep->flags |= DWC3_EP_DELAY_START;
3782 			dwc3_stop_active_transfer(dep, true, true);
3783 			return;
3784 		}
3785 		break;
3786 	}
3787 
3788 out:
3789 	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3790 }
3791 
dwc3_endpoint_interrupt(struct dwc3 * dwc,const struct dwc3_event_depevt * event)3792 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3793 		const struct dwc3_event_depevt *event)
3794 {
3795 	struct dwc3_ep		*dep;
3796 	u8			epnum = event->endpoint_number;
3797 
3798 	dep = dwc->eps[epnum];
3799 
3800 	if (!(dep->flags & DWC3_EP_ENABLED)) {
3801 		if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3802 			return;
3803 
3804 		/* Handle only EPCMDCMPLT when EP disabled */
3805 		if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
3806 			!(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3807 			return;
3808 	}
3809 
3810 	if (epnum == 0 || epnum == 1) {
3811 		dwc3_ep0_interrupt(dwc, event);
3812 		return;
3813 	}
3814 
3815 	switch (event->endpoint_event) {
3816 	case DWC3_DEPEVT_XFERINPROGRESS:
3817 		dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3818 		break;
3819 	case DWC3_DEPEVT_XFERNOTREADY:
3820 		dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3821 		break;
3822 	case DWC3_DEPEVT_EPCMDCMPLT:
3823 		dwc3_gadget_endpoint_command_complete(dep, event);
3824 		break;
3825 	case DWC3_DEPEVT_XFERCOMPLETE:
3826 		dwc3_gadget_endpoint_transfer_complete(dep, event);
3827 		break;
3828 	case DWC3_DEPEVT_STREAMEVT:
3829 		dwc3_gadget_endpoint_stream_event(dep, event);
3830 		break;
3831 	case DWC3_DEPEVT_RXTXFIFOEVT:
3832 		break;
3833 	default:
3834 		dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event);
3835 		break;
3836 	}
3837 }
3838 
dwc3_disconnect_gadget(struct dwc3 * dwc)3839 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3840 {
3841 	if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3842 		spin_unlock(&dwc->lock);
3843 		dwc->gadget_driver->disconnect(dwc->gadget);
3844 		spin_lock(&dwc->lock);
3845 	}
3846 }
3847 
dwc3_suspend_gadget(struct dwc3 * dwc)3848 static void dwc3_suspend_gadget(struct dwc3 *dwc)
3849 {
3850 	if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3851 		spin_unlock(&dwc->lock);
3852 		dwc->gadget_driver->suspend(dwc->gadget);
3853 		spin_lock(&dwc->lock);
3854 	}
3855 }
3856 
dwc3_resume_gadget(struct dwc3 * dwc)3857 static void dwc3_resume_gadget(struct dwc3 *dwc)
3858 {
3859 	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3860 		spin_unlock(&dwc->lock);
3861 		dwc->gadget_driver->resume(dwc->gadget);
3862 		spin_lock(&dwc->lock);
3863 	}
3864 }
3865 
dwc3_reset_gadget(struct dwc3 * dwc)3866 static void dwc3_reset_gadget(struct dwc3 *dwc)
3867 {
3868 	if (!dwc->gadget_driver)
3869 		return;
3870 
3871 	if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3872 		spin_unlock(&dwc->lock);
3873 		usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3874 		spin_lock(&dwc->lock);
3875 	}
3876 }
3877 
dwc3_stop_active_transfer(struct dwc3_ep * dep,bool force,bool interrupt)3878 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3879 	bool interrupt)
3880 {
3881 	struct dwc3 *dwc = dep->dwc;
3882 
3883 	/*
3884 	 * Only issue End Transfer command to the control endpoint of a started
3885 	 * Data Phase. Typically we should only do so in error cases such as
3886 	 * invalid/unexpected direction as described in the control transfer
3887 	 * flow of the programming guide.
3888 	 */
3889 	if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3890 		return;
3891 
3892 	if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
3893 		return;
3894 
3895 	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3896 	    (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3897 		return;
3898 
3899 	/*
3900 	 * If a Setup packet is received but yet to DMA out, the controller will
3901 	 * not process the End Transfer command of any endpoint. Polling of its
3902 	 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
3903 	 * timeout. Delay issuing the End Transfer command until the Setup TRB is
3904 	 * prepared.
3905 	 */
3906 	if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
3907 		dep->flags |= DWC3_EP_DELAY_STOP;
3908 		return;
3909 	}
3910 
3911 	/*
3912 	 * NOTICE: We are violating what the Databook says about the
3913 	 * EndTransfer command. Ideally we would _always_ wait for the
3914 	 * EndTransfer Command Completion IRQ, but that's causing too
3915 	 * much trouble synchronizing between us and gadget driver.
3916 	 *
3917 	 * We have discussed this with the IP Provider and it was
3918 	 * suggested to giveback all requests here.
3919 	 *
3920 	 * Note also that a similar handling was tested by Synopsys
3921 	 * (thanks a lot Paul) and nothing bad has come out of it.
3922 	 * In short, what we're doing is issuing EndTransfer with
3923 	 * CMDIOC bit set and delay kicking transfer until the
3924 	 * EndTransfer command had completed.
3925 	 *
3926 	 * As of IP version 3.10a of the DWC_usb3 IP, the controller
3927 	 * supports a mode to work around the above limitation. The
3928 	 * software can poll the CMDACT bit in the DEPCMD register
3929 	 * after issuing a EndTransfer command. This mode is enabled
3930 	 * by writing GUCTL2[14]. This polling is already done in the
3931 	 * dwc3_send_gadget_ep_cmd() function so if the mode is
3932 	 * enabled, the EndTransfer command will have completed upon
3933 	 * returning from this function.
3934 	 *
3935 	 * This mode is NOT available on the DWC_usb31 IP.  In this
3936 	 * case, if the IOC bit is not set, then delay by 1ms
3937 	 * after issuing the EndTransfer command.  This allows for the
3938 	 * controller to handle the command completely before DWC3
3939 	 * remove requests attempts to unmap USB request buffers.
3940 	 */
3941 
3942 	__dwc3_stop_active_transfer(dep, force, interrupt);
3943 }
3944 
dwc3_clear_stall_all_ep(struct dwc3 * dwc)3945 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3946 {
3947 	u32 epnum;
3948 
3949 	for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3950 		struct dwc3_ep *dep;
3951 		int ret;
3952 
3953 		dep = dwc->eps[epnum];
3954 		if (!dep)
3955 			continue;
3956 
3957 		if (!(dep->flags & DWC3_EP_STALL))
3958 			continue;
3959 
3960 		dep->flags &= ~DWC3_EP_STALL;
3961 
3962 		ret = dwc3_send_clear_stall_ep_cmd(dep);
3963 		WARN_ON_ONCE(ret);
3964 	}
3965 }
3966 
dwc3_gadget_disconnect_interrupt(struct dwc3 * dwc)3967 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3968 {
3969 	int			reg;
3970 
3971 	dwc->suspended = false;
3972 
3973 	dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3974 
3975 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3976 	reg &= ~DWC3_DCTL_INITU1ENA;
3977 	reg &= ~DWC3_DCTL_INITU2ENA;
3978 	dwc3_gadget_dctl_write_safe(dwc, reg);
3979 
3980 	dwc->connected = false;
3981 
3982 	dwc3_disconnect_gadget(dwc);
3983 
3984 	dwc->gadget->speed = USB_SPEED_UNKNOWN;
3985 	dwc->setup_packet_pending = false;
3986 	dwc->gadget->wakeup_armed = false;
3987 	dwc3_gadget_enable_linksts_evts(dwc, false);
3988 	usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3989 
3990 	dwc3_ep0_reset_state(dwc);
3991 
3992 	/*
3993 	 * Request PM idle to address condition where usage count is
3994 	 * already decremented to zero, but waiting for the disconnect
3995 	 * interrupt to set dwc->connected to FALSE.
3996 	 */
3997 	pm_request_idle(dwc->dev);
3998 }
3999 
dwc3_gadget_reset_interrupt(struct dwc3 * dwc)4000 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
4001 {
4002 	u32			reg;
4003 
4004 	dwc->suspended = false;
4005 
4006 	/*
4007 	 * Ideally, dwc3_reset_gadget() would trigger the function
4008 	 * drivers to stop any active transfers through ep disable.
4009 	 * However, for functions which defer ep disable, such as mass
4010 	 * storage, we will need to rely on the call to stop active
4011 	 * transfers here, and avoid allowing of request queuing.
4012 	 */
4013 	dwc->connected = false;
4014 
4015 	/*
4016 	 * WORKAROUND: DWC3 revisions <1.88a have an issue which
4017 	 * would cause a missing Disconnect Event if there's a
4018 	 * pending Setup Packet in the FIFO.
4019 	 *
4020 	 * There's no suggested workaround on the official Bug
4021 	 * report, which states that "unless the driver/application
4022 	 * is doing any special handling of a disconnect event,
4023 	 * there is no functional issue".
4024 	 *
4025 	 * Unfortunately, it turns out that we _do_ some special
4026 	 * handling of a disconnect event, namely complete all
4027 	 * pending transfers, notify gadget driver of the
4028 	 * disconnection, and so on.
4029 	 *
4030 	 * Our suggested workaround is to follow the Disconnect
4031 	 * Event steps here, instead, based on a setup_packet_pending
4032 	 * flag. Such flag gets set whenever we have a SETUP_PENDING
4033 	 * status for EP0 TRBs and gets cleared on XferComplete for the
4034 	 * same endpoint.
4035 	 *
4036 	 * Refers to:
4037 	 *
4038 	 * STAR#9000466709: RTL: Device : Disconnect event not
4039 	 * generated if setup packet pending in FIFO
4040 	 */
4041 	if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
4042 		if (dwc->setup_packet_pending)
4043 			dwc3_gadget_disconnect_interrupt(dwc);
4044 	}
4045 
4046 	dwc3_reset_gadget(dwc);
4047 
4048 	/*
4049 	 * From SNPS databook section 8.1.2, the EP0 should be in setup
4050 	 * phase. So ensure that EP0 is in setup phase by issuing a stall
4051 	 * and restart if EP0 is not in setup phase.
4052 	 */
4053 	dwc3_ep0_reset_state(dwc);
4054 
4055 	/*
4056 	 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
4057 	 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
4058 	 * needs to ensure that it sends "a DEPENDXFER command for any active
4059 	 * transfers."
4060 	 */
4061 	dwc3_stop_active_transfers(dwc);
4062 	dwc->connected = true;
4063 
4064 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4065 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
4066 	dwc3_gadget_dctl_write_safe(dwc, reg);
4067 	dwc->test_mode = false;
4068 	dwc->gadget->wakeup_armed = false;
4069 	dwc3_gadget_enable_linksts_evts(dwc, false);
4070 	dwc3_clear_stall_all_ep(dwc);
4071 
4072 	/* Reset device address to zero */
4073 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4074 	reg &= ~(DWC3_DCFG_DEVADDR_MASK);
4075 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4076 }
4077 
dwc3_gadget_conndone_interrupt(struct dwc3 * dwc)4078 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
4079 {
4080 	struct dwc3_ep		*dep;
4081 	int			ret;
4082 	u32			reg;
4083 	u8			lanes = 1;
4084 	u8			speed;
4085 
4086 	if (!dwc->softconnect)
4087 		return;
4088 
4089 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
4090 	speed = reg & DWC3_DSTS_CONNECTSPD;
4091 	dwc->speed = speed;
4092 
4093 	if (DWC3_IP_IS(DWC32))
4094 		lanes = DWC3_DSTS_CONNLANES(reg) + 1;
4095 
4096 	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4097 
4098 	/*
4099 	 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
4100 	 * each time on Connect Done.
4101 	 *
4102 	 * Currently we always use the reset value. If any platform
4103 	 * wants to set this to a different value, we need to add a
4104 	 * setting and update GCTL.RAMCLKSEL here.
4105 	 */
4106 
4107 	switch (speed) {
4108 	case DWC3_DSTS_SUPERSPEED_PLUS:
4109 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4110 		dwc->gadget->ep0->maxpacket = 512;
4111 		dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4112 
4113 		if (lanes > 1)
4114 			dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
4115 		else
4116 			dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
4117 		break;
4118 	case DWC3_DSTS_SUPERSPEED:
4119 		/*
4120 		 * WORKAROUND: DWC3 revisions <1.90a have an issue which
4121 		 * would cause a missing USB3 Reset event.
4122 		 *
4123 		 * In such situations, we should force a USB3 Reset
4124 		 * event by calling our dwc3_gadget_reset_interrupt()
4125 		 * routine.
4126 		 *
4127 		 * Refers to:
4128 		 *
4129 		 * STAR#9000483510: RTL: SS : USB3 reset event may
4130 		 * not be generated always when the link enters poll
4131 		 */
4132 		if (DWC3_VER_IS_PRIOR(DWC3, 190A))
4133 			dwc3_gadget_reset_interrupt(dwc);
4134 
4135 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4136 		dwc->gadget->ep0->maxpacket = 512;
4137 		dwc->gadget->speed = USB_SPEED_SUPER;
4138 
4139 		if (lanes > 1) {
4140 			dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4141 			dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
4142 		}
4143 		break;
4144 	case DWC3_DSTS_HIGHSPEED:
4145 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4146 		dwc->gadget->ep0->maxpacket = 64;
4147 		dwc->gadget->speed = USB_SPEED_HIGH;
4148 		break;
4149 	case DWC3_DSTS_FULLSPEED:
4150 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4151 		dwc->gadget->ep0->maxpacket = 64;
4152 		dwc->gadget->speed = USB_SPEED_FULL;
4153 		break;
4154 	}
4155 
4156 	dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
4157 
4158 	/* Enable USB2 LPM Capability */
4159 
4160 	if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
4161 	    !dwc->usb2_gadget_lpm_disable &&
4162 	    (speed != DWC3_DSTS_SUPERSPEED) &&
4163 	    (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
4164 		reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4165 		reg |= DWC3_DCFG_LPM_CAP;
4166 		dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4167 
4168 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4169 		reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
4170 
4171 		reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
4172 					    (dwc->is_utmi_l1_suspend << 4));
4173 
4174 		/*
4175 		 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
4176 		 * DCFG.LPMCap is set, core responses with an ACK and the
4177 		 * BESL value in the LPM token is less than or equal to LPM
4178 		 * NYET threshold.
4179 		 */
4180 		WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
4181 				"LPM Erratum not available on dwc3 revisions < 2.40a\n");
4182 
4183 		if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
4184 			reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
4185 
4186 		dwc3_gadget_dctl_write_safe(dwc, reg);
4187 	} else {
4188 		if (dwc->usb2_gadget_lpm_disable) {
4189 			reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4190 			reg &= ~DWC3_DCFG_LPM_CAP;
4191 			dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4192 		}
4193 
4194 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4195 		reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
4196 		dwc3_gadget_dctl_write_safe(dwc, reg);
4197 	}
4198 
4199 	dep = dwc->eps[0];
4200 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4201 	if (ret) {
4202 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4203 		return;
4204 	}
4205 
4206 	dep = dwc->eps[1];
4207 	ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4208 	if (ret) {
4209 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4210 		return;
4211 	}
4212 
4213 	/*
4214 	 * Configure PHY via GUSB3PIPECTLn if required.
4215 	 *
4216 	 * Update GTXFIFOSIZn
4217 	 *
4218 	 * In both cases reset values should be sufficient.
4219 	 */
4220 }
4221 
dwc3_gadget_wakeup_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4222 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
4223 {
4224 	dwc->suspended = false;
4225 
4226 	/*
4227 	 * TODO take core out of low power mode when that's
4228 	 * implemented.
4229 	 */
4230 
4231 	if (dwc->async_callbacks && dwc->gadget_driver->resume) {
4232 		spin_unlock(&dwc->lock);
4233 		dwc->gadget_driver->resume(dwc->gadget);
4234 		spin_lock(&dwc->lock);
4235 	}
4236 
4237 	dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
4238 }
4239 
dwc3_gadget_linksts_change_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4240 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
4241 		unsigned int evtinfo)
4242 {
4243 	enum dwc3_link_state	next = evtinfo & DWC3_LINK_STATE_MASK;
4244 	unsigned int		pwropt;
4245 
4246 	/*
4247 	 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
4248 	 * Hibernation mode enabled which would show up when device detects
4249 	 * host-initiated U3 exit.
4250 	 *
4251 	 * In that case, device will generate a Link State Change Interrupt
4252 	 * from U3 to RESUME which is only necessary if Hibernation is
4253 	 * configured in.
4254 	 *
4255 	 * There are no functional changes due to such spurious event and we
4256 	 * just need to ignore it.
4257 	 *
4258 	 * Refers to:
4259 	 *
4260 	 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4261 	 * operational mode
4262 	 */
4263 	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4264 	if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4265 			(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4266 		if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4267 				(next == DWC3_LINK_STATE_RESUME)) {
4268 			return;
4269 		}
4270 	}
4271 
4272 	/*
4273 	 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4274 	 * on the link partner, the USB session might do multiple entry/exit
4275 	 * of low power states before a transfer takes place.
4276 	 *
4277 	 * Due to this problem, we might experience lower throughput. The
4278 	 * suggested workaround is to disable DCTL[12:9] bits if we're
4279 	 * transitioning from U1/U2 to U0 and enable those bits again
4280 	 * after a transfer completes and there are no pending transfers
4281 	 * on any of the enabled endpoints.
4282 	 *
4283 	 * This is the first half of that workaround.
4284 	 *
4285 	 * Refers to:
4286 	 *
4287 	 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4288 	 * core send LGO_Ux entering U0
4289 	 */
4290 	if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4291 		if (next == DWC3_LINK_STATE_U0) {
4292 			u32	u1u2;
4293 			u32	reg;
4294 
4295 			switch (dwc->link_state) {
4296 			case DWC3_LINK_STATE_U1:
4297 			case DWC3_LINK_STATE_U2:
4298 				reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4299 				u1u2 = reg & (DWC3_DCTL_INITU2ENA
4300 						| DWC3_DCTL_ACCEPTU2ENA
4301 						| DWC3_DCTL_INITU1ENA
4302 						| DWC3_DCTL_ACCEPTU1ENA);
4303 
4304 				if (!dwc->u1u2)
4305 					dwc->u1u2 = reg & u1u2;
4306 
4307 				reg &= ~u1u2;
4308 
4309 				dwc3_gadget_dctl_write_safe(dwc, reg);
4310 				break;
4311 			default:
4312 				/* do nothing */
4313 				break;
4314 			}
4315 		}
4316 	}
4317 
4318 	switch (next) {
4319 	case DWC3_LINK_STATE_U0:
4320 		if (dwc->gadget->wakeup_armed) {
4321 			dwc3_gadget_enable_linksts_evts(dwc, false);
4322 			dwc3_resume_gadget(dwc);
4323 			dwc->suspended = false;
4324 		}
4325 		break;
4326 	case DWC3_LINK_STATE_U1:
4327 		if (dwc->speed == USB_SPEED_SUPER)
4328 			dwc3_suspend_gadget(dwc);
4329 		break;
4330 	case DWC3_LINK_STATE_U2:
4331 	case DWC3_LINK_STATE_U3:
4332 		dwc3_suspend_gadget(dwc);
4333 		break;
4334 	case DWC3_LINK_STATE_RESUME:
4335 		dwc3_resume_gadget(dwc);
4336 		break;
4337 	default:
4338 		/* do nothing */
4339 		break;
4340 	}
4341 
4342 	dwc->link_state = next;
4343 }
4344 
dwc3_gadget_suspend_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4345 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4346 					  unsigned int evtinfo)
4347 {
4348 	enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4349 
4350 	if (!dwc->suspended && next == DWC3_LINK_STATE_U3) {
4351 		dwc->suspended = true;
4352 		dwc3_suspend_gadget(dwc);
4353 	}
4354 
4355 	dwc->link_state = next;
4356 }
4357 
dwc3_gadget_interrupt(struct dwc3 * dwc,const struct dwc3_event_devt * event)4358 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4359 		const struct dwc3_event_devt *event)
4360 {
4361 	switch (event->type) {
4362 	case DWC3_DEVICE_EVENT_DISCONNECT:
4363 		dwc3_gadget_disconnect_interrupt(dwc);
4364 		break;
4365 	case DWC3_DEVICE_EVENT_RESET:
4366 		dwc3_gadget_reset_interrupt(dwc);
4367 		break;
4368 	case DWC3_DEVICE_EVENT_CONNECT_DONE:
4369 		dwc3_gadget_conndone_interrupt(dwc);
4370 		break;
4371 	case DWC3_DEVICE_EVENT_WAKEUP:
4372 		dwc3_gadget_wakeup_interrupt(dwc, event->event_info);
4373 		break;
4374 	case DWC3_DEVICE_EVENT_HIBER_REQ:
4375 		dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");
4376 		break;
4377 	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4378 		dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
4379 		break;
4380 	case DWC3_DEVICE_EVENT_SUSPEND:
4381 		/* It changed to be suspend event for version 2.30a and above */
4382 		if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
4383 			dwc3_gadget_suspend_interrupt(dwc, event->event_info);
4384 		break;
4385 	case DWC3_DEVICE_EVENT_SOF:
4386 	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4387 	case DWC3_DEVICE_EVENT_CMD_CMPL:
4388 	case DWC3_DEVICE_EVENT_OVERFLOW:
4389 		break;
4390 	default:
4391 		dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4392 	}
4393 }
4394 
dwc3_process_event_entry(struct dwc3 * dwc,const union dwc3_event * event)4395 static void dwc3_process_event_entry(struct dwc3 *dwc,
4396 		const union dwc3_event *event)
4397 {
4398 	trace_dwc3_event(event->raw, dwc);
4399 
4400 	if (!event->type.is_devspec)
4401 		dwc3_endpoint_interrupt(dwc, &event->depevt);
4402 	else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4403 		dwc3_gadget_interrupt(dwc, &event->devt);
4404 	else
4405 		dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4406 }
4407 
dwc3_process_event_buf(struct dwc3_event_buffer * evt)4408 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4409 {
4410 	struct dwc3 *dwc = evt->dwc;
4411 	irqreturn_t ret = IRQ_NONE;
4412 	int left;
4413 
4414 	left = evt->count;
4415 
4416 	if (!(evt->flags & DWC3_EVENT_PENDING))
4417 		return IRQ_NONE;
4418 
4419 	while (left > 0) {
4420 		union dwc3_event event;
4421 
4422 		event.raw = *(u32 *) (evt->cache + evt->lpos);
4423 
4424 		dwc3_process_event_entry(dwc, &event);
4425 
4426 		/*
4427 		 * FIXME we wrap around correctly to the next entry as
4428 		 * almost all entries are 4 bytes in size. There is one
4429 		 * entry which has 12 bytes which is a regular entry
4430 		 * followed by 8 bytes data. ATM I don't know how
4431 		 * things are organized if we get next to the a
4432 		 * boundary so I worry about that once we try to handle
4433 		 * that.
4434 		 */
4435 		evt->lpos = (evt->lpos + 4) % evt->length;
4436 		left -= 4;
4437 	}
4438 
4439 	evt->count = 0;
4440 	ret = IRQ_HANDLED;
4441 
4442 	/* Unmask interrupt */
4443 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4444 		    DWC3_GEVNTSIZ_SIZE(evt->length));
4445 
4446 	if (dwc->imod_interval) {
4447 		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4448 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
4449 	}
4450 
4451 	/* Keep the clearing of DWC3_EVENT_PENDING at the end */
4452 	evt->flags &= ~DWC3_EVENT_PENDING;
4453 
4454 	return ret;
4455 }
4456 
dwc3_thread_interrupt(int irq,void * _evt)4457 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4458 {
4459 	struct dwc3_event_buffer *evt = _evt;
4460 	struct dwc3 *dwc = evt->dwc;
4461 	unsigned long flags;
4462 	irqreturn_t ret = IRQ_NONE;
4463 
4464 	local_bh_disable();
4465 	spin_lock_irqsave(&dwc->lock, flags);
4466 	ret = dwc3_process_event_buf(evt);
4467 	spin_unlock_irqrestore(&dwc->lock, flags);
4468 	local_bh_enable();
4469 
4470 	return ret;
4471 }
4472 
dwc3_check_event_buf(struct dwc3_event_buffer * evt)4473 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4474 {
4475 	struct dwc3 *dwc = evt->dwc;
4476 	u32 amount;
4477 	u32 count;
4478 
4479 	if (pm_runtime_suspended(dwc->dev)) {
4480 		dwc->pending_events = true;
4481 		/*
4482 		 * Trigger runtime resume. The get() function will be balanced
4483 		 * after processing the pending events in dwc3_process_pending
4484 		 * events().
4485 		 */
4486 		pm_runtime_get(dwc->dev);
4487 		disable_irq_nosync(dwc->irq_gadget);
4488 		return IRQ_HANDLED;
4489 	}
4490 
4491 	/*
4492 	 * With PCIe legacy interrupt, test shows that top-half irq handler can
4493 	 * be called again after HW interrupt deassertion. Check if bottom-half
4494 	 * irq event handler completes before caching new event to prevent
4495 	 * losing events.
4496 	 */
4497 	if (evt->flags & DWC3_EVENT_PENDING)
4498 		return IRQ_HANDLED;
4499 
4500 	count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
4501 	count &= DWC3_GEVNTCOUNT_MASK;
4502 	if (!count)
4503 		return IRQ_NONE;
4504 
4505 	evt->count = count;
4506 	evt->flags |= DWC3_EVENT_PENDING;
4507 
4508 	/* Mask interrupt */
4509 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4510 		    DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4511 
4512 	amount = min(count, evt->length - evt->lpos);
4513 	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4514 
4515 	if (amount < count)
4516 		memcpy(evt->cache, evt->buf, count - amount);
4517 
4518 	dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
4519 
4520 	return IRQ_WAKE_THREAD;
4521 }
4522 
dwc3_interrupt(int irq,void * _evt)4523 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4524 {
4525 	struct dwc3_event_buffer	*evt = _evt;
4526 
4527 	return dwc3_check_event_buf(evt);
4528 }
4529 
dwc3_gadget_get_irq(struct dwc3 * dwc)4530 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4531 {
4532 	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4533 	int irq;
4534 
4535 	irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
4536 	if (irq > 0)
4537 		goto out;
4538 
4539 	if (irq == -EPROBE_DEFER)
4540 		goto out;
4541 
4542 	irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
4543 	if (irq > 0)
4544 		goto out;
4545 
4546 	if (irq == -EPROBE_DEFER)
4547 		goto out;
4548 
4549 	irq = platform_get_irq(dwc3_pdev, 0);
4550 
4551 out:
4552 	return irq;
4553 }
4554 
dwc_gadget_release(struct device * dev)4555 static void dwc_gadget_release(struct device *dev)
4556 {
4557 	struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4558 
4559 	kfree(gadget);
4560 }
4561 
4562 /**
4563  * dwc3_gadget_init - initializes gadget related registers
4564  * @dwc: pointer to our controller context structure
4565  *
4566  * Returns 0 on success otherwise negative errno.
4567  */
dwc3_gadget_init(struct dwc3 * dwc)4568 int dwc3_gadget_init(struct dwc3 *dwc)
4569 {
4570 	int ret;
4571 	int irq;
4572 	struct device *dev;
4573 
4574 	irq = dwc3_gadget_get_irq(dwc);
4575 	if (irq < 0) {
4576 		ret = irq;
4577 		goto err0;
4578 	}
4579 
4580 	dwc->irq_gadget = irq;
4581 
4582 	dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
4583 					  sizeof(*dwc->ep0_trb) * 2,
4584 					  &dwc->ep0_trb_addr, GFP_KERNEL);
4585 	if (!dwc->ep0_trb) {
4586 		dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4587 		ret = -ENOMEM;
4588 		goto err0;
4589 	}
4590 
4591 	dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4592 	if (!dwc->setup_buf) {
4593 		ret = -ENOMEM;
4594 		goto err1;
4595 	}
4596 
4597 	dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
4598 			&dwc->bounce_addr, GFP_KERNEL);
4599 	if (!dwc->bounce) {
4600 		ret = -ENOMEM;
4601 		goto err2;
4602 	}
4603 
4604 	init_completion(&dwc->ep0_in_setup);
4605 	dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4606 	if (!dwc->gadget) {
4607 		ret = -ENOMEM;
4608 		goto err3;
4609 	}
4610 
4611 
4612 	usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
4613 	dev				= &dwc->gadget->dev;
4614 	dev->platform_data		= dwc;
4615 	dwc->gadget->ops		= &dwc3_gadget_ops;
4616 	dwc->gadget->speed		= USB_SPEED_UNKNOWN;
4617 	dwc->gadget->ssp_rate		= USB_SSP_GEN_UNKNOWN;
4618 	dwc->gadget->sg_supported	= true;
4619 	dwc->gadget->name		= "dwc3-gadget";
4620 	dwc->gadget->lpm_capable	= !dwc->usb2_gadget_lpm_disable;
4621 	dwc->gadget->wakeup_capable	= true;
4622 
4623 	/*
4624 	 * FIXME We might be setting max_speed to <SUPER, however versions
4625 	 * <2.20a of dwc3 have an issue with metastability (documented
4626 	 * elsewhere in this driver) which tells us we can't set max speed to
4627 	 * anything lower than SUPER.
4628 	 *
4629 	 * Because gadget.max_speed is only used by composite.c and function
4630 	 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
4631 	 * to happen so we avoid sending SuperSpeed Capability descriptor
4632 	 * together with our BOS descriptor as that could confuse host into
4633 	 * thinking we can handle super speed.
4634 	 *
4635 	 * Note that, in fact, we won't even support GetBOS requests when speed
4636 	 * is less than super speed because we don't have means, yet, to tell
4637 	 * composite.c that we are USB 2.0 + LPM ECN.
4638 	 */
4639 	if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4640 	    !dwc->dis_metastability_quirk)
4641 		dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4642 				dwc->revision);
4643 
4644 	dwc->gadget->max_speed		= dwc->maximum_speed;
4645 	dwc->gadget->max_ssp_rate	= dwc->max_ssp_rate;
4646 
4647 	/*
4648 	 * REVISIT: Here we should clear all pending IRQs to be
4649 	 * sure we're starting from a well known location.
4650 	 */
4651 
4652 	ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
4653 	if (ret)
4654 		goto err4;
4655 
4656 	ret = usb_add_gadget(dwc->gadget);
4657 	if (ret) {
4658 		dev_err(dwc->dev, "failed to add gadget\n");
4659 		goto err5;
4660 	}
4661 
4662 	if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4663 		dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
4664 	else
4665 		dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
4666 
4667 	/* No system wakeup if no gadget driver bound */
4668 	if (dwc->sys_wakeup)
4669 		device_wakeup_disable(dwc->sysdev);
4670 
4671 	return 0;
4672 
4673 err5:
4674 	dwc3_gadget_free_endpoints(dwc);
4675 err4:
4676 	usb_put_gadget(dwc->gadget);
4677 	dwc->gadget = NULL;
4678 err3:
4679 	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4680 			dwc->bounce_addr);
4681 
4682 err2:
4683 	kfree(dwc->setup_buf);
4684 
4685 err1:
4686 	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4687 			dwc->ep0_trb, dwc->ep0_trb_addr);
4688 
4689 err0:
4690 	return ret;
4691 }
4692 
4693 /* -------------------------------------------------------------------------- */
4694 
dwc3_gadget_exit(struct dwc3 * dwc)4695 void dwc3_gadget_exit(struct dwc3 *dwc)
4696 {
4697 	if (!dwc->gadget)
4698 		return;
4699 
4700 	dwc3_enable_susphy(dwc, false);
4701 	usb_del_gadget(dwc->gadget);
4702 	dwc3_gadget_free_endpoints(dwc);
4703 	usb_put_gadget(dwc->gadget);
4704 	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4705 			  dwc->bounce_addr);
4706 	kfree(dwc->setup_buf);
4707 	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4708 			  dwc->ep0_trb, dwc->ep0_trb_addr);
4709 }
4710 
dwc3_gadget_suspend(struct dwc3 * dwc)4711 int dwc3_gadget_suspend(struct dwc3 *dwc)
4712 {
4713 	unsigned long flags;
4714 	int ret;
4715 
4716 	ret = dwc3_gadget_soft_disconnect(dwc);
4717 	if (ret)
4718 		goto err;
4719 
4720 	spin_lock_irqsave(&dwc->lock, flags);
4721 	if (dwc->gadget_driver)
4722 		dwc3_disconnect_gadget(dwc);
4723 	spin_unlock_irqrestore(&dwc->lock, flags);
4724 
4725 	return 0;
4726 
4727 err:
4728 	/*
4729 	 * Attempt to reset the controller's state. Likely no
4730 	 * communication can be established until the host
4731 	 * performs a port reset.
4732 	 */
4733 	if (dwc->softconnect)
4734 		dwc3_gadget_soft_connect(dwc);
4735 
4736 	return ret;
4737 }
4738 
dwc3_gadget_resume(struct dwc3 * dwc)4739 int dwc3_gadget_resume(struct dwc3 *dwc)
4740 {
4741 	if (!dwc->gadget_driver || !dwc->softconnect)
4742 		return 0;
4743 
4744 	return dwc3_gadget_soft_connect(dwc);
4745 }
4746