1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4 *
5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6 *
7 * Authors: Felipe Balbi <balbi@ti.com>,
8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9 */
10
11 #include <linux/kernel.h>
12 #include <linux/delay.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <linux/platform_device.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/list.h>
20 #include <linux/dma-mapping.h>
21
22 #include <linux/usb/ch9.h>
23 #include <linux/usb/gadget.h>
24
25 #include "debug.h"
26 #include "core.h"
27 #include "gadget.h"
28 #include "io.h"
29
30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
32
33 /**
34 * dwc3_gadget_set_test_mode - enables usb2 test modes
35 * @dwc: pointer to our context structure
36 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
37 *
38 * Caller should take care of locking. This function will return 0 on
39 * success or -EINVAL if wrong Test Selector is passed.
40 */
dwc3_gadget_set_test_mode(struct dwc3 * dwc,int mode)41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42 {
43 u32 reg;
44
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48 switch (mode) {
49 case USB_TEST_J:
50 case USB_TEST_K:
51 case USB_TEST_SE0_NAK:
52 case USB_TEST_PACKET:
53 case USB_TEST_FORCE_ENABLE:
54 reg |= mode << 1;
55 break;
56 default:
57 return -EINVAL;
58 }
59
60 dwc3_gadget_dctl_write_safe(dwc, reg);
61
62 return 0;
63 }
64
65 /**
66 * dwc3_gadget_get_link_state - gets current state of usb link
67 * @dwc: pointer to our context structure
68 *
69 * Caller should take care of locking. This function will
70 * return the link state on success (>= 0) or -ETIMEDOUT.
71 */
dwc3_gadget_get_link_state(struct dwc3 * dwc)72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73 {
74 u32 reg;
75
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77
78 return DWC3_DSTS_USBLNKST(reg);
79 }
80
81 /**
82 * dwc3_gadget_set_link_state - sets usb link to a particular state
83 * @dwc: pointer to our context structure
84 * @state: the state to put link into
85 *
86 * Caller should take care of locking. This function will
87 * return 0 on success or -ETIMEDOUT.
88 */
dwc3_gadget_set_link_state(struct dwc3 * dwc,enum dwc3_link_state state)89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90 {
91 int retries = 10000;
92 u32 reg;
93
94 /*
95 * Wait until device controller is ready. Only applies to 1.94a and
96 * later RTL.
97 */
98 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99 while (--retries) {
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 if (reg & DWC3_DSTS_DCNRD)
102 udelay(5);
103 else
104 break;
105 }
106
107 if (retries <= 0)
108 return -ETIMEDOUT;
109 }
110
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114 /* set no action before sending new link state change */
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116
117 /* set requested state */
118 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
120
121 /*
122 * The following code is racy when called from dwc3_gadget_wakeup,
123 * and is not needed, at least on newer versions
124 */
125 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126 return 0;
127
128 /* wait for a change in DSTS */
129 retries = 10000;
130 while (--retries) {
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
132
133 if (DWC3_DSTS_USBLNKST(reg) == state)
134 return 0;
135
136 udelay(5);
137 }
138
139 return -ETIMEDOUT;
140 }
141
dwc3_ep0_reset_state(struct dwc3 * dwc)142 static void dwc3_ep0_reset_state(struct dwc3 *dwc)
143 {
144 unsigned int dir;
145
146 if (dwc->ep0state != EP0_SETUP_PHASE) {
147 dir = !!dwc->ep0_expect_in;
148 if (dwc->ep0state == EP0_DATA_PHASE)
149 dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
150 else
151 dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
152
153 dwc->eps[0]->trb_enqueue = 0;
154 dwc->eps[1]->trb_enqueue = 0;
155
156 dwc3_ep0_stall_and_restart(dwc);
157 }
158 }
159
160 /**
161 * dwc3_ep_inc_trb - increment a trb index.
162 * @index: Pointer to the TRB index to increment.
163 *
164 * The index should never point to the link TRB. After incrementing,
165 * if it is point to the link TRB, wrap around to the beginning. The
166 * link TRB is always at the last TRB entry.
167 */
dwc3_ep_inc_trb(u8 * index)168 static void dwc3_ep_inc_trb(u8 *index)
169 {
170 (*index)++;
171 if (*index == (DWC3_TRB_NUM - 1))
172 *index = 0;
173 }
174
175 /**
176 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
177 * @dep: The endpoint whose enqueue pointer we're incrementing
178 */
dwc3_ep_inc_enq(struct dwc3_ep * dep)179 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
180 {
181 dwc3_ep_inc_trb(&dep->trb_enqueue);
182 }
183
184 /**
185 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
186 * @dep: The endpoint whose enqueue pointer we're incrementing
187 */
dwc3_ep_inc_deq(struct dwc3_ep * dep)188 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
189 {
190 dwc3_ep_inc_trb(&dep->trb_dequeue);
191 }
192
dwc3_gadget_del_and_unmap_request(struct dwc3_ep * dep,struct dwc3_request * req,int status)193 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
194 struct dwc3_request *req, int status)
195 {
196 struct dwc3 *dwc = dep->dwc;
197
198 list_del(&req->list);
199 req->remaining = 0;
200 req->needs_extra_trb = false;
201 req->num_trbs = 0;
202
203 if (req->request.status == -EINPROGRESS)
204 req->request.status = status;
205
206 if (req->trb)
207 usb_gadget_unmap_request_by_dev(dwc->sysdev,
208 &req->request, req->direction);
209
210 req->trb = NULL;
211 trace_dwc3_gadget_giveback(req);
212
213 if (dep->number > 1)
214 pm_runtime_put(dwc->dev);
215 }
216
217 /**
218 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
219 * @dep: The endpoint to whom the request belongs to
220 * @req: The request we're giving back
221 * @status: completion code for the request
222 *
223 * Must be called with controller's lock held and interrupts disabled. This
224 * function will unmap @req and call its ->complete() callback to notify upper
225 * layers that it has completed.
226 */
dwc3_gadget_giveback(struct dwc3_ep * dep,struct dwc3_request * req,int status)227 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
228 int status)
229 {
230 struct dwc3 *dwc = dep->dwc;
231
232 dwc3_gadget_del_and_unmap_request(dep, req, status);
233 req->status = DWC3_REQUEST_STATUS_COMPLETED;
234
235 spin_unlock(&dwc->lock);
236 usb_gadget_giveback_request(&dep->endpoint, &req->request);
237 spin_lock(&dwc->lock);
238 }
239
240 /**
241 * dwc3_send_gadget_generic_command - issue a generic command for the controller
242 * @dwc: pointer to the controller context
243 * @cmd: the command to be issued
244 * @param: command parameter
245 *
246 * Caller should take care of locking. Issue @cmd with a given @param to @dwc
247 * and wait for its completion.
248 */
dwc3_send_gadget_generic_command(struct dwc3 * dwc,unsigned int cmd,u32 param)249 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
250 u32 param)
251 {
252 u32 timeout = 500;
253 int status = 0;
254 int ret = 0;
255 u32 reg;
256
257 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
258 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
259
260 do {
261 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
262 if (!(reg & DWC3_DGCMD_CMDACT)) {
263 status = DWC3_DGCMD_STATUS(reg);
264 if (status)
265 ret = -EINVAL;
266 break;
267 }
268 } while (--timeout);
269
270 if (!timeout) {
271 ret = -ETIMEDOUT;
272 status = -ETIMEDOUT;
273 }
274
275 trace_dwc3_gadget_generic_cmd(cmd, param, status);
276
277 return ret;
278 }
279
280 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async);
281
282 /**
283 * dwc3_send_gadget_ep_cmd - issue an endpoint command
284 * @dep: the endpoint to which the command is going to be issued
285 * @cmd: the command to be issued
286 * @params: parameters to the command
287 *
288 * Caller should handle locking. This function will issue @cmd with given
289 * @params to @dep and wait for its completion.
290 *
291 * According to the programming guide, if the link state is in L1/L2/U3,
292 * then sending the Start Transfer command may not complete. The
293 * programming guide suggested to bring the link state back to ON/U0 by
294 * performing remote wakeup prior to sending the command. However, don't
295 * initiate remote wakeup when the user/function does not send wakeup
296 * request via wakeup ops. Send the command when it's allowed.
297 *
298 * Notes:
299 * For L1 link state, issuing a command requires the clearing of
300 * GUSB2PHYCFG.SUSPENDUSB2, which turns on the signal required to complete
301 * the given command (usually within 50us). This should happen within the
302 * command timeout set by driver. No additional step is needed.
303 *
304 * For L2 or U3 link state, the gadget is in USB suspend. Care should be
305 * taken when sending Start Transfer command to ensure that it's done after
306 * USB resume.
307 */
dwc3_send_gadget_ep_cmd(struct dwc3_ep * dep,unsigned int cmd,struct dwc3_gadget_ep_cmd_params * params)308 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
309 struct dwc3_gadget_ep_cmd_params *params)
310 {
311 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
312 struct dwc3 *dwc = dep->dwc;
313 u32 timeout = 5000;
314 u32 saved_config = 0;
315 u32 reg;
316
317 int cmd_status = 0;
318 int ret = -EINVAL;
319
320 /*
321 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
322 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
323 * endpoint command.
324 *
325 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
326 * settings. Restore them after the command is completed.
327 *
328 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
329 */
330 if (dwc->gadget->speed <= USB_SPEED_HIGH ||
331 DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
332 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
333 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
334 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
335 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
336 }
337
338 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
339 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
340 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
341 }
342
343 if (saved_config)
344 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
345 }
346
347 /*
348 * For some commands such as Update Transfer command, DEPCMDPARn
349 * registers are reserved. Since the driver often sends Update Transfer
350 * command, don't write to DEPCMDPARn to avoid register write delays and
351 * improve performance.
352 */
353 if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
354 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
355 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
356 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
357 }
358
359 /*
360 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
361 * not relying on XferNotReady, we can make use of a special "No
362 * Response Update Transfer" command where we should clear both CmdAct
363 * and CmdIOC bits.
364 *
365 * With this, we don't need to wait for command completion and can
366 * straight away issue further commands to the endpoint.
367 *
368 * NOTICE: We're making an assumption that control endpoints will never
369 * make use of Update Transfer command. This is a safe assumption
370 * because we can never have more than one request at a time with
371 * Control Endpoints. If anybody changes that assumption, this chunk
372 * needs to be updated accordingly.
373 */
374 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
375 !usb_endpoint_xfer_isoc(desc))
376 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
377 else
378 cmd |= DWC3_DEPCMD_CMDACT;
379
380 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
381
382 if (!(cmd & DWC3_DEPCMD_CMDACT) ||
383 (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
384 !(cmd & DWC3_DEPCMD_CMDIOC))) {
385 ret = 0;
386 goto skip_status;
387 }
388
389 do {
390 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
391 if (!(reg & DWC3_DEPCMD_CMDACT)) {
392 cmd_status = DWC3_DEPCMD_STATUS(reg);
393
394 switch (cmd_status) {
395 case 0:
396 ret = 0;
397 break;
398 case DEPEVT_TRANSFER_NO_RESOURCE:
399 dev_WARN(dwc->dev, "No resource for %s\n",
400 dep->name);
401 ret = -EINVAL;
402 break;
403 case DEPEVT_TRANSFER_BUS_EXPIRY:
404 /*
405 * SW issues START TRANSFER command to
406 * isochronous ep with future frame interval. If
407 * future interval time has already passed when
408 * core receives the command, it will respond
409 * with an error status of 'Bus Expiry'.
410 *
411 * Instead of always returning -EINVAL, let's
412 * give a hint to the gadget driver that this is
413 * the case by returning -EAGAIN.
414 */
415 ret = -EAGAIN;
416 break;
417 default:
418 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
419 }
420
421 break;
422 }
423 } while (--timeout);
424
425 if (timeout == 0) {
426 ret = -ETIMEDOUT;
427 cmd_status = -ETIMEDOUT;
428 }
429
430 skip_status:
431 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
432
433 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
434 if (ret == 0)
435 dep->flags |= DWC3_EP_TRANSFER_STARTED;
436
437 if (ret != -ETIMEDOUT)
438 dwc3_gadget_ep_get_transfer_index(dep);
439 }
440
441 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
442 !(cmd & DWC3_DEPCMD_CMDIOC))
443 mdelay(1);
444
445 if (saved_config) {
446 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
447 reg |= saved_config;
448 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
449 }
450
451 return ret;
452 }
453
dwc3_send_clear_stall_ep_cmd(struct dwc3_ep * dep)454 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
455 {
456 struct dwc3 *dwc = dep->dwc;
457 struct dwc3_gadget_ep_cmd_params params;
458 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
459
460 /*
461 * As of core revision 2.60a the recommended programming model
462 * is to set the ClearPendIN bit when issuing a Clear Stall EP
463 * command for IN endpoints. This is to prevent an issue where
464 * some (non-compliant) hosts may not send ACK TPs for pending
465 * IN transfers due to a mishandled error condition. Synopsys
466 * STAR 9000614252.
467 */
468 if (dep->direction &&
469 !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
470 (dwc->gadget->speed >= USB_SPEED_SUPER))
471 cmd |= DWC3_DEPCMD_CLEARPENDIN;
472
473 memset(¶ms, 0, sizeof(params));
474
475 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
476 }
477
dwc3_trb_dma_offset(struct dwc3_ep * dep,struct dwc3_trb * trb)478 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
479 struct dwc3_trb *trb)
480 {
481 u32 offset = (char *) trb - (char *) dep->trb_pool;
482
483 return dep->trb_pool_dma + offset;
484 }
485
dwc3_alloc_trb_pool(struct dwc3_ep * dep)486 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
487 {
488 struct dwc3 *dwc = dep->dwc;
489
490 if (dep->trb_pool)
491 return 0;
492
493 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
494 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
495 &dep->trb_pool_dma, GFP_KERNEL);
496 if (!dep->trb_pool) {
497 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
498 dep->name);
499 return -ENOMEM;
500 }
501
502 return 0;
503 }
504
dwc3_free_trb_pool(struct dwc3_ep * dep)505 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
506 {
507 struct dwc3 *dwc = dep->dwc;
508
509 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
510 dep->trb_pool, dep->trb_pool_dma);
511
512 dep->trb_pool = NULL;
513 dep->trb_pool_dma = 0;
514 }
515
dwc3_gadget_set_xfer_resource(struct dwc3_ep * dep)516 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
517 {
518 struct dwc3_gadget_ep_cmd_params params;
519
520 memset(¶ms, 0x00, sizeof(params));
521
522 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
523
524 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
525 ¶ms);
526 }
527
528 /**
529 * dwc3_gadget_start_config - configure ep resources
530 * @dep: endpoint that is being enabled
531 *
532 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
533 * completion, it will set Transfer Resource for all available endpoints.
534 *
535 * The assignment of transfer resources cannot perfectly follow the data book
536 * due to the fact that the controller driver does not have all knowledge of the
537 * configuration in advance. It is given this information piecemeal by the
538 * composite gadget framework after every SET_CONFIGURATION and
539 * SET_INTERFACE. Trying to follow the databook programming model in this
540 * scenario can cause errors. For two reasons:
541 *
542 * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
543 * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
544 * incorrect in the scenario of multiple interfaces.
545 *
546 * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
547 * endpoint on alt setting (8.1.6).
548 *
549 * The following simplified method is used instead:
550 *
551 * All hardware endpoints can be assigned a transfer resource and this setting
552 * will stay persistent until either a core reset or hibernation. So whenever we
553 * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
554 * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
555 * guaranteed that there are as many transfer resources as endpoints.
556 *
557 * This function is called for each endpoint when it is being enabled but is
558 * triggered only when called for EP0-out, which always happens first, and which
559 * should only happen in one of the above conditions.
560 */
dwc3_gadget_start_config(struct dwc3_ep * dep)561 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
562 {
563 struct dwc3_gadget_ep_cmd_params params;
564 struct dwc3 *dwc;
565 u32 cmd;
566 int i;
567 int ret;
568
569 if (dep->number)
570 return 0;
571
572 memset(¶ms, 0x00, sizeof(params));
573 cmd = DWC3_DEPCMD_DEPSTARTCFG;
574 dwc = dep->dwc;
575
576 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
577 if (ret)
578 return ret;
579
580 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
581 struct dwc3_ep *dep = dwc->eps[i];
582
583 if (!dep)
584 continue;
585
586 ret = dwc3_gadget_set_xfer_resource(dep);
587 if (ret)
588 return ret;
589 }
590
591 return 0;
592 }
593
dwc3_gadget_set_ep_config(struct dwc3_ep * dep,unsigned int action)594 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
595 {
596 const struct usb_ss_ep_comp_descriptor *comp_desc;
597 const struct usb_endpoint_descriptor *desc;
598 struct dwc3_gadget_ep_cmd_params params;
599 struct dwc3 *dwc = dep->dwc;
600
601 comp_desc = dep->endpoint.comp_desc;
602 desc = dep->endpoint.desc;
603
604 memset(¶ms, 0x00, sizeof(params));
605
606 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
607 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
608
609 /* Burst size is only needed in SuperSpeed mode */
610 if (dwc->gadget->speed >= USB_SPEED_SUPER) {
611 u32 burst = dep->endpoint.maxburst;
612
613 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
614 }
615
616 params.param0 |= action;
617 if (action == DWC3_DEPCFG_ACTION_RESTORE)
618 params.param2 |= dep->saved_state;
619
620 if (usb_endpoint_xfer_control(desc))
621 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
622
623 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
624 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
625
626 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
627 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
628 | DWC3_DEPCFG_XFER_COMPLETE_EN
629 | DWC3_DEPCFG_STREAM_EVENT_EN;
630 dep->stream_capable = true;
631 }
632
633 if (!usb_endpoint_xfer_control(desc))
634 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
635
636 /*
637 * We are doing 1:1 mapping for endpoints, meaning
638 * Physical Endpoints 2 maps to Logical Endpoint 2 and
639 * so on. We consider the direction bit as part of the physical
640 * endpoint number. So USB endpoint 0x81 is 0x03.
641 */
642 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
643
644 /*
645 * We must use the lower 16 TX FIFOs even though
646 * HW might have more
647 */
648 if (dep->direction)
649 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
650
651 if (desc->bInterval) {
652 u8 bInterval_m1;
653
654 /*
655 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
656 *
657 * NOTE: The programming guide incorrectly stated bInterval_m1
658 * must be set to 0 when operating in fullspeed. Internally the
659 * controller does not have this limitation. See DWC_usb3x
660 * programming guide section 3.2.2.1.
661 */
662 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
663
664 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
665 dwc->gadget->speed == USB_SPEED_FULL)
666 dep->interval = desc->bInterval;
667 else
668 dep->interval = 1 << (desc->bInterval - 1);
669
670 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
671 }
672
673 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms);
674 }
675
676 /**
677 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
678 * @dwc: pointer to the DWC3 context
679 * @mult: multiplier to be used when calculating the fifo_size
680 *
681 * Calculates the size value based on the equation below:
682 *
683 * DWC3 revision 280A and prior:
684 * fifo_size = mult * (max_packet / mdwidth) + 1;
685 *
686 * DWC3 revision 290A and onwards:
687 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
688 *
689 * The max packet size is set to 1024, as the txfifo requirements mainly apply
690 * to super speed USB use cases. However, it is safe to overestimate the fifo
691 * allocations for other scenarios, i.e. high speed USB.
692 */
dwc3_gadget_calc_tx_fifo_size(struct dwc3 * dwc,int mult)693 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
694 {
695 int max_packet = 1024;
696 int fifo_size;
697 int mdwidth;
698
699 mdwidth = dwc3_mdwidth(dwc);
700
701 /* MDWIDTH is represented in bits, we need it in bytes */
702 mdwidth >>= 3;
703
704 if (DWC3_VER_IS_PRIOR(DWC3, 290A))
705 fifo_size = mult * (max_packet / mdwidth) + 1;
706 else
707 fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
708 return fifo_size;
709 }
710
711 /**
712 * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
713 * @dwc: pointer to the DWC3 context
714 *
715 * Iterates through all the endpoint registers and clears the previous txfifo
716 * allocations.
717 */
dwc3_gadget_clear_tx_fifos(struct dwc3 * dwc)718 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
719 {
720 struct dwc3_ep *dep;
721 int fifo_depth;
722 int size;
723 int num;
724
725 if (!dwc->do_fifo_resize)
726 return;
727
728 /* Read ep0IN related TXFIFO size */
729 dep = dwc->eps[1];
730 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
731 if (DWC3_IP_IS(DWC3))
732 fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
733 else
734 fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
735
736 dwc->last_fifo_depth = fifo_depth;
737 /* Clear existing TXFIFO for all IN eps except ep0 */
738 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
739 num += 2) {
740 dep = dwc->eps[num];
741 /* Don't change TXFRAMNUM on usb31 version */
742 size = DWC3_IP_IS(DWC3) ? 0 :
743 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
744 DWC31_GTXFIFOSIZ_TXFRAMNUM;
745
746 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
747 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
748 }
749 dwc->num_ep_resized = 0;
750 }
751
752 /*
753 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
754 * @dwc: pointer to our context structure
755 *
756 * This function will a best effort FIFO allocation in order
757 * to improve FIFO usage and throughput, while still allowing
758 * us to enable as many endpoints as possible.
759 *
760 * Keep in mind that this operation will be highly dependent
761 * on the configured size for RAM1 - which contains TxFifo -,
762 * the amount of endpoints enabled on coreConsultant tool, and
763 * the width of the Master Bus.
764 *
765 * In general, FIFO depths are represented with the following equation:
766 *
767 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
768 *
769 * In conjunction with dwc3_gadget_check_config(), this resizing logic will
770 * ensure that all endpoints will have enough internal memory for one max
771 * packet per endpoint.
772 */
dwc3_gadget_resize_tx_fifos(struct dwc3_ep * dep)773 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
774 {
775 struct dwc3 *dwc = dep->dwc;
776 int fifo_0_start;
777 int ram1_depth;
778 int fifo_size;
779 int min_depth;
780 int num_in_ep;
781 int remaining;
782 int num_fifos = 1;
783 int fifo;
784 int tmp;
785
786 if (!dwc->do_fifo_resize)
787 return 0;
788
789 /* resize IN endpoints except ep0 */
790 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
791 return 0;
792
793 /* bail if already resized */
794 if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
795 return 0;
796
797 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
798
799 if ((dep->endpoint.maxburst > 1 &&
800 usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||
801 usb_endpoint_xfer_isoc(dep->endpoint.desc))
802 num_fifos = 3;
803
804 if (dep->endpoint.maxburst > 6 &&
805 (usb_endpoint_xfer_bulk(dep->endpoint.desc) ||
806 usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
807 num_fifos = dwc->tx_fifo_resize_max_num;
808
809 /* FIFO size for a single buffer */
810 fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
811
812 /* Calculate the number of remaining EPs w/o any FIFO */
813 num_in_ep = dwc->max_cfg_eps;
814 num_in_ep -= dwc->num_ep_resized;
815
816 /* Reserve at least one FIFO for the number of IN EPs */
817 min_depth = num_in_ep * (fifo + 1);
818 remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
819 remaining = max_t(int, 0, remaining);
820 /*
821 * We've already reserved 1 FIFO per EP, so check what we can fit in
822 * addition to it. If there is not enough remaining space, allocate
823 * all the remaining space to the EP.
824 */
825 fifo_size = (num_fifos - 1) * fifo;
826 if (remaining < fifo_size)
827 fifo_size = remaining;
828
829 fifo_size += fifo;
830 /* Last increment according to the TX FIFO size equation */
831 fifo_size++;
832
833 /* Check if TXFIFOs start at non-zero addr */
834 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
835 fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
836
837 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
838 if (DWC3_IP_IS(DWC3))
839 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
840 else
841 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
842
843 /* Check fifo size allocation doesn't exceed available RAM size. */
844 if (dwc->last_fifo_depth >= ram1_depth) {
845 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
846 dwc->last_fifo_depth, ram1_depth,
847 dep->endpoint.name, fifo_size);
848 if (DWC3_IP_IS(DWC3))
849 fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
850 else
851 fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
852
853 dwc->last_fifo_depth -= fifo_size;
854 return -ENOMEM;
855 }
856
857 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
858 dep->flags |= DWC3_EP_TXFIFO_RESIZED;
859 dwc->num_ep_resized++;
860
861 return 0;
862 }
863
864 /**
865 * __dwc3_gadget_ep_enable - initializes a hw endpoint
866 * @dep: endpoint to be initialized
867 * @action: one of INIT, MODIFY or RESTORE
868 *
869 * Caller should take care of locking. Execute all necessary commands to
870 * initialize a HW endpoint so it can be used by a gadget driver.
871 */
__dwc3_gadget_ep_enable(struct dwc3_ep * dep,unsigned int action)872 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
873 {
874 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
875 struct dwc3 *dwc = dep->dwc;
876
877 u32 reg;
878 int ret;
879
880 if (!(dep->flags & DWC3_EP_ENABLED)) {
881 ret = dwc3_gadget_resize_tx_fifos(dep);
882 if (ret)
883 return ret;
884
885 ret = dwc3_gadget_start_config(dep);
886 if (ret)
887 return ret;
888 }
889
890 ret = dwc3_gadget_set_ep_config(dep, action);
891 if (ret)
892 return ret;
893
894 if (!(dep->flags & DWC3_EP_ENABLED)) {
895 struct dwc3_trb *trb_st_hw;
896 struct dwc3_trb *trb_link;
897
898 dep->type = usb_endpoint_type(desc);
899 dep->flags |= DWC3_EP_ENABLED;
900
901 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
902 reg |= DWC3_DALEPENA_EP(dep->number);
903 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
904
905 dep->trb_dequeue = 0;
906 dep->trb_enqueue = 0;
907
908 if (usb_endpoint_xfer_control(desc))
909 goto out;
910
911 /* Initialize the TRB ring */
912 memset(dep->trb_pool, 0,
913 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
914
915 /* Link TRB. The HWO bit is never reset */
916 trb_st_hw = &dep->trb_pool[0];
917
918 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
919 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
920 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
921 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
922 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
923 }
924
925 /*
926 * Issue StartTransfer here with no-op TRB so we can always rely on No
927 * Response Update Transfer command.
928 */
929 if (usb_endpoint_xfer_bulk(desc) ||
930 usb_endpoint_xfer_int(desc)) {
931 struct dwc3_gadget_ep_cmd_params params;
932 struct dwc3_trb *trb;
933 dma_addr_t trb_dma;
934 u32 cmd;
935
936 memset(¶ms, 0, sizeof(params));
937 trb = &dep->trb_pool[0];
938 trb_dma = dwc3_trb_dma_offset(dep, trb);
939
940 params.param0 = upper_32_bits(trb_dma);
941 params.param1 = lower_32_bits(trb_dma);
942
943 cmd = DWC3_DEPCMD_STARTTRANSFER;
944
945 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
946 if (ret < 0)
947 return ret;
948
949 if (dep->stream_capable) {
950 /*
951 * For streams, at start, there maybe a race where the
952 * host primes the endpoint before the function driver
953 * queues a request to initiate a stream. In that case,
954 * the controller will not see the prime to generate the
955 * ERDY and start stream. To workaround this, issue a
956 * no-op TRB as normal, but end it immediately. As a
957 * result, when the function driver queues the request,
958 * the next START_TRANSFER command will cause the
959 * controller to generate an ERDY to initiate the
960 * stream.
961 */
962 dwc3_stop_active_transfer(dep, true, true);
963
964 /*
965 * All stream eps will reinitiate stream on NoStream
966 * rejection until we can determine that the host can
967 * prime after the first transfer.
968 *
969 * However, if the controller is capable of
970 * TXF_FLUSH_BYPASS, then IN direction endpoints will
971 * automatically restart the stream without the driver
972 * initiation.
973 */
974 if (!dep->direction ||
975 !(dwc->hwparams.hwparams9 &
976 DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
977 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
978 }
979 }
980
981 out:
982 trace_dwc3_gadget_ep_enable(dep);
983
984 return 0;
985 }
986
dwc3_remove_requests(struct dwc3 * dwc,struct dwc3_ep * dep,int status)987 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
988 {
989 struct dwc3_request *req;
990
991 dwc3_stop_active_transfer(dep, true, false);
992
993 /* If endxfer is delayed, avoid unmapping requests */
994 if (dep->flags & DWC3_EP_DELAY_STOP)
995 return;
996
997 /* - giveback all requests to gadget driver */
998 while (!list_empty(&dep->started_list)) {
999 req = next_request(&dep->started_list);
1000
1001 dwc3_gadget_giveback(dep, req, status);
1002 }
1003
1004 while (!list_empty(&dep->pending_list)) {
1005 req = next_request(&dep->pending_list);
1006
1007 dwc3_gadget_giveback(dep, req, status);
1008 }
1009
1010 while (!list_empty(&dep->cancelled_list)) {
1011 req = next_request(&dep->cancelled_list);
1012
1013 dwc3_gadget_giveback(dep, req, status);
1014 }
1015 }
1016
1017 /**
1018 * __dwc3_gadget_ep_disable - disables a hw endpoint
1019 * @dep: the endpoint to disable
1020 *
1021 * This function undoes what __dwc3_gadget_ep_enable did and also removes
1022 * requests which are currently being processed by the hardware and those which
1023 * are not yet scheduled.
1024 *
1025 * Caller should take care of locking.
1026 */
__dwc3_gadget_ep_disable(struct dwc3_ep * dep)1027 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1028 {
1029 struct dwc3 *dwc = dep->dwc;
1030 u32 reg;
1031 u32 mask;
1032
1033 trace_dwc3_gadget_ep_disable(dep);
1034
1035 /* make sure HW endpoint isn't stalled */
1036 if (dep->flags & DWC3_EP_STALL)
1037 __dwc3_gadget_ep_set_halt(dep, 0, false);
1038
1039 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
1040 reg &= ~DWC3_DALEPENA_EP(dep->number);
1041 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
1042
1043 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
1044
1045 dep->stream_capable = false;
1046 dep->type = 0;
1047 mask = DWC3_EP_TXFIFO_RESIZED;
1048 /*
1049 * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
1050 * set. Do not clear DEP flags, so that the end transfer command will
1051 * be reattempted during the next SETUP stage.
1052 */
1053 if (dep->flags & DWC3_EP_DELAY_STOP)
1054 mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED);
1055 dep->flags &= mask;
1056
1057 /* Clear out the ep descriptors for non-ep0 */
1058 if (dep->number > 1) {
1059 dep->endpoint.comp_desc = NULL;
1060 dep->endpoint.desc = NULL;
1061 }
1062
1063 return 0;
1064 }
1065
1066 /* -------------------------------------------------------------------------- */
1067
dwc3_gadget_ep0_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)1068 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1069 const struct usb_endpoint_descriptor *desc)
1070 {
1071 return -EINVAL;
1072 }
1073
dwc3_gadget_ep0_disable(struct usb_ep * ep)1074 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1075 {
1076 return -EINVAL;
1077 }
1078
1079 /* -------------------------------------------------------------------------- */
1080
dwc3_gadget_ep_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)1081 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1082 const struct usb_endpoint_descriptor *desc)
1083 {
1084 struct dwc3_ep *dep;
1085 struct dwc3 *dwc;
1086 unsigned long flags;
1087 int ret;
1088
1089 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1090 pr_debug("dwc3: invalid parameters\n");
1091 return -EINVAL;
1092 }
1093
1094 if (!desc->wMaxPacketSize) {
1095 pr_debug("dwc3: missing wMaxPacketSize\n");
1096 return -EINVAL;
1097 }
1098
1099 dep = to_dwc3_ep(ep);
1100 dwc = dep->dwc;
1101
1102 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1103 "%s is already enabled\n",
1104 dep->name))
1105 return 0;
1106
1107 spin_lock_irqsave(&dwc->lock, flags);
1108 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1109 spin_unlock_irqrestore(&dwc->lock, flags);
1110
1111 return ret;
1112 }
1113
dwc3_gadget_ep_disable(struct usb_ep * ep)1114 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1115 {
1116 struct dwc3_ep *dep;
1117 struct dwc3 *dwc;
1118 unsigned long flags;
1119 int ret;
1120
1121 if (!ep) {
1122 pr_debug("dwc3: invalid parameters\n");
1123 return -EINVAL;
1124 }
1125
1126 dep = to_dwc3_ep(ep);
1127 dwc = dep->dwc;
1128
1129 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1130 "%s is already disabled\n",
1131 dep->name))
1132 return 0;
1133
1134 spin_lock_irqsave(&dwc->lock, flags);
1135 ret = __dwc3_gadget_ep_disable(dep);
1136 spin_unlock_irqrestore(&dwc->lock, flags);
1137
1138 return ret;
1139 }
1140
dwc3_gadget_ep_alloc_request(struct usb_ep * ep,gfp_t gfp_flags)1141 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1142 gfp_t gfp_flags)
1143 {
1144 struct dwc3_request *req;
1145 struct dwc3_ep *dep = to_dwc3_ep(ep);
1146
1147 req = kzalloc(sizeof(*req), gfp_flags);
1148 if (!req)
1149 return NULL;
1150
1151 req->direction = dep->direction;
1152 req->epnum = dep->number;
1153 req->dep = dep;
1154 req->status = DWC3_REQUEST_STATUS_UNKNOWN;
1155
1156 trace_dwc3_alloc_request(req);
1157
1158 return &req->request;
1159 }
1160
dwc3_gadget_ep_free_request(struct usb_ep * ep,struct usb_request * request)1161 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1162 struct usb_request *request)
1163 {
1164 struct dwc3_request *req = to_dwc3_request(request);
1165
1166 trace_dwc3_free_request(req);
1167 kfree(req);
1168 }
1169
1170 /**
1171 * dwc3_ep_prev_trb - returns the previous TRB in the ring
1172 * @dep: The endpoint with the TRB ring
1173 * @index: The index of the current TRB in the ring
1174 *
1175 * Returns the TRB prior to the one pointed to by the index. If the
1176 * index is 0, we will wrap backwards, skip the link TRB, and return
1177 * the one just before that.
1178 */
dwc3_ep_prev_trb(struct dwc3_ep * dep,u8 index)1179 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1180 {
1181 u8 tmp = index;
1182
1183 if (!tmp)
1184 tmp = DWC3_TRB_NUM - 1;
1185
1186 return &dep->trb_pool[tmp - 1];
1187 }
1188
dwc3_calc_trbs_left(struct dwc3_ep * dep)1189 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1190 {
1191 u8 trbs_left;
1192
1193 /*
1194 * If the enqueue & dequeue are equal then the TRB ring is either full
1195 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1196 * pending to be processed by the driver.
1197 */
1198 if (dep->trb_enqueue == dep->trb_dequeue) {
1199 /*
1200 * If there is any request remained in the started_list at
1201 * this point, that means there is no TRB available.
1202 */
1203 if (!list_empty(&dep->started_list))
1204 return 0;
1205
1206 return DWC3_TRB_NUM - 1;
1207 }
1208
1209 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1210 trbs_left &= (DWC3_TRB_NUM - 1);
1211
1212 if (dep->trb_dequeue < dep->trb_enqueue)
1213 trbs_left--;
1214
1215 return trbs_left;
1216 }
1217
1218 /**
1219 * dwc3_prepare_one_trb - setup one TRB from one request
1220 * @dep: endpoint for which this request is prepared
1221 * @req: dwc3_request pointer
1222 * @trb_length: buffer size of the TRB
1223 * @chain: should this TRB be chained to the next?
1224 * @node: only for isochronous endpoints. First TRB needs different type.
1225 * @use_bounce_buffer: set to use bounce buffer
1226 * @must_interrupt: set to interrupt on TRB completion
1227 */
dwc3_prepare_one_trb(struct dwc3_ep * dep,struct dwc3_request * req,unsigned int trb_length,unsigned int chain,unsigned int node,bool use_bounce_buffer,bool must_interrupt)1228 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1229 struct dwc3_request *req, unsigned int trb_length,
1230 unsigned int chain, unsigned int node, bool use_bounce_buffer,
1231 bool must_interrupt)
1232 {
1233 struct dwc3_trb *trb;
1234 dma_addr_t dma;
1235 unsigned int stream_id = req->request.stream_id;
1236 unsigned int short_not_ok = req->request.short_not_ok;
1237 unsigned int no_interrupt = req->request.no_interrupt;
1238 unsigned int is_last = req->request.is_last;
1239 struct dwc3 *dwc = dep->dwc;
1240 struct usb_gadget *gadget = dwc->gadget;
1241 enum usb_device_speed speed = gadget->speed;
1242
1243 if (use_bounce_buffer)
1244 dma = dep->dwc->bounce_addr;
1245 else if (req->request.num_sgs > 0)
1246 dma = sg_dma_address(req->start_sg);
1247 else
1248 dma = req->request.dma;
1249
1250 trb = &dep->trb_pool[dep->trb_enqueue];
1251
1252 if (!req->trb) {
1253 dwc3_gadget_move_started_request(req);
1254 req->trb = trb;
1255 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1256 }
1257
1258 req->num_trbs++;
1259
1260 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1261 trb->bpl = lower_32_bits(dma);
1262 trb->bph = upper_32_bits(dma);
1263
1264 switch (usb_endpoint_type(dep->endpoint.desc)) {
1265 case USB_ENDPOINT_XFER_CONTROL:
1266 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1267 break;
1268
1269 case USB_ENDPOINT_XFER_ISOC:
1270 if (!node) {
1271 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1272
1273 /*
1274 * USB Specification 2.0 Section 5.9.2 states that: "If
1275 * there is only a single transaction in the microframe,
1276 * only a DATA0 data packet PID is used. If there are
1277 * two transactions per microframe, DATA1 is used for
1278 * the first transaction data packet and DATA0 is used
1279 * for the second transaction data packet. If there are
1280 * three transactions per microframe, DATA2 is used for
1281 * the first transaction data packet, DATA1 is used for
1282 * the second, and DATA0 is used for the third."
1283 *
1284 * IOW, we should satisfy the following cases:
1285 *
1286 * 1) length <= maxpacket
1287 * - DATA0
1288 *
1289 * 2) maxpacket < length <= (2 * maxpacket)
1290 * - DATA1, DATA0
1291 *
1292 * 3) (2 * maxpacket) < length <= (3 * maxpacket)
1293 * - DATA2, DATA1, DATA0
1294 */
1295 if (speed == USB_SPEED_HIGH) {
1296 struct usb_ep *ep = &dep->endpoint;
1297 unsigned int mult = 2;
1298 unsigned int maxp = usb_endpoint_maxp(ep->desc);
1299
1300 if (req->request.length <= (2 * maxp))
1301 mult--;
1302
1303 if (req->request.length <= maxp)
1304 mult--;
1305
1306 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1307 }
1308 } else {
1309 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1310 }
1311
1312 if (!no_interrupt && !chain)
1313 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1314 break;
1315
1316 case USB_ENDPOINT_XFER_BULK:
1317 case USB_ENDPOINT_XFER_INT:
1318 trb->ctrl = DWC3_TRBCTL_NORMAL;
1319 break;
1320 default:
1321 /*
1322 * This is only possible with faulty memory because we
1323 * checked it already :)
1324 */
1325 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1326 usb_endpoint_type(dep->endpoint.desc));
1327 }
1328
1329 /*
1330 * Enable Continue on Short Packet
1331 * when endpoint is not a stream capable
1332 */
1333 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1334 if (!dep->stream_capable)
1335 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1336
1337 if (short_not_ok)
1338 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1339 }
1340
1341 /* All TRBs setup for MST must set CSP=1 when LST=0 */
1342 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1343 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1344
1345 if ((!no_interrupt && !chain) || must_interrupt)
1346 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1347
1348 if (chain)
1349 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1350 else if (dep->stream_capable && is_last &&
1351 !DWC3_MST_CAPABLE(&dwc->hwparams))
1352 trb->ctrl |= DWC3_TRB_CTRL_LST;
1353
1354 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1355 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1356
1357 /*
1358 * As per data book 4.2.3.2TRB Control Bit Rules section
1359 *
1360 * The controller autonomously checks the HWO field of a TRB to determine if the
1361 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1362 * is valid before setting the HWO field to '1'. In most systems, this means that
1363 * software must update the fourth DWORD of a TRB last.
1364 *
1365 * However there is a possibility of CPU re-ordering here which can cause
1366 * controller to observe the HWO bit set prematurely.
1367 * Add a write memory barrier to prevent CPU re-ordering.
1368 */
1369 wmb();
1370 trb->ctrl |= DWC3_TRB_CTRL_HWO;
1371
1372 dwc3_ep_inc_enq(dep);
1373
1374 trace_dwc3_prepare_trb(dep, trb);
1375 }
1376
dwc3_needs_extra_trb(struct dwc3_ep * dep,struct dwc3_request * req)1377 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1378 {
1379 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1380 unsigned int rem = req->request.length % maxp;
1381
1382 if ((req->request.length && req->request.zero && !rem &&
1383 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1384 (!req->direction && rem))
1385 return true;
1386
1387 return false;
1388 }
1389
1390 /**
1391 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1392 * @dep: The endpoint that the request belongs to
1393 * @req: The request to prepare
1394 * @entry_length: The last SG entry size
1395 * @node: Indicates whether this is not the first entry (for isoc only)
1396 *
1397 * Return the number of TRBs prepared.
1398 */
dwc3_prepare_last_sg(struct dwc3_ep * dep,struct dwc3_request * req,unsigned int entry_length,unsigned int node)1399 static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1400 struct dwc3_request *req, unsigned int entry_length,
1401 unsigned int node)
1402 {
1403 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1404 unsigned int rem = req->request.length % maxp;
1405 unsigned int num_trbs = 1;
1406
1407 if (dwc3_needs_extra_trb(dep, req))
1408 num_trbs++;
1409
1410 if (dwc3_calc_trbs_left(dep) < num_trbs)
1411 return 0;
1412
1413 req->needs_extra_trb = num_trbs > 1;
1414
1415 /* Prepare a normal TRB */
1416 if (req->direction || req->request.length)
1417 dwc3_prepare_one_trb(dep, req, entry_length,
1418 req->needs_extra_trb, node, false, false);
1419
1420 /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1421 if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1422 dwc3_prepare_one_trb(dep, req,
1423 req->direction ? 0 : maxp - rem,
1424 false, 1, true, false);
1425
1426 return num_trbs;
1427 }
1428
dwc3_prepare_trbs_sg(struct dwc3_ep * dep,struct dwc3_request * req)1429 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1430 struct dwc3_request *req)
1431 {
1432 struct scatterlist *sg = req->start_sg;
1433 struct scatterlist *s;
1434 int i;
1435 unsigned int length = req->request.length;
1436 unsigned int remaining = req->request.num_mapped_sgs
1437 - req->num_queued_sgs;
1438 unsigned int num_trbs = req->num_trbs;
1439 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1440
1441 /*
1442 * If we resume preparing the request, then get the remaining length of
1443 * the request and resume where we left off.
1444 */
1445 for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1446 length -= sg_dma_len(s);
1447
1448 for_each_sg(sg, s, remaining, i) {
1449 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1450 unsigned int trb_length;
1451 bool must_interrupt = false;
1452 bool last_sg = false;
1453
1454 trb_length = min_t(unsigned int, length, sg_dma_len(s));
1455
1456 length -= trb_length;
1457
1458 /*
1459 * IOMMU driver is coalescing the list of sgs which shares a
1460 * page boundary into one and giving it to USB driver. With
1461 * this the number of sgs mapped is not equal to the number of
1462 * sgs passed. So mark the chain bit to false if it isthe last
1463 * mapped sg.
1464 */
1465 if ((i == remaining - 1) || !length)
1466 last_sg = true;
1467
1468 if (!num_trbs_left)
1469 break;
1470
1471 if (last_sg) {
1472 if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1473 break;
1474 } else {
1475 /*
1476 * Look ahead to check if we have enough TRBs for the
1477 * next SG entry. If not, set interrupt on this TRB to
1478 * resume preparing the next SG entry when more TRBs are
1479 * free.
1480 */
1481 if (num_trbs_left == 1 || (needs_extra_trb &&
1482 num_trbs_left <= 2 &&
1483 sg_dma_len(sg_next(s)) >= length)) {
1484 struct dwc3_request *r;
1485
1486 /* Check if previous requests already set IOC */
1487 list_for_each_entry(r, &dep->started_list, list) {
1488 if (r != req && !r->request.no_interrupt)
1489 break;
1490
1491 if (r == req)
1492 must_interrupt = true;
1493 }
1494 }
1495
1496 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1497 must_interrupt);
1498 }
1499
1500 /*
1501 * There can be a situation where all sgs in sglist are not
1502 * queued because of insufficient trb number. To handle this
1503 * case, update start_sg to next sg to be queued, so that
1504 * we have free trbs we can continue queuing from where we
1505 * previously stopped
1506 */
1507 if (!last_sg)
1508 req->start_sg = sg_next(s);
1509
1510 req->num_queued_sgs++;
1511 req->num_pending_sgs--;
1512
1513 /*
1514 * The number of pending SG entries may not correspond to the
1515 * number of mapped SG entries. If all the data are queued, then
1516 * don't include unused SG entries.
1517 */
1518 if (length == 0) {
1519 req->num_pending_sgs = 0;
1520 break;
1521 }
1522
1523 if (must_interrupt)
1524 break;
1525 }
1526
1527 return req->num_trbs - num_trbs;
1528 }
1529
dwc3_prepare_trbs_linear(struct dwc3_ep * dep,struct dwc3_request * req)1530 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1531 struct dwc3_request *req)
1532 {
1533 return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1534 }
1535
1536 /*
1537 * dwc3_prepare_trbs - setup TRBs from requests
1538 * @dep: endpoint for which requests are being prepared
1539 *
1540 * The function goes through the requests list and sets up TRBs for the
1541 * transfers. The function returns once there are no more TRBs available or
1542 * it runs out of requests.
1543 *
1544 * Returns the number of TRBs prepared or negative errno.
1545 */
dwc3_prepare_trbs(struct dwc3_ep * dep)1546 static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1547 {
1548 struct dwc3_request *req, *n;
1549 int ret = 0;
1550
1551 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1552
1553 /*
1554 * We can get in a situation where there's a request in the started list
1555 * but there weren't enough TRBs to fully kick it in the first time
1556 * around, so it has been waiting for more TRBs to be freed up.
1557 *
1558 * In that case, we should check if we have a request with pending_sgs
1559 * in the started list and prepare TRBs for that request first,
1560 * otherwise we will prepare TRBs completely out of order and that will
1561 * break things.
1562 */
1563 list_for_each_entry(req, &dep->started_list, list) {
1564 if (req->num_pending_sgs > 0) {
1565 ret = dwc3_prepare_trbs_sg(dep, req);
1566 if (!ret || req->num_pending_sgs)
1567 return ret;
1568 }
1569
1570 if (!dwc3_calc_trbs_left(dep))
1571 return ret;
1572
1573 /*
1574 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1575 * burst capability may try to read and use TRBs beyond the
1576 * active transfer instead of stopping.
1577 */
1578 if (dep->stream_capable && req->request.is_last &&
1579 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1580 return ret;
1581 }
1582
1583 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1584 struct dwc3 *dwc = dep->dwc;
1585
1586 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1587 dep->direction);
1588 if (ret)
1589 return ret;
1590
1591 req->sg = req->request.sg;
1592 req->start_sg = req->sg;
1593 req->num_queued_sgs = 0;
1594 req->num_pending_sgs = req->request.num_mapped_sgs;
1595
1596 if (req->num_pending_sgs > 0) {
1597 ret = dwc3_prepare_trbs_sg(dep, req);
1598 if (req->num_pending_sgs)
1599 return ret;
1600 } else {
1601 ret = dwc3_prepare_trbs_linear(dep, req);
1602 }
1603
1604 if (!ret || !dwc3_calc_trbs_left(dep))
1605 return ret;
1606
1607 /*
1608 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1609 * burst capability may try to read and use TRBs beyond the
1610 * active transfer instead of stopping.
1611 */
1612 if (dep->stream_capable && req->request.is_last &&
1613 !DWC3_MST_CAPABLE(&dwc->hwparams))
1614 return ret;
1615 }
1616
1617 return ret;
1618 }
1619
1620 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1621
__dwc3_gadget_kick_transfer(struct dwc3_ep * dep)1622 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1623 {
1624 struct dwc3_gadget_ep_cmd_params params;
1625 struct dwc3_request *req;
1626 int starting;
1627 int ret;
1628 u32 cmd;
1629
1630 /*
1631 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1632 * This happens when we need to stop and restart a transfer such as in
1633 * the case of reinitiating a stream or retrying an isoc transfer.
1634 */
1635 ret = dwc3_prepare_trbs(dep);
1636 if (ret < 0)
1637 return ret;
1638
1639 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1640
1641 /*
1642 * If there's no new TRB prepared and we don't need to restart a
1643 * transfer, there's no need to update the transfer.
1644 */
1645 if (!ret && !starting)
1646 return ret;
1647
1648 req = next_request(&dep->started_list);
1649 if (!req) {
1650 dep->flags |= DWC3_EP_PENDING_REQUEST;
1651 return 0;
1652 }
1653
1654 memset(¶ms, 0, sizeof(params));
1655
1656 if (starting) {
1657 params.param0 = upper_32_bits(req->trb_dma);
1658 params.param1 = lower_32_bits(req->trb_dma);
1659 cmd = DWC3_DEPCMD_STARTTRANSFER;
1660
1661 if (dep->stream_capable)
1662 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1663
1664 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1665 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1666 } else {
1667 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1668 DWC3_DEPCMD_PARAM(dep->resource_index);
1669 }
1670
1671 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1672 if (ret < 0) {
1673 struct dwc3_request *tmp;
1674
1675 if (ret == -EAGAIN)
1676 return ret;
1677
1678 dwc3_stop_active_transfer(dep, true, true);
1679
1680 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1681 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1682
1683 /* If ep isn't started, then there's no end transfer pending */
1684 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1685 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1686
1687 return ret;
1688 }
1689
1690 if (dep->stream_capable && req->request.is_last &&
1691 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1692 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1693
1694 return 0;
1695 }
1696
__dwc3_gadget_get_frame(struct dwc3 * dwc)1697 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1698 {
1699 u32 reg;
1700
1701 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1702 return DWC3_DSTS_SOFFN(reg);
1703 }
1704
1705 /**
1706 * __dwc3_stop_active_transfer - stop the current active transfer
1707 * @dep: isoc endpoint
1708 * @force: set forcerm bit in the command
1709 * @interrupt: command complete interrupt after End Transfer command
1710 *
1711 * When setting force, the ForceRM bit will be set. In that case
1712 * the controller won't update the TRB progress on command
1713 * completion. It also won't clear the HWO bit in the TRB.
1714 * The command will also not complete immediately in that case.
1715 */
__dwc3_stop_active_transfer(struct dwc3_ep * dep,bool force,bool interrupt)1716 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1717 {
1718 struct dwc3_gadget_ep_cmd_params params;
1719 u32 cmd;
1720 int ret;
1721
1722 cmd = DWC3_DEPCMD_ENDTRANSFER;
1723 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1724 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1725 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1726 memset(¶ms, 0, sizeof(params));
1727 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1728 /*
1729 * If the End Transfer command was timed out while the device is
1730 * not in SETUP phase, it's possible that an incoming Setup packet
1731 * may prevent the command's completion. Let's retry when the
1732 * ep0state returns to EP0_SETUP_PHASE.
1733 */
1734 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
1735 dep->flags |= DWC3_EP_DELAY_STOP;
1736 return 0;
1737 }
1738 WARN_ON_ONCE(ret);
1739 dep->resource_index = 0;
1740
1741 if (!interrupt)
1742 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1743 else if (!ret)
1744 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1745
1746 dep->flags &= ~DWC3_EP_DELAY_STOP;
1747 return ret;
1748 }
1749
1750 /**
1751 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1752 * @dep: isoc endpoint
1753 *
1754 * This function tests for the correct combination of BIT[15:14] from the 16-bit
1755 * microframe number reported by the XferNotReady event for the future frame
1756 * number to start the isoc transfer.
1757 *
1758 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1759 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1760 * XferNotReady event are invalid. The driver uses this number to schedule the
1761 * isochronous transfer and passes it to the START TRANSFER command. Because
1762 * this number is invalid, the command may fail. If BIT[15:14] matches the
1763 * internal 16-bit microframe, the START TRANSFER command will pass and the
1764 * transfer will start at the scheduled time, if it is off by 1, the command
1765 * will still pass, but the transfer will start 2 seconds in the future. For all
1766 * other conditions, the START TRANSFER command will fail with bus-expiry.
1767 *
1768 * In order to workaround this issue, we can test for the correct combination of
1769 * BIT[15:14] by sending START TRANSFER commands with different values of
1770 * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1771 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1772 * As the result, within the 4 possible combinations for BIT[15:14], there will
1773 * be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1774 * command status will result in a 2-second delay start. The smaller BIT[15:14]
1775 * value is the correct combination.
1776 *
1777 * Since there are only 4 outcomes and the results are ordered, we can simply
1778 * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1779 * deduce the smaller successful combination.
1780 *
1781 * Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1782 * of BIT[15:14]. The correct combination is as follow:
1783 *
1784 * if test0 fails and test1 passes, BIT[15:14] is 'b01
1785 * if test0 fails and test1 fails, BIT[15:14] is 'b10
1786 * if test0 passes and test1 fails, BIT[15:14] is 'b11
1787 * if test0 passes and test1 passes, BIT[15:14] is 'b00
1788 *
1789 * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1790 * endpoints.
1791 */
dwc3_gadget_start_isoc_quirk(struct dwc3_ep * dep)1792 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1793 {
1794 int cmd_status = 0;
1795 bool test0;
1796 bool test1;
1797
1798 while (dep->combo_num < 2) {
1799 struct dwc3_gadget_ep_cmd_params params;
1800 u32 test_frame_number;
1801 u32 cmd;
1802
1803 /*
1804 * Check if we can start isoc transfer on the next interval or
1805 * 4 uframes in the future with BIT[15:14] as dep->combo_num
1806 */
1807 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1808 test_frame_number |= dep->combo_num << 14;
1809 test_frame_number += max_t(u32, 4, dep->interval);
1810
1811 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1812 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1813
1814 cmd = DWC3_DEPCMD_STARTTRANSFER;
1815 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1816 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1817
1818 /* Redo if some other failure beside bus-expiry is received */
1819 if (cmd_status && cmd_status != -EAGAIN) {
1820 dep->start_cmd_status = 0;
1821 dep->combo_num = 0;
1822 return 0;
1823 }
1824
1825 /* Store the first test status */
1826 if (dep->combo_num == 0)
1827 dep->start_cmd_status = cmd_status;
1828
1829 dep->combo_num++;
1830
1831 /*
1832 * End the transfer if the START_TRANSFER command is successful
1833 * to wait for the next XferNotReady to test the command again
1834 */
1835 if (cmd_status == 0) {
1836 dwc3_stop_active_transfer(dep, true, true);
1837 return 0;
1838 }
1839 }
1840
1841 /* test0 and test1 are both completed at this point */
1842 test0 = (dep->start_cmd_status == 0);
1843 test1 = (cmd_status == 0);
1844
1845 if (!test0 && test1)
1846 dep->combo_num = 1;
1847 else if (!test0 && !test1)
1848 dep->combo_num = 2;
1849 else if (test0 && !test1)
1850 dep->combo_num = 3;
1851 else if (test0 && test1)
1852 dep->combo_num = 0;
1853
1854 dep->frame_number &= DWC3_FRNUMBER_MASK;
1855 dep->frame_number |= dep->combo_num << 14;
1856 dep->frame_number += max_t(u32, 4, dep->interval);
1857
1858 /* Reinitialize test variables */
1859 dep->start_cmd_status = 0;
1860 dep->combo_num = 0;
1861
1862 return __dwc3_gadget_kick_transfer(dep);
1863 }
1864
__dwc3_gadget_start_isoc(struct dwc3_ep * dep)1865 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1866 {
1867 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1868 struct dwc3 *dwc = dep->dwc;
1869 int ret;
1870 int i;
1871
1872 if (list_empty(&dep->pending_list) &&
1873 list_empty(&dep->started_list)) {
1874 dep->flags |= DWC3_EP_PENDING_REQUEST;
1875 return -EAGAIN;
1876 }
1877
1878 if (!dwc->dis_start_transfer_quirk &&
1879 (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1880 DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1881 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1882 return dwc3_gadget_start_isoc_quirk(dep);
1883 }
1884
1885 if (desc->bInterval <= 14 &&
1886 dwc->gadget->speed >= USB_SPEED_HIGH) {
1887 u32 frame = __dwc3_gadget_get_frame(dwc);
1888 bool rollover = frame <
1889 (dep->frame_number & DWC3_FRNUMBER_MASK);
1890
1891 /*
1892 * frame_number is set from XferNotReady and may be already
1893 * out of date. DSTS only provides the lower 14 bit of the
1894 * current frame number. So add the upper two bits of
1895 * frame_number and handle a possible rollover.
1896 * This will provide the correct frame_number unless more than
1897 * rollover has happened since XferNotReady.
1898 */
1899
1900 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1901 frame;
1902 if (rollover)
1903 dep->frame_number += BIT(14);
1904 }
1905
1906 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1907 int future_interval = i + 1;
1908
1909 /* Give the controller at least 500us to schedule transfers */
1910 if (desc->bInterval < 3)
1911 future_interval += 3 - desc->bInterval;
1912
1913 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1914
1915 ret = __dwc3_gadget_kick_transfer(dep);
1916 if (ret != -EAGAIN)
1917 break;
1918 }
1919
1920 /*
1921 * After a number of unsuccessful start attempts due to bus-expiry
1922 * status, issue END_TRANSFER command and retry on the next XferNotReady
1923 * event.
1924 */
1925 if (ret == -EAGAIN)
1926 ret = __dwc3_stop_active_transfer(dep, false, true);
1927
1928 return ret;
1929 }
1930
__dwc3_gadget_ep_queue(struct dwc3_ep * dep,struct dwc3_request * req)1931 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1932 {
1933 struct dwc3 *dwc = dep->dwc;
1934
1935 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1936 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1937 dep->name);
1938 return -ESHUTDOWN;
1939 }
1940
1941 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1942 &req->request, req->dep->name))
1943 return -EINVAL;
1944
1945 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1946 "%s: request %pK already in flight\n",
1947 dep->name, &req->request))
1948 return -EINVAL;
1949
1950 pm_runtime_get(dwc->dev);
1951
1952 req->request.actual = 0;
1953 req->request.status = -EINPROGRESS;
1954
1955 trace_dwc3_ep_queue(req);
1956
1957 list_add_tail(&req->list, &dep->pending_list);
1958 req->status = DWC3_REQUEST_STATUS_QUEUED;
1959
1960 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1961 return 0;
1962
1963 /*
1964 * Start the transfer only after the END_TRANSFER is completed
1965 * and endpoint STALL is cleared.
1966 */
1967 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1968 (dep->flags & DWC3_EP_WEDGE) ||
1969 (dep->flags & DWC3_EP_DELAY_STOP) ||
1970 (dep->flags & DWC3_EP_STALL)) {
1971 dep->flags |= DWC3_EP_DELAY_START;
1972 return 0;
1973 }
1974
1975 /*
1976 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1977 * wait for a XferNotReady event so we will know what's the current
1978 * (micro-)frame number.
1979 *
1980 * Without this trick, we are very, very likely gonna get Bus Expiry
1981 * errors which will force us issue EndTransfer command.
1982 */
1983 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1984 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1985 if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1986 return __dwc3_gadget_start_isoc(dep);
1987
1988 return 0;
1989 }
1990 }
1991
1992 __dwc3_gadget_kick_transfer(dep);
1993
1994 return 0;
1995 }
1996
dwc3_gadget_ep_queue(struct usb_ep * ep,struct usb_request * request,gfp_t gfp_flags)1997 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1998 gfp_t gfp_flags)
1999 {
2000 struct dwc3_request *req = to_dwc3_request(request);
2001 struct dwc3_ep *dep = to_dwc3_ep(ep);
2002 struct dwc3 *dwc = dep->dwc;
2003
2004 unsigned long flags;
2005
2006 int ret;
2007
2008 spin_lock_irqsave(&dwc->lock, flags);
2009 ret = __dwc3_gadget_ep_queue(dep, req);
2010 spin_unlock_irqrestore(&dwc->lock, flags);
2011
2012 return ret;
2013 }
2014
dwc3_gadget_ep_skip_trbs(struct dwc3_ep * dep,struct dwc3_request * req)2015 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
2016 {
2017 int i;
2018
2019 /* If req->trb is not set, then the request has not started */
2020 if (!req->trb)
2021 return;
2022
2023 /*
2024 * If request was already started, this means we had to
2025 * stop the transfer. With that we also need to ignore
2026 * all TRBs used by the request, however TRBs can only
2027 * be modified after completion of END_TRANSFER
2028 * command. So what we do here is that we wait for
2029 * END_TRANSFER completion and only after that, we jump
2030 * over TRBs by clearing HWO and incrementing dequeue
2031 * pointer.
2032 */
2033 for (i = 0; i < req->num_trbs; i++) {
2034 struct dwc3_trb *trb;
2035
2036 trb = &dep->trb_pool[dep->trb_dequeue];
2037 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2038 dwc3_ep_inc_deq(dep);
2039 }
2040
2041 req->num_trbs = 0;
2042 }
2043
dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep * dep)2044 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2045 {
2046 struct dwc3_request *req;
2047 struct dwc3 *dwc = dep->dwc;
2048
2049 while (!list_empty(&dep->cancelled_list)) {
2050 req = next_request(&dep->cancelled_list);
2051 dwc3_gadget_ep_skip_trbs(dep, req);
2052 switch (req->status) {
2053 case DWC3_REQUEST_STATUS_DISCONNECTED:
2054 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
2055 break;
2056 case DWC3_REQUEST_STATUS_DEQUEUED:
2057 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2058 break;
2059 case DWC3_REQUEST_STATUS_STALLED:
2060 dwc3_gadget_giveback(dep, req, -EPIPE);
2061 break;
2062 default:
2063 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2064 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2065 break;
2066 }
2067 /*
2068 * The endpoint is disabled, let the dwc3_remove_requests()
2069 * handle the cleanup.
2070 */
2071 if (!dep->endpoint.desc)
2072 break;
2073 }
2074 }
2075
dwc3_gadget_ep_dequeue(struct usb_ep * ep,struct usb_request * request)2076 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2077 struct usb_request *request)
2078 {
2079 struct dwc3_request *req = to_dwc3_request(request);
2080 struct dwc3_request *r = NULL;
2081
2082 struct dwc3_ep *dep = to_dwc3_ep(ep);
2083 struct dwc3 *dwc = dep->dwc;
2084
2085 unsigned long flags;
2086 int ret = 0;
2087
2088 trace_dwc3_ep_dequeue(req);
2089
2090 spin_lock_irqsave(&dwc->lock, flags);
2091
2092 list_for_each_entry(r, &dep->cancelled_list, list) {
2093 if (r == req)
2094 goto out;
2095 }
2096
2097 list_for_each_entry(r, &dep->pending_list, list) {
2098 if (r == req) {
2099 /*
2100 * Explicitly check for EP0/1 as dequeue for those
2101 * EPs need to be handled differently. Control EP
2102 * only deals with one USB req, and giveback will
2103 * occur during dwc3_ep0_stall_and_restart(). EP0
2104 * requests are never added to started_list.
2105 */
2106 if (dep->number > 1)
2107 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2108 else
2109 dwc3_ep0_reset_state(dwc);
2110 goto out;
2111 }
2112 }
2113
2114 list_for_each_entry(r, &dep->started_list, list) {
2115 if (r == req) {
2116 struct dwc3_request *t;
2117
2118 /* wait until it is processed */
2119 dwc3_stop_active_transfer(dep, true, true);
2120
2121 /*
2122 * Remove any started request if the transfer is
2123 * cancelled.
2124 */
2125 list_for_each_entry_safe(r, t, &dep->started_list, list)
2126 dwc3_gadget_move_cancelled_request(r,
2127 DWC3_REQUEST_STATUS_DEQUEUED);
2128
2129 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2130
2131 goto out;
2132 }
2133 }
2134
2135 dev_err(dwc->dev, "request %pK was not queued to %s\n",
2136 request, ep->name);
2137 ret = -EINVAL;
2138 out:
2139 spin_unlock_irqrestore(&dwc->lock, flags);
2140
2141 return ret;
2142 }
2143
__dwc3_gadget_ep_set_halt(struct dwc3_ep * dep,int value,int protocol)2144 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2145 {
2146 struct dwc3_gadget_ep_cmd_params params;
2147 struct dwc3 *dwc = dep->dwc;
2148 struct dwc3_request *req;
2149 struct dwc3_request *tmp;
2150 int ret;
2151
2152 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2153 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2154 return -EINVAL;
2155 }
2156
2157 memset(¶ms, 0x00, sizeof(params));
2158
2159 if (value) {
2160 struct dwc3_trb *trb;
2161
2162 unsigned int transfer_in_flight;
2163 unsigned int started;
2164
2165 if (dep->number > 1)
2166 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
2167 else
2168 trb = &dwc->ep0_trb[dep->trb_enqueue];
2169
2170 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2171 started = !list_empty(&dep->started_list);
2172
2173 if (!protocol && ((dep->direction && transfer_in_flight) ||
2174 (!dep->direction && started))) {
2175 return -EAGAIN;
2176 }
2177
2178 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2179 ¶ms);
2180 if (ret)
2181 dev_err(dwc->dev, "failed to set STALL on %s\n",
2182 dep->name);
2183 else
2184 dep->flags |= DWC3_EP_STALL;
2185 } else {
2186 /*
2187 * Don't issue CLEAR_STALL command to control endpoints. The
2188 * controller automatically clears the STALL when it receives
2189 * the SETUP token.
2190 */
2191 if (dep->number <= 1) {
2192 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2193 return 0;
2194 }
2195
2196 dwc3_stop_active_transfer(dep, true, true);
2197
2198 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2199 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2200
2201 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2202 (dep->flags & DWC3_EP_DELAY_STOP)) {
2203 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2204 if (protocol)
2205 dwc->clear_stall_protocol = dep->number;
2206
2207 return 0;
2208 }
2209
2210 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2211
2212 ret = dwc3_send_clear_stall_ep_cmd(dep);
2213 if (ret) {
2214 dev_err(dwc->dev, "failed to clear STALL on %s\n",
2215 dep->name);
2216 return ret;
2217 }
2218
2219 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2220
2221 if ((dep->flags & DWC3_EP_DELAY_START) &&
2222 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2223 __dwc3_gadget_kick_transfer(dep);
2224
2225 dep->flags &= ~DWC3_EP_DELAY_START;
2226 }
2227
2228 return ret;
2229 }
2230
dwc3_gadget_ep_set_halt(struct usb_ep * ep,int value)2231 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2232 {
2233 struct dwc3_ep *dep = to_dwc3_ep(ep);
2234 struct dwc3 *dwc = dep->dwc;
2235
2236 unsigned long flags;
2237
2238 int ret;
2239
2240 spin_lock_irqsave(&dwc->lock, flags);
2241 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
2242 spin_unlock_irqrestore(&dwc->lock, flags);
2243
2244 return ret;
2245 }
2246
dwc3_gadget_ep_set_wedge(struct usb_ep * ep)2247 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2248 {
2249 struct dwc3_ep *dep = to_dwc3_ep(ep);
2250 struct dwc3 *dwc = dep->dwc;
2251 unsigned long flags;
2252 int ret;
2253
2254 spin_lock_irqsave(&dwc->lock, flags);
2255 dep->flags |= DWC3_EP_WEDGE;
2256
2257 if (dep->number == 0 || dep->number == 1)
2258 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
2259 else
2260 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2261 spin_unlock_irqrestore(&dwc->lock, flags);
2262
2263 return ret;
2264 }
2265
2266 /* -------------------------------------------------------------------------- */
2267
2268 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2269 .bLength = USB_DT_ENDPOINT_SIZE,
2270 .bDescriptorType = USB_DT_ENDPOINT,
2271 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
2272 };
2273
2274 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2275 .enable = dwc3_gadget_ep0_enable,
2276 .disable = dwc3_gadget_ep0_disable,
2277 .alloc_request = dwc3_gadget_ep_alloc_request,
2278 .free_request = dwc3_gadget_ep_free_request,
2279 .queue = dwc3_gadget_ep0_queue,
2280 .dequeue = dwc3_gadget_ep_dequeue,
2281 .set_halt = dwc3_gadget_ep0_set_halt,
2282 .set_wedge = dwc3_gadget_ep_set_wedge,
2283 };
2284
2285 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2286 .enable = dwc3_gadget_ep_enable,
2287 .disable = dwc3_gadget_ep_disable,
2288 .alloc_request = dwc3_gadget_ep_alloc_request,
2289 .free_request = dwc3_gadget_ep_free_request,
2290 .queue = dwc3_gadget_ep_queue,
2291 .dequeue = dwc3_gadget_ep_dequeue,
2292 .set_halt = dwc3_gadget_ep_set_halt,
2293 .set_wedge = dwc3_gadget_ep_set_wedge,
2294 };
2295
2296 /* -------------------------------------------------------------------------- */
2297
dwc3_gadget_enable_linksts_evts(struct dwc3 * dwc,bool set)2298 static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set)
2299 {
2300 u32 reg;
2301
2302 if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2303 return;
2304
2305 reg = dwc3_readl(dwc->regs, DWC3_DEVTEN);
2306 if (set)
2307 reg |= DWC3_DEVTEN_ULSTCNGEN;
2308 else
2309 reg &= ~DWC3_DEVTEN_ULSTCNGEN;
2310
2311 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2312 }
2313
dwc3_gadget_get_frame(struct usb_gadget * g)2314 static int dwc3_gadget_get_frame(struct usb_gadget *g)
2315 {
2316 struct dwc3 *dwc = gadget_to_dwc(g);
2317
2318 return __dwc3_gadget_get_frame(dwc);
2319 }
2320
__dwc3_gadget_wakeup(struct dwc3 * dwc,bool async)2321 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async)
2322 {
2323 int retries;
2324
2325 int ret;
2326 u32 reg;
2327
2328 u8 link_state;
2329
2330 /*
2331 * According to the Databook Remote wakeup request should
2332 * be issued only when the device is in early suspend state.
2333 *
2334 * We can check that via USB Link State bits in DSTS register.
2335 */
2336 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2337
2338 link_state = DWC3_DSTS_USBLNKST(reg);
2339
2340 switch (link_state) {
2341 case DWC3_LINK_STATE_RESET:
2342 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
2343 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
2344 case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */
2345 case DWC3_LINK_STATE_U1:
2346 case DWC3_LINK_STATE_RESUME:
2347 break;
2348 default:
2349 return -EINVAL;
2350 }
2351
2352 if (async)
2353 dwc3_gadget_enable_linksts_evts(dwc, true);
2354
2355 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2356 if (ret < 0) {
2357 dev_err(dwc->dev, "failed to put link in Recovery\n");
2358 dwc3_gadget_enable_linksts_evts(dwc, false);
2359 return ret;
2360 }
2361
2362 /* Recent versions do this automatically */
2363 if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2364 /* write zeroes to Link Change Request */
2365 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2366 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2367 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2368 }
2369
2370 /*
2371 * Since link status change events are enabled we will receive
2372 * an U0 event when wakeup is successful. So bail out.
2373 */
2374 if (async)
2375 return 0;
2376
2377 /* poll until Link State changes to ON */
2378 retries = 20000;
2379
2380 while (retries--) {
2381 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2382
2383 /* in HS, means ON */
2384 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2385 break;
2386 }
2387
2388 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2389 dev_err(dwc->dev, "failed to send remote wakeup\n");
2390 return -EINVAL;
2391 }
2392
2393 return 0;
2394 }
2395
dwc3_gadget_wakeup(struct usb_gadget * g)2396 static int dwc3_gadget_wakeup(struct usb_gadget *g)
2397 {
2398 struct dwc3 *dwc = gadget_to_dwc(g);
2399 unsigned long flags;
2400 int ret;
2401
2402 if (!dwc->wakeup_configured) {
2403 dev_err(dwc->dev, "remote wakeup not configured\n");
2404 return -EINVAL;
2405 }
2406
2407 spin_lock_irqsave(&dwc->lock, flags);
2408 if (!dwc->gadget->wakeup_armed) {
2409 dev_err(dwc->dev, "not armed for remote wakeup\n");
2410 spin_unlock_irqrestore(&dwc->lock, flags);
2411 return -EINVAL;
2412 }
2413 ret = __dwc3_gadget_wakeup(dwc, true);
2414
2415 spin_unlock_irqrestore(&dwc->lock, flags);
2416
2417 return ret;
2418 }
2419
2420 static void dwc3_resume_gadget(struct dwc3 *dwc);
2421
dwc3_gadget_func_wakeup(struct usb_gadget * g,int intf_id)2422 static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id)
2423 {
2424 struct dwc3 *dwc = gadget_to_dwc(g);
2425 unsigned long flags;
2426 int ret;
2427 int link_state;
2428
2429 if (!dwc->wakeup_configured) {
2430 dev_err(dwc->dev, "remote wakeup not configured\n");
2431 return -EINVAL;
2432 }
2433
2434 spin_lock_irqsave(&dwc->lock, flags);
2435 /*
2436 * If the link is in U3, signal for remote wakeup and wait for the
2437 * link to transition to U0 before sending device notification.
2438 */
2439 link_state = dwc3_gadget_get_link_state(dwc);
2440 if (link_state == DWC3_LINK_STATE_U3) {
2441 ret = __dwc3_gadget_wakeup(dwc, false);
2442 if (ret) {
2443 spin_unlock_irqrestore(&dwc->lock, flags);
2444 return -EINVAL;
2445 }
2446 dwc3_resume_gadget(dwc);
2447 dwc->suspended = false;
2448 dwc->link_state = DWC3_LINK_STATE_U0;
2449 }
2450
2451 ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
2452 DWC3_DGCMDPAR_DN_FUNC_WAKE |
2453 DWC3_DGCMDPAR_INTF_SEL(intf_id));
2454 if (ret)
2455 dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);
2456
2457 spin_unlock_irqrestore(&dwc->lock, flags);
2458
2459 return ret;
2460 }
2461
dwc3_gadget_set_remote_wakeup(struct usb_gadget * g,int set)2462 static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set)
2463 {
2464 struct dwc3 *dwc = gadget_to_dwc(g);
2465 unsigned long flags;
2466
2467 spin_lock_irqsave(&dwc->lock, flags);
2468 dwc->wakeup_configured = !!set;
2469 spin_unlock_irqrestore(&dwc->lock, flags);
2470
2471 return 0;
2472 }
2473
dwc3_gadget_set_selfpowered(struct usb_gadget * g,int is_selfpowered)2474 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2475 int is_selfpowered)
2476 {
2477 struct dwc3 *dwc = gadget_to_dwc(g);
2478 unsigned long flags;
2479
2480 spin_lock_irqsave(&dwc->lock, flags);
2481 g->is_selfpowered = !!is_selfpowered;
2482 spin_unlock_irqrestore(&dwc->lock, flags);
2483
2484 return 0;
2485 }
2486
dwc3_stop_active_transfers(struct dwc3 * dwc)2487 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2488 {
2489 u32 epnum;
2490
2491 for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2492 struct dwc3_ep *dep;
2493
2494 dep = dwc->eps[epnum];
2495 if (!dep)
2496 continue;
2497
2498 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
2499 }
2500 }
2501
__dwc3_gadget_set_ssp_rate(struct dwc3 * dwc)2502 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2503 {
2504 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate;
2505 u32 reg;
2506
2507 if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2508 ssp_rate = dwc->max_ssp_rate;
2509
2510 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2511 reg &= ~DWC3_DCFG_SPEED_MASK;
2512 reg &= ~DWC3_DCFG_NUMLANES(~0);
2513
2514 if (ssp_rate == USB_SSP_GEN_1x2)
2515 reg |= DWC3_DCFG_SUPERSPEED;
2516 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2517 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2518
2519 if (ssp_rate != USB_SSP_GEN_2x1 &&
2520 dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2521 reg |= DWC3_DCFG_NUMLANES(1);
2522
2523 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2524 }
2525
__dwc3_gadget_set_speed(struct dwc3 * dwc)2526 static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2527 {
2528 enum usb_device_speed speed;
2529 u32 reg;
2530
2531 speed = dwc->gadget_max_speed;
2532 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2533 speed = dwc->maximum_speed;
2534
2535 if (speed == USB_SPEED_SUPER_PLUS &&
2536 DWC3_IP_IS(DWC32)) {
2537 __dwc3_gadget_set_ssp_rate(dwc);
2538 return;
2539 }
2540
2541 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2542 reg &= ~(DWC3_DCFG_SPEED_MASK);
2543
2544 /*
2545 * WORKAROUND: DWC3 revision < 2.20a have an issue
2546 * which would cause metastability state on Run/Stop
2547 * bit if we try to force the IP to USB2-only mode.
2548 *
2549 * Because of that, we cannot configure the IP to any
2550 * speed other than the SuperSpeed
2551 *
2552 * Refers to:
2553 *
2554 * STAR#9000525659: Clock Domain Crossing on DCTL in
2555 * USB 2.0 Mode
2556 */
2557 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2558 !dwc->dis_metastability_quirk) {
2559 reg |= DWC3_DCFG_SUPERSPEED;
2560 } else {
2561 switch (speed) {
2562 case USB_SPEED_FULL:
2563 reg |= DWC3_DCFG_FULLSPEED;
2564 break;
2565 case USB_SPEED_HIGH:
2566 reg |= DWC3_DCFG_HIGHSPEED;
2567 break;
2568 case USB_SPEED_SUPER:
2569 reg |= DWC3_DCFG_SUPERSPEED;
2570 break;
2571 case USB_SPEED_SUPER_PLUS:
2572 if (DWC3_IP_IS(DWC3))
2573 reg |= DWC3_DCFG_SUPERSPEED;
2574 else
2575 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2576 break;
2577 default:
2578 dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2579
2580 if (DWC3_IP_IS(DWC3))
2581 reg |= DWC3_DCFG_SUPERSPEED;
2582 else
2583 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2584 }
2585 }
2586
2587 if (DWC3_IP_IS(DWC32) &&
2588 speed > USB_SPEED_UNKNOWN &&
2589 speed < USB_SPEED_SUPER_PLUS)
2590 reg &= ~DWC3_DCFG_NUMLANES(~0);
2591
2592 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2593 }
2594
dwc3_gadget_run_stop(struct dwc3 * dwc,int is_on)2595 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
2596 {
2597 u32 reg;
2598 u32 timeout = 2000;
2599
2600 if (pm_runtime_suspended(dwc->dev))
2601 return 0;
2602
2603 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2604 if (is_on) {
2605 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2606 reg &= ~DWC3_DCTL_TRGTULST_MASK;
2607 reg |= DWC3_DCTL_TRGTULST_RX_DET;
2608 }
2609
2610 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2611 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2612 reg |= DWC3_DCTL_RUN_STOP;
2613
2614 __dwc3_gadget_set_speed(dwc);
2615 dwc->pullups_connected = true;
2616 } else {
2617 reg &= ~DWC3_DCTL_RUN_STOP;
2618
2619 dwc->pullups_connected = false;
2620 }
2621
2622 dwc3_gadget_dctl_write_safe(dwc, reg);
2623
2624 do {
2625 usleep_range(1000, 2000);
2626 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2627 reg &= DWC3_DSTS_DEVCTRLHLT;
2628 } while (--timeout && !(!is_on ^ !reg));
2629
2630 if (!timeout)
2631 return -ETIMEDOUT;
2632
2633 return 0;
2634 }
2635
2636 static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2637 static void __dwc3_gadget_stop(struct dwc3 *dwc);
2638 static int __dwc3_gadget_start(struct dwc3 *dwc);
2639
dwc3_gadget_soft_disconnect(struct dwc3 * dwc)2640 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
2641 {
2642 unsigned long flags;
2643 int ret;
2644
2645 spin_lock_irqsave(&dwc->lock, flags);
2646 if (!dwc->pullups_connected) {
2647 spin_unlock_irqrestore(&dwc->lock, flags);
2648 return 0;
2649 }
2650
2651 dwc->connected = false;
2652
2653 /*
2654 * Attempt to end pending SETUP status phase, and not wait for the
2655 * function to do so.
2656 */
2657 if (dwc->delayed_status)
2658 dwc3_ep0_send_delayed_status(dwc);
2659
2660 /*
2661 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
2662 * Section 4.1.8 Table 4-7, it states that for a device-initiated
2663 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2664 * command for any active transfers" before clearing the RunStop
2665 * bit.
2666 */
2667 dwc3_stop_active_transfers(dwc);
2668 spin_unlock_irqrestore(&dwc->lock, flags);
2669
2670 /*
2671 * Per databook, when we want to stop the gadget, if a control transfer
2672 * is still in process, complete it and get the core into setup phase.
2673 * In case the host is unresponsive to a SETUP transaction, forcefully
2674 * stall the transfer, and move back to the SETUP phase, so that any
2675 * pending endxfers can be executed.
2676 */
2677 if (dwc->ep0state != EP0_SETUP_PHASE) {
2678 reinit_completion(&dwc->ep0_in_setup);
2679
2680 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2681 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2682 if (ret == 0) {
2683 dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
2684 spin_lock_irqsave(&dwc->lock, flags);
2685 dwc3_ep0_reset_state(dwc);
2686 spin_unlock_irqrestore(&dwc->lock, flags);
2687 }
2688 }
2689
2690 /*
2691 * Note: if the GEVNTCOUNT indicates events in the event buffer, the
2692 * driver needs to acknowledge them before the controller can halt.
2693 * Simply let the interrupt handler acknowledges and handle the
2694 * remaining event generated by the controller while polling for
2695 * DSTS.DEVCTLHLT.
2696 */
2697 ret = dwc3_gadget_run_stop(dwc, false);
2698
2699 /*
2700 * Stop the gadget after controller is halted, so that if needed, the
2701 * events to update EP0 state can still occur while the run/stop
2702 * routine polls for the halted state. DEVTEN is cleared as part of
2703 * gadget stop.
2704 */
2705 spin_lock_irqsave(&dwc->lock, flags);
2706 __dwc3_gadget_stop(dwc);
2707 spin_unlock_irqrestore(&dwc->lock, flags);
2708
2709 return ret;
2710 }
2711
dwc3_gadget_soft_connect(struct dwc3 * dwc)2712 static int dwc3_gadget_soft_connect(struct dwc3 *dwc)
2713 {
2714 int ret;
2715
2716 /*
2717 * In the Synopsys DWC_usb31 1.90a programming guide section
2718 * 4.1.9, it specifies that for a reconnect after a
2719 * device-initiated disconnect requires a core soft reset
2720 * (DCTL.CSftRst) before enabling the run/stop bit.
2721 */
2722 ret = dwc3_core_soft_reset(dwc);
2723 if (ret)
2724 return ret;
2725
2726 dwc3_event_buffers_setup(dwc);
2727 __dwc3_gadget_start(dwc);
2728 return dwc3_gadget_run_stop(dwc, true);
2729 }
2730
dwc3_gadget_pullup(struct usb_gadget * g,int is_on)2731 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2732 {
2733 struct dwc3 *dwc = gadget_to_dwc(g);
2734 int ret;
2735
2736 is_on = !!is_on;
2737
2738 dwc->softconnect = is_on;
2739
2740 /*
2741 * Avoid issuing a runtime resume if the device is already in the
2742 * suspended state during gadget disconnect. DWC3 gadget was already
2743 * halted/stopped during runtime suspend.
2744 */
2745 if (!is_on) {
2746 pm_runtime_barrier(dwc->dev);
2747 if (pm_runtime_suspended(dwc->dev))
2748 return 0;
2749 }
2750
2751 /*
2752 * Check the return value for successful resume, or error. For a
2753 * successful resume, the DWC3 runtime PM resume routine will handle
2754 * the run stop sequence, so avoid duplicate operations here.
2755 */
2756 ret = pm_runtime_get_sync(dwc->dev);
2757 if (!ret || ret < 0) {
2758 pm_runtime_put(dwc->dev);
2759 if (ret < 0)
2760 pm_runtime_set_suspended(dwc->dev);
2761 return ret;
2762 }
2763
2764 if (dwc->pullups_connected == is_on) {
2765 pm_runtime_put(dwc->dev);
2766 return 0;
2767 }
2768
2769 synchronize_irq(dwc->irq_gadget);
2770
2771 if (!is_on)
2772 ret = dwc3_gadget_soft_disconnect(dwc);
2773 else
2774 ret = dwc3_gadget_soft_connect(dwc);
2775
2776 pm_runtime_put(dwc->dev);
2777
2778 return ret;
2779 }
2780
dwc3_gadget_enable_irq(struct dwc3 * dwc)2781 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2782 {
2783 u32 reg;
2784
2785 /* Enable all but Start and End of Frame IRQs */
2786 reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2787 DWC3_DEVTEN_CMDCMPLTEN |
2788 DWC3_DEVTEN_ERRTICERREN |
2789 DWC3_DEVTEN_WKUPEVTEN |
2790 DWC3_DEVTEN_CONNECTDONEEN |
2791 DWC3_DEVTEN_USBRSTEN |
2792 DWC3_DEVTEN_DISCONNEVTEN);
2793
2794 if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2795 reg |= DWC3_DEVTEN_ULSTCNGEN;
2796
2797 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2798 if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2799 reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2800
2801 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2802 }
2803
dwc3_gadget_disable_irq(struct dwc3 * dwc)2804 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2805 {
2806 /* mask all interrupts */
2807 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2808 }
2809
2810 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2811 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2812
2813 /**
2814 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2815 * @dwc: pointer to our context structure
2816 *
2817 * The following looks like complex but it's actually very simple. In order to
2818 * calculate the number of packets we can burst at once on OUT transfers, we're
2819 * gonna use RxFIFO size.
2820 *
2821 * To calculate RxFIFO size we need two numbers:
2822 * MDWIDTH = size, in bits, of the internal memory bus
2823 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2824 *
2825 * Given these two numbers, the formula is simple:
2826 *
2827 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2828 *
2829 * 24 bytes is for 3x SETUP packets
2830 * 16 bytes is a clock domain crossing tolerance
2831 *
2832 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2833 */
dwc3_gadget_setup_nump(struct dwc3 * dwc)2834 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2835 {
2836 u32 ram2_depth;
2837 u32 mdwidth;
2838 u32 nump;
2839 u32 reg;
2840
2841 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2842 mdwidth = dwc3_mdwidth(dwc);
2843
2844 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2845 nump = min_t(u32, nump, 16);
2846
2847 /* update NumP */
2848 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2849 reg &= ~DWC3_DCFG_NUMP_MASK;
2850 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2851 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2852 }
2853
__dwc3_gadget_start(struct dwc3 * dwc)2854 static int __dwc3_gadget_start(struct dwc3 *dwc)
2855 {
2856 struct dwc3_ep *dep;
2857 int ret = 0;
2858 u32 reg;
2859
2860 /*
2861 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2862 * the core supports IMOD, disable it.
2863 */
2864 if (dwc->imod_interval) {
2865 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2866 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2867 } else if (dwc3_has_imod(dwc)) {
2868 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2869 }
2870
2871 /*
2872 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2873 * field instead of letting dwc3 itself calculate that automatically.
2874 *
2875 * This way, we maximize the chances that we'll be able to get several
2876 * bursts of data without going through any sort of endpoint throttling.
2877 */
2878 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2879 if (DWC3_IP_IS(DWC3))
2880 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2881 else
2882 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2883
2884 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2885
2886 dwc3_gadget_setup_nump(dwc);
2887
2888 /*
2889 * Currently the controller handles single stream only. So, Ignore
2890 * Packet Pending bit for stream selection and don't search for another
2891 * stream if the host sends Data Packet with PP=0 (for OUT direction) or
2892 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2893 * the stream performance.
2894 */
2895 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2896 reg |= DWC3_DCFG_IGNSTRMPP;
2897 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2898
2899 /* Enable MST by default if the device is capable of MST */
2900 if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2901 reg = dwc3_readl(dwc->regs, DWC3_DCFG1);
2902 reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2903 dwc3_writel(dwc->regs, DWC3_DCFG1, reg);
2904 }
2905
2906 /* Start with SuperSpeed Default */
2907 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2908
2909 dep = dwc->eps[0];
2910 dep->flags = 0;
2911 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2912 if (ret) {
2913 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2914 goto err0;
2915 }
2916
2917 dep = dwc->eps[1];
2918 dep->flags = 0;
2919 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2920 if (ret) {
2921 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2922 goto err1;
2923 }
2924
2925 /* begin to receive SETUP packets */
2926 dwc->ep0state = EP0_SETUP_PHASE;
2927 dwc->ep0_bounced = false;
2928 dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2929 dwc->delayed_status = false;
2930 dwc3_ep0_out_start(dwc);
2931
2932 dwc3_gadget_enable_irq(dwc);
2933 dwc3_enable_susphy(dwc, true);
2934
2935 return 0;
2936
2937 err1:
2938 __dwc3_gadget_ep_disable(dwc->eps[0]);
2939
2940 err0:
2941 return ret;
2942 }
2943
dwc3_gadget_start(struct usb_gadget * g,struct usb_gadget_driver * driver)2944 static int dwc3_gadget_start(struct usb_gadget *g,
2945 struct usb_gadget_driver *driver)
2946 {
2947 struct dwc3 *dwc = gadget_to_dwc(g);
2948 unsigned long flags;
2949 int ret;
2950 int irq;
2951
2952 irq = dwc->irq_gadget;
2953 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2954 IRQF_SHARED, "dwc3", dwc->ev_buf);
2955 if (ret) {
2956 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2957 irq, ret);
2958 return ret;
2959 }
2960
2961 spin_lock_irqsave(&dwc->lock, flags);
2962 dwc->gadget_driver = driver;
2963 spin_unlock_irqrestore(&dwc->lock, flags);
2964
2965 if (dwc->sys_wakeup)
2966 device_wakeup_enable(dwc->sysdev);
2967
2968 return 0;
2969 }
2970
__dwc3_gadget_stop(struct dwc3 * dwc)2971 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2972 {
2973 dwc3_gadget_disable_irq(dwc);
2974 __dwc3_gadget_ep_disable(dwc->eps[0]);
2975 __dwc3_gadget_ep_disable(dwc->eps[1]);
2976 }
2977
dwc3_gadget_stop(struct usb_gadget * g)2978 static int dwc3_gadget_stop(struct usb_gadget *g)
2979 {
2980 struct dwc3 *dwc = gadget_to_dwc(g);
2981 unsigned long flags;
2982
2983 if (dwc->sys_wakeup)
2984 device_wakeup_disable(dwc->sysdev);
2985
2986 spin_lock_irqsave(&dwc->lock, flags);
2987 dwc->gadget_driver = NULL;
2988 dwc->max_cfg_eps = 0;
2989 spin_unlock_irqrestore(&dwc->lock, flags);
2990
2991 free_irq(dwc->irq_gadget, dwc->ev_buf);
2992
2993 return 0;
2994 }
2995
dwc3_gadget_config_params(struct usb_gadget * g,struct usb_dcd_config_params * params)2996 static void dwc3_gadget_config_params(struct usb_gadget *g,
2997 struct usb_dcd_config_params *params)
2998 {
2999 struct dwc3 *dwc = gadget_to_dwc(g);
3000
3001 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
3002 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
3003
3004 /* Recommended BESL */
3005 if (!dwc->dis_enblslpm_quirk) {
3006 /*
3007 * If the recommended BESL baseline is 0 or if the BESL deep is
3008 * less than 2, Microsoft's Windows 10 host usb stack will issue
3009 * a usb reset immediately after it receives the extended BOS
3010 * descriptor and the enumeration will fail. To maintain
3011 * compatibility with the Windows' usb stack, let's set the
3012 * recommended BESL baseline to 1 and clamp the BESL deep to be
3013 * within 2 to 15.
3014 */
3015 params->besl_baseline = 1;
3016 if (dwc->is_utmi_l1_suspend)
3017 params->besl_deep =
3018 clamp_t(u8, dwc->hird_threshold, 2, 15);
3019 }
3020
3021 /* U1 Device exit Latency */
3022 if (dwc->dis_u1_entry_quirk)
3023 params->bU1devExitLat = 0;
3024 else
3025 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
3026
3027 /* U2 Device exit Latency */
3028 if (dwc->dis_u2_entry_quirk)
3029 params->bU2DevExitLat = 0;
3030 else
3031 params->bU2DevExitLat =
3032 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
3033 }
3034
dwc3_gadget_set_speed(struct usb_gadget * g,enum usb_device_speed speed)3035 static void dwc3_gadget_set_speed(struct usb_gadget *g,
3036 enum usb_device_speed speed)
3037 {
3038 struct dwc3 *dwc = gadget_to_dwc(g);
3039 unsigned long flags;
3040
3041 spin_lock_irqsave(&dwc->lock, flags);
3042 dwc->gadget_max_speed = speed;
3043 spin_unlock_irqrestore(&dwc->lock, flags);
3044 }
3045
dwc3_gadget_set_ssp_rate(struct usb_gadget * g,enum usb_ssp_rate rate)3046 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
3047 enum usb_ssp_rate rate)
3048 {
3049 struct dwc3 *dwc = gadget_to_dwc(g);
3050 unsigned long flags;
3051
3052 spin_lock_irqsave(&dwc->lock, flags);
3053 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
3054 dwc->gadget_ssp_rate = rate;
3055 spin_unlock_irqrestore(&dwc->lock, flags);
3056 }
3057
dwc3_gadget_vbus_draw(struct usb_gadget * g,unsigned int mA)3058 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
3059 {
3060 struct dwc3 *dwc = gadget_to_dwc(g);
3061 union power_supply_propval val = {0};
3062 int ret;
3063
3064 if (dwc->usb2_phy)
3065 return usb_phy_set_power(dwc->usb2_phy, mA);
3066
3067 if (!dwc->usb_psy)
3068 return -EOPNOTSUPP;
3069
3070 val.intval = 1000 * mA;
3071 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
3072
3073 return ret;
3074 }
3075
3076 /**
3077 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3078 * @g: pointer to the USB gadget
3079 *
3080 * Used to record the maximum number of endpoints being used in a USB composite
3081 * device. (across all configurations) This is to be used in the calculation
3082 * of the TXFIFO sizes when resizing internal memory for individual endpoints.
3083 * It will help ensured that the resizing logic reserves enough space for at
3084 * least one max packet.
3085 */
dwc3_gadget_check_config(struct usb_gadget * g)3086 static int dwc3_gadget_check_config(struct usb_gadget *g)
3087 {
3088 struct dwc3 *dwc = gadget_to_dwc(g);
3089 struct usb_ep *ep;
3090 int fifo_size = 0;
3091 int ram1_depth;
3092 int ep_num = 0;
3093
3094 if (!dwc->do_fifo_resize)
3095 return 0;
3096
3097 list_for_each_entry(ep, &g->ep_list, ep_list) {
3098 /* Only interested in the IN endpoints */
3099 if (ep->claimed && (ep->address & USB_DIR_IN))
3100 ep_num++;
3101 }
3102
3103 if (ep_num <= dwc->max_cfg_eps)
3104 return 0;
3105
3106 /* Update the max number of eps in the composition */
3107 dwc->max_cfg_eps = ep_num;
3108
3109 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
3110 /* Based on the equation, increment by one for every ep */
3111 fifo_size += dwc->max_cfg_eps;
3112
3113 /* Check if we can fit a single fifo per endpoint */
3114 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
3115 if (fifo_size > ram1_depth)
3116 return -ENOMEM;
3117
3118 return 0;
3119 }
3120
dwc3_gadget_async_callbacks(struct usb_gadget * g,bool enable)3121 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
3122 {
3123 struct dwc3 *dwc = gadget_to_dwc(g);
3124 unsigned long flags;
3125
3126 spin_lock_irqsave(&dwc->lock, flags);
3127 dwc->async_callbacks = enable;
3128 spin_unlock_irqrestore(&dwc->lock, flags);
3129 }
3130
3131 static const struct usb_gadget_ops dwc3_gadget_ops = {
3132 .get_frame = dwc3_gadget_get_frame,
3133 .wakeup = dwc3_gadget_wakeup,
3134 .func_wakeup = dwc3_gadget_func_wakeup,
3135 .set_remote_wakeup = dwc3_gadget_set_remote_wakeup,
3136 .set_selfpowered = dwc3_gadget_set_selfpowered,
3137 .pullup = dwc3_gadget_pullup,
3138 .udc_start = dwc3_gadget_start,
3139 .udc_stop = dwc3_gadget_stop,
3140 .udc_set_speed = dwc3_gadget_set_speed,
3141 .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate,
3142 .get_config_params = dwc3_gadget_config_params,
3143 .vbus_draw = dwc3_gadget_vbus_draw,
3144 .check_config = dwc3_gadget_check_config,
3145 .udc_async_callbacks = dwc3_gadget_async_callbacks,
3146 };
3147
3148 /* -------------------------------------------------------------------------- */
3149
dwc3_gadget_init_control_endpoint(struct dwc3_ep * dep)3150 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
3151 {
3152 struct dwc3 *dwc = dep->dwc;
3153
3154 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
3155 dep->endpoint.maxburst = 1;
3156 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
3157 if (!dep->direction)
3158 dwc->gadget->ep0 = &dep->endpoint;
3159
3160 dep->endpoint.caps.type_control = true;
3161
3162 return 0;
3163 }
3164
dwc3_gadget_init_in_endpoint(struct dwc3_ep * dep)3165 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
3166 {
3167 struct dwc3 *dwc = dep->dwc;
3168 u32 mdwidth;
3169 int size;
3170 int maxpacket;
3171
3172 mdwidth = dwc3_mdwidth(dwc);
3173
3174 /* MDWIDTH is represented in bits, we need it in bytes */
3175 mdwidth /= 8;
3176
3177 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
3178 if (DWC3_IP_IS(DWC3))
3179 size = DWC3_GTXFIFOSIZ_TXFDEP(size);
3180 else
3181 size = DWC31_GTXFIFOSIZ_TXFDEP(size);
3182
3183 /*
3184 * maxpacket size is determined as part of the following, after assuming
3185 * a mult value of one maxpacket:
3186 * DWC3 revision 280A and prior:
3187 * fifo_size = mult * (max_packet / mdwidth) + 1;
3188 * maxpacket = mdwidth * (fifo_size - 1);
3189 *
3190 * DWC3 revision 290A and onwards:
3191 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
3192 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
3193 */
3194 if (DWC3_VER_IS_PRIOR(DWC3, 290A))
3195 maxpacket = mdwidth * (size - 1);
3196 else
3197 maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
3198
3199 /* Functionally, space for one max packet is sufficient */
3200 size = min_t(int, maxpacket, 1024);
3201 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3202
3203 dep->endpoint.max_streams = 16;
3204 dep->endpoint.ops = &dwc3_gadget_ep_ops;
3205 list_add_tail(&dep->endpoint.ep_list,
3206 &dwc->gadget->ep_list);
3207 dep->endpoint.caps.type_iso = true;
3208 dep->endpoint.caps.type_bulk = true;
3209 dep->endpoint.caps.type_int = true;
3210
3211 return dwc3_alloc_trb_pool(dep);
3212 }
3213
dwc3_gadget_init_out_endpoint(struct dwc3_ep * dep)3214 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3215 {
3216 struct dwc3 *dwc = dep->dwc;
3217 u32 mdwidth;
3218 int size;
3219
3220 mdwidth = dwc3_mdwidth(dwc);
3221
3222 /* MDWIDTH is represented in bits, convert to bytes */
3223 mdwidth /= 8;
3224
3225 /* All OUT endpoints share a single RxFIFO space */
3226 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
3227 if (DWC3_IP_IS(DWC3))
3228 size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3229 else
3230 size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3231
3232 /* FIFO depth is in MDWDITH bytes */
3233 size *= mdwidth;
3234
3235 /*
3236 * To meet performance requirement, a minimum recommended RxFIFO size
3237 * is defined as follow:
3238 * RxFIFO size >= (3 x MaxPacketSize) +
3239 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3240 *
3241 * Then calculate the max packet limit as below.
3242 */
3243 size -= (3 * 8) + 16;
3244 if (size < 0)
3245 size = 0;
3246 else
3247 size /= 3;
3248
3249 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3250 dep->endpoint.max_streams = 16;
3251 dep->endpoint.ops = &dwc3_gadget_ep_ops;
3252 list_add_tail(&dep->endpoint.ep_list,
3253 &dwc->gadget->ep_list);
3254 dep->endpoint.caps.type_iso = true;
3255 dep->endpoint.caps.type_bulk = true;
3256 dep->endpoint.caps.type_int = true;
3257
3258 return dwc3_alloc_trb_pool(dep);
3259 }
3260
dwc3_gadget_init_endpoint(struct dwc3 * dwc,u8 epnum)3261 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3262 {
3263 struct dwc3_ep *dep;
3264 bool direction = epnum & 1;
3265 int ret;
3266 u8 num = epnum >> 1;
3267
3268 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3269 if (!dep)
3270 return -ENOMEM;
3271
3272 dep->dwc = dwc;
3273 dep->number = epnum;
3274 dep->direction = direction;
3275 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3276 dwc->eps[epnum] = dep;
3277 dep->combo_num = 0;
3278 dep->start_cmd_status = 0;
3279
3280 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
3281 direction ? "in" : "out");
3282
3283 dep->endpoint.name = dep->name;
3284
3285 if (!(dep->number > 1)) {
3286 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3287 dep->endpoint.comp_desc = NULL;
3288 }
3289
3290 if (num == 0)
3291 ret = dwc3_gadget_init_control_endpoint(dep);
3292 else if (direction)
3293 ret = dwc3_gadget_init_in_endpoint(dep);
3294 else
3295 ret = dwc3_gadget_init_out_endpoint(dep);
3296
3297 if (ret)
3298 return ret;
3299
3300 dep->endpoint.caps.dir_in = direction;
3301 dep->endpoint.caps.dir_out = !direction;
3302
3303 INIT_LIST_HEAD(&dep->pending_list);
3304 INIT_LIST_HEAD(&dep->started_list);
3305 INIT_LIST_HEAD(&dep->cancelled_list);
3306
3307 dwc3_debugfs_create_endpoint_dir(dep);
3308
3309 return 0;
3310 }
3311
dwc3_gadget_init_endpoints(struct dwc3 * dwc,u8 total)3312 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3313 {
3314 u8 epnum;
3315
3316 INIT_LIST_HEAD(&dwc->gadget->ep_list);
3317
3318 for (epnum = 0; epnum < total; epnum++) {
3319 int ret;
3320
3321 ret = dwc3_gadget_init_endpoint(dwc, epnum);
3322 if (ret)
3323 return ret;
3324 }
3325
3326 return 0;
3327 }
3328
dwc3_gadget_free_endpoints(struct dwc3 * dwc)3329 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3330 {
3331 struct dwc3_ep *dep;
3332 u8 epnum;
3333
3334 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3335 dep = dwc->eps[epnum];
3336 if (!dep)
3337 continue;
3338 /*
3339 * Physical endpoints 0 and 1 are special; they form the
3340 * bi-directional USB endpoint 0.
3341 *
3342 * For those two physical endpoints, we don't allocate a TRB
3343 * pool nor do we add them the endpoints list. Due to that, we
3344 * shouldn't do these two operations otherwise we would end up
3345 * with all sorts of bugs when removing dwc3.ko.
3346 */
3347 if (epnum != 0 && epnum != 1) {
3348 dwc3_free_trb_pool(dep);
3349 list_del(&dep->endpoint.ep_list);
3350 }
3351
3352 dwc3_debugfs_remove_endpoint_dir(dep);
3353 kfree(dep);
3354 }
3355 }
3356
3357 /* -------------------------------------------------------------------------- */
3358
dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep * dep,struct dwc3_request * req,struct dwc3_trb * trb,const struct dwc3_event_depevt * event,int status,int chain)3359 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3360 struct dwc3_request *req, struct dwc3_trb *trb,
3361 const struct dwc3_event_depevt *event, int status, int chain)
3362 {
3363 unsigned int count;
3364
3365 dwc3_ep_inc_deq(dep);
3366
3367 trace_dwc3_complete_trb(dep, trb);
3368 req->num_trbs--;
3369
3370 /*
3371 * If we're in the middle of series of chained TRBs and we
3372 * receive a short transfer along the way, DWC3 will skip
3373 * through all TRBs including the last TRB in the chain (the
3374 * where CHN bit is zero. DWC3 will also avoid clearing HWO
3375 * bit and SW has to do it manually.
3376 *
3377 * We're going to do that here to avoid problems of HW trying
3378 * to use bogus TRBs for transfers.
3379 */
3380 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3381 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3382
3383 /*
3384 * For isochronous transfers, the first TRB in a service interval must
3385 * have the Isoc-First type. Track and report its interval frame number.
3386 */
3387 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3388 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3389 unsigned int frame_number;
3390
3391 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3392 frame_number &= ~(dep->interval - 1);
3393 req->request.frame_number = frame_number;
3394 }
3395
3396 /*
3397 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3398 * this TRB points to the bounce buffer address, it's a MPS alignment
3399 * TRB. Don't add it to req->remaining calculation.
3400 */
3401 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3402 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3403 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3404 return 1;
3405 }
3406
3407 count = trb->size & DWC3_TRB_SIZE_MASK;
3408 req->remaining += count;
3409
3410 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3411 return 1;
3412
3413 if (event->status & DEPEVT_STATUS_SHORT && !chain)
3414 return 1;
3415
3416 if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
3417 DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
3418 return 1;
3419
3420 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3421 (trb->ctrl & DWC3_TRB_CTRL_LST))
3422 return 1;
3423
3424 return 0;
3425 }
3426
dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep * dep,struct dwc3_request * req,const struct dwc3_event_depevt * event,int status)3427 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3428 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3429 int status)
3430 {
3431 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3432 struct scatterlist *sg = req->sg;
3433 struct scatterlist *s;
3434 unsigned int num_queued = req->num_queued_sgs;
3435 unsigned int i;
3436 int ret = 0;
3437
3438 for_each_sg(sg, s, num_queued, i) {
3439 trb = &dep->trb_pool[dep->trb_dequeue];
3440
3441 req->sg = sg_next(s);
3442 req->num_queued_sgs--;
3443
3444 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3445 trb, event, status, true);
3446 if (ret)
3447 break;
3448 }
3449
3450 return ret;
3451 }
3452
dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep * dep,struct dwc3_request * req,const struct dwc3_event_depevt * event,int status)3453 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3454 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3455 int status)
3456 {
3457 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3458
3459 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3460 event, status, false);
3461 }
3462
dwc3_gadget_ep_request_completed(struct dwc3_request * req)3463 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3464 {
3465 return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3466 }
3467
dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,struct dwc3_request * req,int status)3468 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3469 const struct dwc3_event_depevt *event,
3470 struct dwc3_request *req, int status)
3471 {
3472 int request_status;
3473 int ret;
3474
3475 if (req->request.num_mapped_sgs)
3476 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3477 status);
3478 else
3479 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3480 status);
3481
3482 req->request.actual = req->request.length - req->remaining;
3483
3484 if (!dwc3_gadget_ep_request_completed(req))
3485 goto out;
3486
3487 if (req->needs_extra_trb) {
3488 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3489 status);
3490 req->needs_extra_trb = false;
3491 }
3492
3493 /*
3494 * The event status only reflects the status of the TRB with IOC set.
3495 * For the requests that don't set interrupt on completion, the driver
3496 * needs to check and return the status of the completed TRBs associated
3497 * with the request. Use the status of the last TRB of the request.
3498 */
3499 if (req->request.no_interrupt) {
3500 struct dwc3_trb *trb;
3501
3502 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
3503 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3504 case DWC3_TRBSTS_MISSED_ISOC:
3505 /* Isoc endpoint only */
3506 request_status = -EXDEV;
3507 break;
3508 case DWC3_TRB_STS_XFER_IN_PROG:
3509 /* Applicable when End Transfer with ForceRM=0 */
3510 case DWC3_TRBSTS_SETUP_PENDING:
3511 /* Control endpoint only */
3512 case DWC3_TRBSTS_OK:
3513 default:
3514 request_status = 0;
3515 break;
3516 }
3517 } else {
3518 request_status = status;
3519 }
3520
3521 dwc3_gadget_giveback(dep, req, request_status);
3522
3523 out:
3524 return ret;
3525 }
3526
dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,int status)3527 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3528 const struct dwc3_event_depevt *event, int status)
3529 {
3530 struct dwc3_request *req;
3531
3532 while (!list_empty(&dep->started_list)) {
3533 int ret;
3534
3535 req = next_request(&dep->started_list);
3536 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3537 req, status);
3538 if (ret)
3539 break;
3540 /*
3541 * The endpoint is disabled, let the dwc3_remove_requests()
3542 * handle the cleanup.
3543 */
3544 if (!dep->endpoint.desc)
3545 break;
3546 }
3547 }
3548
dwc3_gadget_ep_should_continue(struct dwc3_ep * dep)3549 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3550 {
3551 struct dwc3_request *req;
3552 struct dwc3 *dwc = dep->dwc;
3553
3554 if (!dep->endpoint.desc || !dwc->pullups_connected ||
3555 !dwc->connected)
3556 return false;
3557
3558 if (!list_empty(&dep->pending_list))
3559 return true;
3560
3561 /*
3562 * We only need to check the first entry of the started list. We can
3563 * assume the completed requests are removed from the started list.
3564 */
3565 req = next_request(&dep->started_list);
3566 if (!req)
3567 return false;
3568
3569 return !dwc3_gadget_ep_request_completed(req);
3570 }
3571
dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3572 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3573 const struct dwc3_event_depevt *event)
3574 {
3575 dep->frame_number = event->parameters;
3576 }
3577
dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,int status)3578 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3579 const struct dwc3_event_depevt *event, int status)
3580 {
3581 struct dwc3 *dwc = dep->dwc;
3582 bool no_started_trb = true;
3583
3584 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3585
3586 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3587 goto out;
3588
3589 if (!dep->endpoint.desc)
3590 return no_started_trb;
3591
3592 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3593 list_empty(&dep->started_list) &&
3594 (list_empty(&dep->pending_list) || status == -EXDEV))
3595 dwc3_stop_active_transfer(dep, true, true);
3596 else if (dwc3_gadget_ep_should_continue(dep))
3597 if (__dwc3_gadget_kick_transfer(dep) == 0)
3598 no_started_trb = false;
3599
3600 out:
3601 /*
3602 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3603 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
3604 */
3605 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3606 u32 reg;
3607 int i;
3608
3609 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3610 dep = dwc->eps[i];
3611
3612 if (!(dep->flags & DWC3_EP_ENABLED))
3613 continue;
3614
3615 if (!list_empty(&dep->started_list))
3616 return no_started_trb;
3617 }
3618
3619 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3620 reg |= dwc->u1u2;
3621 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3622
3623 dwc->u1u2 = 0;
3624 }
3625
3626 return no_started_trb;
3627 }
3628
dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3629 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3630 const struct dwc3_event_depevt *event)
3631 {
3632 int status = 0;
3633
3634 if (!dep->endpoint.desc)
3635 return;
3636
3637 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3638 dwc3_gadget_endpoint_frame_from_event(dep, event);
3639
3640 if (event->status & DEPEVT_STATUS_BUSERR)
3641 status = -ECONNRESET;
3642
3643 if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3644 status = -EXDEV;
3645
3646 dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3647 }
3648
dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3649 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3650 const struct dwc3_event_depevt *event)
3651 {
3652 int status = 0;
3653
3654 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3655
3656 if (event->status & DEPEVT_STATUS_BUSERR)
3657 status = -ECONNRESET;
3658
3659 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3660 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3661 }
3662
dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3663 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3664 const struct dwc3_event_depevt *event)
3665 {
3666 dwc3_gadget_endpoint_frame_from_event(dep, event);
3667
3668 /*
3669 * The XferNotReady event is generated only once before the endpoint
3670 * starts. It will be generated again when END_TRANSFER command is
3671 * issued. For some controller versions, the XferNotReady event may be
3672 * generated while the END_TRANSFER command is still in process. Ignore
3673 * it and wait for the next XferNotReady event after the command is
3674 * completed.
3675 */
3676 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3677 return;
3678
3679 (void) __dwc3_gadget_start_isoc(dep);
3680 }
3681
dwc3_gadget_endpoint_command_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3682 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3683 const struct dwc3_event_depevt *event)
3684 {
3685 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3686
3687 if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3688 return;
3689
3690 /*
3691 * The END_TRANSFER command will cause the controller to generate a
3692 * NoStream Event, and it's not due to the host DP NoStream rejection.
3693 * Ignore the next NoStream event.
3694 */
3695 if (dep->stream_capable)
3696 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3697
3698 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3699 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3700 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3701
3702 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3703 struct dwc3 *dwc = dep->dwc;
3704
3705 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3706 if (dwc3_send_clear_stall_ep_cmd(dep)) {
3707 struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3708
3709 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3710 if (dwc->delayed_status)
3711 __dwc3_gadget_ep0_set_halt(ep0, 1);
3712 return;
3713 }
3714
3715 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3716 if (dwc->clear_stall_protocol == dep->number)
3717 dwc3_ep0_send_delayed_status(dwc);
3718 }
3719
3720 if ((dep->flags & DWC3_EP_DELAY_START) &&
3721 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3722 __dwc3_gadget_kick_transfer(dep);
3723
3724 dep->flags &= ~DWC3_EP_DELAY_START;
3725 }
3726
dwc3_gadget_endpoint_stream_event(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3727 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3728 const struct dwc3_event_depevt *event)
3729 {
3730 struct dwc3 *dwc = dep->dwc;
3731
3732 if (event->status == DEPEVT_STREAMEVT_FOUND) {
3733 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3734 goto out;
3735 }
3736
3737 /* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3738 switch (event->parameters) {
3739 case DEPEVT_STREAM_PRIME:
3740 /*
3741 * If the host can properly transition the endpoint state from
3742 * idle to prime after a NoStream rejection, there's no need to
3743 * force restarting the endpoint to reinitiate the stream. To
3744 * simplify the check, assume the host follows the USB spec if
3745 * it primed the endpoint more than once.
3746 */
3747 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3748 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3749 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3750 else
3751 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3752 }
3753
3754 break;
3755 case DEPEVT_STREAM_NOSTREAM:
3756 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3757 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3758 (!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3759 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3760 break;
3761
3762 /*
3763 * If the host rejects a stream due to no active stream, by the
3764 * USB and xHCI spec, the endpoint will be put back to idle
3765 * state. When the host is ready (buffer added/updated), it will
3766 * prime the endpoint to inform the usb device controller. This
3767 * triggers the device controller to issue ERDY to restart the
3768 * stream. However, some hosts don't follow this and keep the
3769 * endpoint in the idle state. No prime will come despite host
3770 * streams are updated, and the device controller will not be
3771 * triggered to generate ERDY to move the next stream data. To
3772 * workaround this and maintain compatibility with various
3773 * hosts, force to reinitiate the stream until the host is ready
3774 * instead of waiting for the host to prime the endpoint.
3775 */
3776 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3777 unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3778
3779 dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3780 } else {
3781 dep->flags |= DWC3_EP_DELAY_START;
3782 dwc3_stop_active_transfer(dep, true, true);
3783 return;
3784 }
3785 break;
3786 }
3787
3788 out:
3789 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3790 }
3791
dwc3_endpoint_interrupt(struct dwc3 * dwc,const struct dwc3_event_depevt * event)3792 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3793 const struct dwc3_event_depevt *event)
3794 {
3795 struct dwc3_ep *dep;
3796 u8 epnum = event->endpoint_number;
3797
3798 dep = dwc->eps[epnum];
3799
3800 if (!(dep->flags & DWC3_EP_ENABLED)) {
3801 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3802 return;
3803
3804 /* Handle only EPCMDCMPLT when EP disabled */
3805 if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
3806 !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3807 return;
3808 }
3809
3810 if (epnum == 0 || epnum == 1) {
3811 dwc3_ep0_interrupt(dwc, event);
3812 return;
3813 }
3814
3815 switch (event->endpoint_event) {
3816 case DWC3_DEPEVT_XFERINPROGRESS:
3817 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3818 break;
3819 case DWC3_DEPEVT_XFERNOTREADY:
3820 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3821 break;
3822 case DWC3_DEPEVT_EPCMDCMPLT:
3823 dwc3_gadget_endpoint_command_complete(dep, event);
3824 break;
3825 case DWC3_DEPEVT_XFERCOMPLETE:
3826 dwc3_gadget_endpoint_transfer_complete(dep, event);
3827 break;
3828 case DWC3_DEPEVT_STREAMEVT:
3829 dwc3_gadget_endpoint_stream_event(dep, event);
3830 break;
3831 case DWC3_DEPEVT_RXTXFIFOEVT:
3832 break;
3833 default:
3834 dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event);
3835 break;
3836 }
3837 }
3838
dwc3_disconnect_gadget(struct dwc3 * dwc)3839 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3840 {
3841 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3842 spin_unlock(&dwc->lock);
3843 dwc->gadget_driver->disconnect(dwc->gadget);
3844 spin_lock(&dwc->lock);
3845 }
3846 }
3847
dwc3_suspend_gadget(struct dwc3 * dwc)3848 static void dwc3_suspend_gadget(struct dwc3 *dwc)
3849 {
3850 if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3851 spin_unlock(&dwc->lock);
3852 dwc->gadget_driver->suspend(dwc->gadget);
3853 spin_lock(&dwc->lock);
3854 }
3855 }
3856
dwc3_resume_gadget(struct dwc3 * dwc)3857 static void dwc3_resume_gadget(struct dwc3 *dwc)
3858 {
3859 if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3860 spin_unlock(&dwc->lock);
3861 dwc->gadget_driver->resume(dwc->gadget);
3862 spin_lock(&dwc->lock);
3863 }
3864 }
3865
dwc3_reset_gadget(struct dwc3 * dwc)3866 static void dwc3_reset_gadget(struct dwc3 *dwc)
3867 {
3868 if (!dwc->gadget_driver)
3869 return;
3870
3871 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3872 spin_unlock(&dwc->lock);
3873 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3874 spin_lock(&dwc->lock);
3875 }
3876 }
3877
dwc3_stop_active_transfer(struct dwc3_ep * dep,bool force,bool interrupt)3878 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3879 bool interrupt)
3880 {
3881 struct dwc3 *dwc = dep->dwc;
3882
3883 /*
3884 * Only issue End Transfer command to the control endpoint of a started
3885 * Data Phase. Typically we should only do so in error cases such as
3886 * invalid/unexpected direction as described in the control transfer
3887 * flow of the programming guide.
3888 */
3889 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3890 return;
3891
3892 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
3893 return;
3894
3895 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3896 (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3897 return;
3898
3899 /*
3900 * If a Setup packet is received but yet to DMA out, the controller will
3901 * not process the End Transfer command of any endpoint. Polling of its
3902 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
3903 * timeout. Delay issuing the End Transfer command until the Setup TRB is
3904 * prepared.
3905 */
3906 if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
3907 dep->flags |= DWC3_EP_DELAY_STOP;
3908 return;
3909 }
3910
3911 /*
3912 * NOTICE: We are violating what the Databook says about the
3913 * EndTransfer command. Ideally we would _always_ wait for the
3914 * EndTransfer Command Completion IRQ, but that's causing too
3915 * much trouble synchronizing between us and gadget driver.
3916 *
3917 * We have discussed this with the IP Provider and it was
3918 * suggested to giveback all requests here.
3919 *
3920 * Note also that a similar handling was tested by Synopsys
3921 * (thanks a lot Paul) and nothing bad has come out of it.
3922 * In short, what we're doing is issuing EndTransfer with
3923 * CMDIOC bit set and delay kicking transfer until the
3924 * EndTransfer command had completed.
3925 *
3926 * As of IP version 3.10a of the DWC_usb3 IP, the controller
3927 * supports a mode to work around the above limitation. The
3928 * software can poll the CMDACT bit in the DEPCMD register
3929 * after issuing a EndTransfer command. This mode is enabled
3930 * by writing GUCTL2[14]. This polling is already done in the
3931 * dwc3_send_gadget_ep_cmd() function so if the mode is
3932 * enabled, the EndTransfer command will have completed upon
3933 * returning from this function.
3934 *
3935 * This mode is NOT available on the DWC_usb31 IP. In this
3936 * case, if the IOC bit is not set, then delay by 1ms
3937 * after issuing the EndTransfer command. This allows for the
3938 * controller to handle the command completely before DWC3
3939 * remove requests attempts to unmap USB request buffers.
3940 */
3941
3942 __dwc3_stop_active_transfer(dep, force, interrupt);
3943 }
3944
dwc3_clear_stall_all_ep(struct dwc3 * dwc)3945 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3946 {
3947 u32 epnum;
3948
3949 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3950 struct dwc3_ep *dep;
3951 int ret;
3952
3953 dep = dwc->eps[epnum];
3954 if (!dep)
3955 continue;
3956
3957 if (!(dep->flags & DWC3_EP_STALL))
3958 continue;
3959
3960 dep->flags &= ~DWC3_EP_STALL;
3961
3962 ret = dwc3_send_clear_stall_ep_cmd(dep);
3963 WARN_ON_ONCE(ret);
3964 }
3965 }
3966
dwc3_gadget_disconnect_interrupt(struct dwc3 * dwc)3967 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3968 {
3969 int reg;
3970
3971 dwc->suspended = false;
3972
3973 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3974
3975 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3976 reg &= ~DWC3_DCTL_INITU1ENA;
3977 reg &= ~DWC3_DCTL_INITU2ENA;
3978 dwc3_gadget_dctl_write_safe(dwc, reg);
3979
3980 dwc->connected = false;
3981
3982 dwc3_disconnect_gadget(dwc);
3983
3984 dwc->gadget->speed = USB_SPEED_UNKNOWN;
3985 dwc->setup_packet_pending = false;
3986 dwc->gadget->wakeup_armed = false;
3987 dwc3_gadget_enable_linksts_evts(dwc, false);
3988 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3989
3990 dwc3_ep0_reset_state(dwc);
3991
3992 /*
3993 * Request PM idle to address condition where usage count is
3994 * already decremented to zero, but waiting for the disconnect
3995 * interrupt to set dwc->connected to FALSE.
3996 */
3997 pm_request_idle(dwc->dev);
3998 }
3999
dwc3_gadget_reset_interrupt(struct dwc3 * dwc)4000 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
4001 {
4002 u32 reg;
4003
4004 dwc->suspended = false;
4005
4006 /*
4007 * Ideally, dwc3_reset_gadget() would trigger the function
4008 * drivers to stop any active transfers through ep disable.
4009 * However, for functions which defer ep disable, such as mass
4010 * storage, we will need to rely on the call to stop active
4011 * transfers here, and avoid allowing of request queuing.
4012 */
4013 dwc->connected = false;
4014
4015 /*
4016 * WORKAROUND: DWC3 revisions <1.88a have an issue which
4017 * would cause a missing Disconnect Event if there's a
4018 * pending Setup Packet in the FIFO.
4019 *
4020 * There's no suggested workaround on the official Bug
4021 * report, which states that "unless the driver/application
4022 * is doing any special handling of a disconnect event,
4023 * there is no functional issue".
4024 *
4025 * Unfortunately, it turns out that we _do_ some special
4026 * handling of a disconnect event, namely complete all
4027 * pending transfers, notify gadget driver of the
4028 * disconnection, and so on.
4029 *
4030 * Our suggested workaround is to follow the Disconnect
4031 * Event steps here, instead, based on a setup_packet_pending
4032 * flag. Such flag gets set whenever we have a SETUP_PENDING
4033 * status for EP0 TRBs and gets cleared on XferComplete for the
4034 * same endpoint.
4035 *
4036 * Refers to:
4037 *
4038 * STAR#9000466709: RTL: Device : Disconnect event not
4039 * generated if setup packet pending in FIFO
4040 */
4041 if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
4042 if (dwc->setup_packet_pending)
4043 dwc3_gadget_disconnect_interrupt(dwc);
4044 }
4045
4046 dwc3_reset_gadget(dwc);
4047
4048 /*
4049 * From SNPS databook section 8.1.2, the EP0 should be in setup
4050 * phase. So ensure that EP0 is in setup phase by issuing a stall
4051 * and restart if EP0 is not in setup phase.
4052 */
4053 dwc3_ep0_reset_state(dwc);
4054
4055 /*
4056 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
4057 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
4058 * needs to ensure that it sends "a DEPENDXFER command for any active
4059 * transfers."
4060 */
4061 dwc3_stop_active_transfers(dwc);
4062 dwc->connected = true;
4063
4064 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4065 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
4066 dwc3_gadget_dctl_write_safe(dwc, reg);
4067 dwc->test_mode = false;
4068 dwc->gadget->wakeup_armed = false;
4069 dwc3_gadget_enable_linksts_evts(dwc, false);
4070 dwc3_clear_stall_all_ep(dwc);
4071
4072 /* Reset device address to zero */
4073 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4074 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
4075 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4076 }
4077
dwc3_gadget_conndone_interrupt(struct dwc3 * dwc)4078 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
4079 {
4080 struct dwc3_ep *dep;
4081 int ret;
4082 u32 reg;
4083 u8 lanes = 1;
4084 u8 speed;
4085
4086 if (!dwc->softconnect)
4087 return;
4088
4089 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
4090 speed = reg & DWC3_DSTS_CONNECTSPD;
4091 dwc->speed = speed;
4092
4093 if (DWC3_IP_IS(DWC32))
4094 lanes = DWC3_DSTS_CONNLANES(reg) + 1;
4095
4096 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4097
4098 /*
4099 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
4100 * each time on Connect Done.
4101 *
4102 * Currently we always use the reset value. If any platform
4103 * wants to set this to a different value, we need to add a
4104 * setting and update GCTL.RAMCLKSEL here.
4105 */
4106
4107 switch (speed) {
4108 case DWC3_DSTS_SUPERSPEED_PLUS:
4109 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4110 dwc->gadget->ep0->maxpacket = 512;
4111 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4112
4113 if (lanes > 1)
4114 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
4115 else
4116 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
4117 break;
4118 case DWC3_DSTS_SUPERSPEED:
4119 /*
4120 * WORKAROUND: DWC3 revisions <1.90a have an issue which
4121 * would cause a missing USB3 Reset event.
4122 *
4123 * In such situations, we should force a USB3 Reset
4124 * event by calling our dwc3_gadget_reset_interrupt()
4125 * routine.
4126 *
4127 * Refers to:
4128 *
4129 * STAR#9000483510: RTL: SS : USB3 reset event may
4130 * not be generated always when the link enters poll
4131 */
4132 if (DWC3_VER_IS_PRIOR(DWC3, 190A))
4133 dwc3_gadget_reset_interrupt(dwc);
4134
4135 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4136 dwc->gadget->ep0->maxpacket = 512;
4137 dwc->gadget->speed = USB_SPEED_SUPER;
4138
4139 if (lanes > 1) {
4140 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4141 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
4142 }
4143 break;
4144 case DWC3_DSTS_HIGHSPEED:
4145 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4146 dwc->gadget->ep0->maxpacket = 64;
4147 dwc->gadget->speed = USB_SPEED_HIGH;
4148 break;
4149 case DWC3_DSTS_FULLSPEED:
4150 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4151 dwc->gadget->ep0->maxpacket = 64;
4152 dwc->gadget->speed = USB_SPEED_FULL;
4153 break;
4154 }
4155
4156 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
4157
4158 /* Enable USB2 LPM Capability */
4159
4160 if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
4161 !dwc->usb2_gadget_lpm_disable &&
4162 (speed != DWC3_DSTS_SUPERSPEED) &&
4163 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
4164 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4165 reg |= DWC3_DCFG_LPM_CAP;
4166 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4167
4168 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4169 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
4170
4171 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
4172 (dwc->is_utmi_l1_suspend << 4));
4173
4174 /*
4175 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
4176 * DCFG.LPMCap is set, core responses with an ACK and the
4177 * BESL value in the LPM token is less than or equal to LPM
4178 * NYET threshold.
4179 */
4180 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
4181 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
4182
4183 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
4184 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
4185
4186 dwc3_gadget_dctl_write_safe(dwc, reg);
4187 } else {
4188 if (dwc->usb2_gadget_lpm_disable) {
4189 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4190 reg &= ~DWC3_DCFG_LPM_CAP;
4191 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4192 }
4193
4194 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4195 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
4196 dwc3_gadget_dctl_write_safe(dwc, reg);
4197 }
4198
4199 dep = dwc->eps[0];
4200 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4201 if (ret) {
4202 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4203 return;
4204 }
4205
4206 dep = dwc->eps[1];
4207 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4208 if (ret) {
4209 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4210 return;
4211 }
4212
4213 /*
4214 * Configure PHY via GUSB3PIPECTLn if required.
4215 *
4216 * Update GTXFIFOSIZn
4217 *
4218 * In both cases reset values should be sufficient.
4219 */
4220 }
4221
dwc3_gadget_wakeup_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4222 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
4223 {
4224 dwc->suspended = false;
4225
4226 /*
4227 * TODO take core out of low power mode when that's
4228 * implemented.
4229 */
4230
4231 if (dwc->async_callbacks && dwc->gadget_driver->resume) {
4232 spin_unlock(&dwc->lock);
4233 dwc->gadget_driver->resume(dwc->gadget);
4234 spin_lock(&dwc->lock);
4235 }
4236
4237 dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
4238 }
4239
dwc3_gadget_linksts_change_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4240 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
4241 unsigned int evtinfo)
4242 {
4243 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4244 unsigned int pwropt;
4245
4246 /*
4247 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
4248 * Hibernation mode enabled which would show up when device detects
4249 * host-initiated U3 exit.
4250 *
4251 * In that case, device will generate a Link State Change Interrupt
4252 * from U3 to RESUME which is only necessary if Hibernation is
4253 * configured in.
4254 *
4255 * There are no functional changes due to such spurious event and we
4256 * just need to ignore it.
4257 *
4258 * Refers to:
4259 *
4260 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4261 * operational mode
4262 */
4263 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4264 if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4265 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4266 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4267 (next == DWC3_LINK_STATE_RESUME)) {
4268 return;
4269 }
4270 }
4271
4272 /*
4273 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4274 * on the link partner, the USB session might do multiple entry/exit
4275 * of low power states before a transfer takes place.
4276 *
4277 * Due to this problem, we might experience lower throughput. The
4278 * suggested workaround is to disable DCTL[12:9] bits if we're
4279 * transitioning from U1/U2 to U0 and enable those bits again
4280 * after a transfer completes and there are no pending transfers
4281 * on any of the enabled endpoints.
4282 *
4283 * This is the first half of that workaround.
4284 *
4285 * Refers to:
4286 *
4287 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4288 * core send LGO_Ux entering U0
4289 */
4290 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4291 if (next == DWC3_LINK_STATE_U0) {
4292 u32 u1u2;
4293 u32 reg;
4294
4295 switch (dwc->link_state) {
4296 case DWC3_LINK_STATE_U1:
4297 case DWC3_LINK_STATE_U2:
4298 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4299 u1u2 = reg & (DWC3_DCTL_INITU2ENA
4300 | DWC3_DCTL_ACCEPTU2ENA
4301 | DWC3_DCTL_INITU1ENA
4302 | DWC3_DCTL_ACCEPTU1ENA);
4303
4304 if (!dwc->u1u2)
4305 dwc->u1u2 = reg & u1u2;
4306
4307 reg &= ~u1u2;
4308
4309 dwc3_gadget_dctl_write_safe(dwc, reg);
4310 break;
4311 default:
4312 /* do nothing */
4313 break;
4314 }
4315 }
4316 }
4317
4318 switch (next) {
4319 case DWC3_LINK_STATE_U0:
4320 if (dwc->gadget->wakeup_armed) {
4321 dwc3_gadget_enable_linksts_evts(dwc, false);
4322 dwc3_resume_gadget(dwc);
4323 dwc->suspended = false;
4324 }
4325 break;
4326 case DWC3_LINK_STATE_U1:
4327 if (dwc->speed == USB_SPEED_SUPER)
4328 dwc3_suspend_gadget(dwc);
4329 break;
4330 case DWC3_LINK_STATE_U2:
4331 case DWC3_LINK_STATE_U3:
4332 dwc3_suspend_gadget(dwc);
4333 break;
4334 case DWC3_LINK_STATE_RESUME:
4335 dwc3_resume_gadget(dwc);
4336 break;
4337 default:
4338 /* do nothing */
4339 break;
4340 }
4341
4342 dwc->link_state = next;
4343 }
4344
dwc3_gadget_suspend_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4345 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4346 unsigned int evtinfo)
4347 {
4348 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4349
4350 if (!dwc->suspended && next == DWC3_LINK_STATE_U3) {
4351 dwc->suspended = true;
4352 dwc3_suspend_gadget(dwc);
4353 }
4354
4355 dwc->link_state = next;
4356 }
4357
dwc3_gadget_interrupt(struct dwc3 * dwc,const struct dwc3_event_devt * event)4358 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4359 const struct dwc3_event_devt *event)
4360 {
4361 switch (event->type) {
4362 case DWC3_DEVICE_EVENT_DISCONNECT:
4363 dwc3_gadget_disconnect_interrupt(dwc);
4364 break;
4365 case DWC3_DEVICE_EVENT_RESET:
4366 dwc3_gadget_reset_interrupt(dwc);
4367 break;
4368 case DWC3_DEVICE_EVENT_CONNECT_DONE:
4369 dwc3_gadget_conndone_interrupt(dwc);
4370 break;
4371 case DWC3_DEVICE_EVENT_WAKEUP:
4372 dwc3_gadget_wakeup_interrupt(dwc, event->event_info);
4373 break;
4374 case DWC3_DEVICE_EVENT_HIBER_REQ:
4375 dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");
4376 break;
4377 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4378 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
4379 break;
4380 case DWC3_DEVICE_EVENT_SUSPEND:
4381 /* It changed to be suspend event for version 2.30a and above */
4382 if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
4383 dwc3_gadget_suspend_interrupt(dwc, event->event_info);
4384 break;
4385 case DWC3_DEVICE_EVENT_SOF:
4386 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4387 case DWC3_DEVICE_EVENT_CMD_CMPL:
4388 case DWC3_DEVICE_EVENT_OVERFLOW:
4389 break;
4390 default:
4391 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4392 }
4393 }
4394
dwc3_process_event_entry(struct dwc3 * dwc,const union dwc3_event * event)4395 static void dwc3_process_event_entry(struct dwc3 *dwc,
4396 const union dwc3_event *event)
4397 {
4398 trace_dwc3_event(event->raw, dwc);
4399
4400 if (!event->type.is_devspec)
4401 dwc3_endpoint_interrupt(dwc, &event->depevt);
4402 else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4403 dwc3_gadget_interrupt(dwc, &event->devt);
4404 else
4405 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4406 }
4407
dwc3_process_event_buf(struct dwc3_event_buffer * evt)4408 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4409 {
4410 struct dwc3 *dwc = evt->dwc;
4411 irqreturn_t ret = IRQ_NONE;
4412 int left;
4413
4414 left = evt->count;
4415
4416 if (!(evt->flags & DWC3_EVENT_PENDING))
4417 return IRQ_NONE;
4418
4419 while (left > 0) {
4420 union dwc3_event event;
4421
4422 event.raw = *(u32 *) (evt->cache + evt->lpos);
4423
4424 dwc3_process_event_entry(dwc, &event);
4425
4426 /*
4427 * FIXME we wrap around correctly to the next entry as
4428 * almost all entries are 4 bytes in size. There is one
4429 * entry which has 12 bytes which is a regular entry
4430 * followed by 8 bytes data. ATM I don't know how
4431 * things are organized if we get next to the a
4432 * boundary so I worry about that once we try to handle
4433 * that.
4434 */
4435 evt->lpos = (evt->lpos + 4) % evt->length;
4436 left -= 4;
4437 }
4438
4439 evt->count = 0;
4440 ret = IRQ_HANDLED;
4441
4442 /* Unmask interrupt */
4443 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4444 DWC3_GEVNTSIZ_SIZE(evt->length));
4445
4446 if (dwc->imod_interval) {
4447 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4448 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
4449 }
4450
4451 /* Keep the clearing of DWC3_EVENT_PENDING at the end */
4452 evt->flags &= ~DWC3_EVENT_PENDING;
4453
4454 return ret;
4455 }
4456
dwc3_thread_interrupt(int irq,void * _evt)4457 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4458 {
4459 struct dwc3_event_buffer *evt = _evt;
4460 struct dwc3 *dwc = evt->dwc;
4461 unsigned long flags;
4462 irqreturn_t ret = IRQ_NONE;
4463
4464 local_bh_disable();
4465 spin_lock_irqsave(&dwc->lock, flags);
4466 ret = dwc3_process_event_buf(evt);
4467 spin_unlock_irqrestore(&dwc->lock, flags);
4468 local_bh_enable();
4469
4470 return ret;
4471 }
4472
dwc3_check_event_buf(struct dwc3_event_buffer * evt)4473 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4474 {
4475 struct dwc3 *dwc = evt->dwc;
4476 u32 amount;
4477 u32 count;
4478
4479 if (pm_runtime_suspended(dwc->dev)) {
4480 dwc->pending_events = true;
4481 /*
4482 * Trigger runtime resume. The get() function will be balanced
4483 * after processing the pending events in dwc3_process_pending
4484 * events().
4485 */
4486 pm_runtime_get(dwc->dev);
4487 disable_irq_nosync(dwc->irq_gadget);
4488 return IRQ_HANDLED;
4489 }
4490
4491 /*
4492 * With PCIe legacy interrupt, test shows that top-half irq handler can
4493 * be called again after HW interrupt deassertion. Check if bottom-half
4494 * irq event handler completes before caching new event to prevent
4495 * losing events.
4496 */
4497 if (evt->flags & DWC3_EVENT_PENDING)
4498 return IRQ_HANDLED;
4499
4500 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
4501 count &= DWC3_GEVNTCOUNT_MASK;
4502 if (!count)
4503 return IRQ_NONE;
4504
4505 evt->count = count;
4506 evt->flags |= DWC3_EVENT_PENDING;
4507
4508 /* Mask interrupt */
4509 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4510 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4511
4512 amount = min(count, evt->length - evt->lpos);
4513 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4514
4515 if (amount < count)
4516 memcpy(evt->cache, evt->buf, count - amount);
4517
4518 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
4519
4520 return IRQ_WAKE_THREAD;
4521 }
4522
dwc3_interrupt(int irq,void * _evt)4523 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4524 {
4525 struct dwc3_event_buffer *evt = _evt;
4526
4527 return dwc3_check_event_buf(evt);
4528 }
4529
dwc3_gadget_get_irq(struct dwc3 * dwc)4530 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4531 {
4532 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4533 int irq;
4534
4535 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
4536 if (irq > 0)
4537 goto out;
4538
4539 if (irq == -EPROBE_DEFER)
4540 goto out;
4541
4542 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
4543 if (irq > 0)
4544 goto out;
4545
4546 if (irq == -EPROBE_DEFER)
4547 goto out;
4548
4549 irq = platform_get_irq(dwc3_pdev, 0);
4550
4551 out:
4552 return irq;
4553 }
4554
dwc_gadget_release(struct device * dev)4555 static void dwc_gadget_release(struct device *dev)
4556 {
4557 struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4558
4559 kfree(gadget);
4560 }
4561
4562 /**
4563 * dwc3_gadget_init - initializes gadget related registers
4564 * @dwc: pointer to our controller context structure
4565 *
4566 * Returns 0 on success otherwise negative errno.
4567 */
dwc3_gadget_init(struct dwc3 * dwc)4568 int dwc3_gadget_init(struct dwc3 *dwc)
4569 {
4570 int ret;
4571 int irq;
4572 struct device *dev;
4573
4574 irq = dwc3_gadget_get_irq(dwc);
4575 if (irq < 0) {
4576 ret = irq;
4577 goto err0;
4578 }
4579
4580 dwc->irq_gadget = irq;
4581
4582 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
4583 sizeof(*dwc->ep0_trb) * 2,
4584 &dwc->ep0_trb_addr, GFP_KERNEL);
4585 if (!dwc->ep0_trb) {
4586 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4587 ret = -ENOMEM;
4588 goto err0;
4589 }
4590
4591 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4592 if (!dwc->setup_buf) {
4593 ret = -ENOMEM;
4594 goto err1;
4595 }
4596
4597 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
4598 &dwc->bounce_addr, GFP_KERNEL);
4599 if (!dwc->bounce) {
4600 ret = -ENOMEM;
4601 goto err2;
4602 }
4603
4604 init_completion(&dwc->ep0_in_setup);
4605 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4606 if (!dwc->gadget) {
4607 ret = -ENOMEM;
4608 goto err3;
4609 }
4610
4611
4612 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
4613 dev = &dwc->gadget->dev;
4614 dev->platform_data = dwc;
4615 dwc->gadget->ops = &dwc3_gadget_ops;
4616 dwc->gadget->speed = USB_SPEED_UNKNOWN;
4617 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4618 dwc->gadget->sg_supported = true;
4619 dwc->gadget->name = "dwc3-gadget";
4620 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable;
4621 dwc->gadget->wakeup_capable = true;
4622
4623 /*
4624 * FIXME We might be setting max_speed to <SUPER, however versions
4625 * <2.20a of dwc3 have an issue with metastability (documented
4626 * elsewhere in this driver) which tells us we can't set max speed to
4627 * anything lower than SUPER.
4628 *
4629 * Because gadget.max_speed is only used by composite.c and function
4630 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
4631 * to happen so we avoid sending SuperSpeed Capability descriptor
4632 * together with our BOS descriptor as that could confuse host into
4633 * thinking we can handle super speed.
4634 *
4635 * Note that, in fact, we won't even support GetBOS requests when speed
4636 * is less than super speed because we don't have means, yet, to tell
4637 * composite.c that we are USB 2.0 + LPM ECN.
4638 */
4639 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4640 !dwc->dis_metastability_quirk)
4641 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4642 dwc->revision);
4643
4644 dwc->gadget->max_speed = dwc->maximum_speed;
4645 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate;
4646
4647 /*
4648 * REVISIT: Here we should clear all pending IRQs to be
4649 * sure we're starting from a well known location.
4650 */
4651
4652 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
4653 if (ret)
4654 goto err4;
4655
4656 ret = usb_add_gadget(dwc->gadget);
4657 if (ret) {
4658 dev_err(dwc->dev, "failed to add gadget\n");
4659 goto err5;
4660 }
4661
4662 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4663 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
4664 else
4665 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
4666
4667 /* No system wakeup if no gadget driver bound */
4668 if (dwc->sys_wakeup)
4669 device_wakeup_disable(dwc->sysdev);
4670
4671 return 0;
4672
4673 err5:
4674 dwc3_gadget_free_endpoints(dwc);
4675 err4:
4676 usb_put_gadget(dwc->gadget);
4677 dwc->gadget = NULL;
4678 err3:
4679 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4680 dwc->bounce_addr);
4681
4682 err2:
4683 kfree(dwc->setup_buf);
4684
4685 err1:
4686 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4687 dwc->ep0_trb, dwc->ep0_trb_addr);
4688
4689 err0:
4690 return ret;
4691 }
4692
4693 /* -------------------------------------------------------------------------- */
4694
dwc3_gadget_exit(struct dwc3 * dwc)4695 void dwc3_gadget_exit(struct dwc3 *dwc)
4696 {
4697 if (!dwc->gadget)
4698 return;
4699
4700 dwc3_enable_susphy(dwc, false);
4701 usb_del_gadget(dwc->gadget);
4702 dwc3_gadget_free_endpoints(dwc);
4703 usb_put_gadget(dwc->gadget);
4704 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4705 dwc->bounce_addr);
4706 kfree(dwc->setup_buf);
4707 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4708 dwc->ep0_trb, dwc->ep0_trb_addr);
4709 }
4710
dwc3_gadget_suspend(struct dwc3 * dwc)4711 int dwc3_gadget_suspend(struct dwc3 *dwc)
4712 {
4713 unsigned long flags;
4714 int ret;
4715
4716 ret = dwc3_gadget_soft_disconnect(dwc);
4717 if (ret)
4718 goto err;
4719
4720 spin_lock_irqsave(&dwc->lock, flags);
4721 if (dwc->gadget_driver)
4722 dwc3_disconnect_gadget(dwc);
4723 spin_unlock_irqrestore(&dwc->lock, flags);
4724
4725 return 0;
4726
4727 err:
4728 /*
4729 * Attempt to reset the controller's state. Likely no
4730 * communication can be established until the host
4731 * performs a port reset.
4732 */
4733 if (dwc->softconnect)
4734 dwc3_gadget_soft_connect(dwc);
4735
4736 return ret;
4737 }
4738
dwc3_gadget_resume(struct dwc3 * dwc)4739 int dwc3_gadget_resume(struct dwc3 *dwc)
4740 {
4741 if (!dwc->gadget_driver || !dwc->softconnect)
4742 return 0;
4743
4744 return dwc3_gadget_soft_connect(dwc);
4745 }
4746