1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4 *
5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
6 *
7 * Authors: Felipe Balbi <balbi@ti.com>,
8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9 */
10
11 #include <linux/kernel.h>
12 #include <linux/delay.h>
13 #include <linux/slab.h>
14 #include <linux/spinlock.h>
15 #include <linux/platform_device.h>
16 #include <linux/pm_runtime.h>
17 #include <linux/interrupt.h>
18 #include <linux/io.h>
19 #include <linux/list.h>
20 #include <linux/dma-mapping.h>
21
22 #include <linux/usb/ch9.h>
23 #include <linux/usb/gadget.h>
24
25 #include "debug.h"
26 #include "core.h"
27 #include "gadget.h"
28 #include "io.h"
29
30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \
31 & ~((d)->interval - 1))
32
33 /**
34 * dwc3_gadget_set_test_mode - enables usb2 test modes
35 * @dwc: pointer to our context structure
36 * @mode: the mode to set (J, K SE0 NAK, Force Enable)
37 *
38 * Caller should take care of locking. This function will return 0 on
39 * success or -EINVAL if wrong Test Selector is passed.
40 */
dwc3_gadget_set_test_mode(struct dwc3 * dwc,int mode)41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
42 {
43 u32 reg;
44
45 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
46 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
47
48 switch (mode) {
49 case USB_TEST_J:
50 case USB_TEST_K:
51 case USB_TEST_SE0_NAK:
52 case USB_TEST_PACKET:
53 case USB_TEST_FORCE_ENABLE:
54 reg |= mode << 1;
55 break;
56 default:
57 return -EINVAL;
58 }
59
60 dwc3_gadget_dctl_write_safe(dwc, reg);
61
62 return 0;
63 }
64
65 /**
66 * dwc3_gadget_get_link_state - gets current state of usb link
67 * @dwc: pointer to our context structure
68 *
69 * Caller should take care of locking. This function will
70 * return the link state on success (>= 0) or -ETIMEDOUT.
71 */
dwc3_gadget_get_link_state(struct dwc3 * dwc)72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
73 {
74 u32 reg;
75
76 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
77
78 return DWC3_DSTS_USBLNKST(reg);
79 }
80
81 /**
82 * dwc3_gadget_set_link_state - sets usb link to a particular state
83 * @dwc: pointer to our context structure
84 * @state: the state to put link into
85 *
86 * Caller should take care of locking. This function will
87 * return 0 on success or -ETIMEDOUT.
88 */
dwc3_gadget_set_link_state(struct dwc3 * dwc,enum dwc3_link_state state)89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
90 {
91 int retries = 10000;
92 u32 reg;
93
94 /*
95 * Wait until device controller is ready. Only applies to 1.94a and
96 * later RTL.
97 */
98 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) {
99 while (--retries) {
100 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
101 if (reg & DWC3_DSTS_DCNRD)
102 udelay(5);
103 else
104 break;
105 }
106
107 if (retries <= 0)
108 return -ETIMEDOUT;
109 }
110
111 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
113
114 /* set no action before sending new link state change */
115 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116
117 /* set requested state */
118 reg |= DWC3_DCTL_ULSTCHNGREQ(state);
119 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
120
121 /*
122 * The following code is racy when called from dwc3_gadget_wakeup,
123 * and is not needed, at least on newer versions
124 */
125 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
126 return 0;
127
128 /* wait for a change in DSTS */
129 retries = 10000;
130 while (--retries) {
131 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
132
133 if (DWC3_DSTS_USBLNKST(reg) == state)
134 return 0;
135
136 udelay(5);
137 }
138
139 return -ETIMEDOUT;
140 }
141
dwc3_ep0_reset_state(struct dwc3 * dwc)142 static void dwc3_ep0_reset_state(struct dwc3 *dwc)
143 {
144 unsigned int dir;
145
146 if (dwc->ep0state != EP0_SETUP_PHASE) {
147 dir = !!dwc->ep0_expect_in;
148 if (dwc->ep0state == EP0_DATA_PHASE)
149 dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);
150 else
151 dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);
152
153 dwc->eps[0]->trb_enqueue = 0;
154 dwc->eps[1]->trb_enqueue = 0;
155
156 dwc3_ep0_stall_and_restart(dwc);
157 }
158 }
159
160 /**
161 * dwc3_ep_inc_trb - increment a trb index.
162 * @index: Pointer to the TRB index to increment.
163 *
164 * The index should never point to the link TRB. After incrementing,
165 * if it is point to the link TRB, wrap around to the beginning. The
166 * link TRB is always at the last TRB entry.
167 */
dwc3_ep_inc_trb(u8 * index)168 static void dwc3_ep_inc_trb(u8 *index)
169 {
170 (*index)++;
171 if (*index == (DWC3_TRB_NUM - 1))
172 *index = 0;
173 }
174
175 /**
176 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
177 * @dep: The endpoint whose enqueue pointer we're incrementing
178 */
dwc3_ep_inc_enq(struct dwc3_ep * dep)179 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
180 {
181 dwc3_ep_inc_trb(&dep->trb_enqueue);
182 }
183
184 /**
185 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
186 * @dep: The endpoint whose enqueue pointer we're incrementing
187 */
dwc3_ep_inc_deq(struct dwc3_ep * dep)188 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
189 {
190 dwc3_ep_inc_trb(&dep->trb_dequeue);
191 }
192
dwc3_gadget_del_and_unmap_request(struct dwc3_ep * dep,struct dwc3_request * req,int status)193 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
194 struct dwc3_request *req, int status)
195 {
196 struct dwc3 *dwc = dep->dwc;
197
198 list_del(&req->list);
199 req->remaining = 0;
200 req->needs_extra_trb = false;
201 req->num_trbs = 0;
202
203 if (req->request.status == -EINPROGRESS)
204 req->request.status = status;
205
206 if (req->trb)
207 usb_gadget_unmap_request_by_dev(dwc->sysdev,
208 &req->request, req->direction);
209
210 req->trb = NULL;
211 trace_dwc3_gadget_giveback(req);
212
213 if (dep->number > 1)
214 pm_runtime_put(dwc->dev);
215 }
216
217 /**
218 * dwc3_gadget_giveback - call struct usb_request's ->complete callback
219 * @dep: The endpoint to whom the request belongs to
220 * @req: The request we're giving back
221 * @status: completion code for the request
222 *
223 * Must be called with controller's lock held and interrupts disabled. This
224 * function will unmap @req and call its ->complete() callback to notify upper
225 * layers that it has completed.
226 */
dwc3_gadget_giveback(struct dwc3_ep * dep,struct dwc3_request * req,int status)227 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
228 int status)
229 {
230 struct dwc3 *dwc = dep->dwc;
231
232 dwc3_gadget_del_and_unmap_request(dep, req, status);
233 req->status = DWC3_REQUEST_STATUS_COMPLETED;
234
235 spin_unlock(&dwc->lock);
236 usb_gadget_giveback_request(&dep->endpoint, &req->request);
237 spin_lock(&dwc->lock);
238 }
239
240 /**
241 * dwc3_send_gadget_generic_command - issue a generic command for the controller
242 * @dwc: pointer to the controller context
243 * @cmd: the command to be issued
244 * @param: command parameter
245 *
246 * Caller should take care of locking. Issue @cmd with a given @param to @dwc
247 * and wait for its completion.
248 */
dwc3_send_gadget_generic_command(struct dwc3 * dwc,unsigned int cmd,u32 param)249 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd,
250 u32 param)
251 {
252 u32 timeout = 500;
253 int status = 0;
254 int ret = 0;
255 u32 reg;
256
257 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
258 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
259
260 do {
261 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
262 if (!(reg & DWC3_DGCMD_CMDACT)) {
263 status = DWC3_DGCMD_STATUS(reg);
264 if (status)
265 ret = -EINVAL;
266 break;
267 }
268 } while (--timeout);
269
270 if (!timeout) {
271 ret = -ETIMEDOUT;
272 status = -ETIMEDOUT;
273 }
274
275 trace_dwc3_gadget_generic_cmd(cmd, param, status);
276
277 return ret;
278 }
279
280 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async);
281
282 /**
283 * dwc3_send_gadget_ep_cmd - issue an endpoint command
284 * @dep: the endpoint to which the command is going to be issued
285 * @cmd: the command to be issued
286 * @params: parameters to the command
287 *
288 * Caller should handle locking. This function will issue @cmd with given
289 * @params to @dep and wait for its completion.
290 *
291 * According to the programming guide, if the link state is in L1/L2/U3,
292 * then sending the Start Transfer command may not complete. The
293 * programming guide suggested to bring the link state back to ON/U0 by
294 * performing remote wakeup prior to sending the command. However, don't
295 * initiate remote wakeup when the user/function does not send wakeup
296 * request via wakeup ops. Send the command when it's allowed.
297 *
298 * Notes:
299 * For L1 link state, issuing a command requires the clearing of
300 * GUSB2PHYCFG.SUSPENDUSB2, which turns on the signal required to complete
301 * the given command (usually within 50us). This should happen within the
302 * command timeout set by driver. No additional step is needed.
303 *
304 * For L2 or U3 link state, the gadget is in USB suspend. Care should be
305 * taken when sending Start Transfer command to ensure that it's done after
306 * USB resume.
307 */
dwc3_send_gadget_ep_cmd(struct dwc3_ep * dep,unsigned int cmd,struct dwc3_gadget_ep_cmd_params * params)308 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd,
309 struct dwc3_gadget_ep_cmd_params *params)
310 {
311 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
312 struct dwc3 *dwc = dep->dwc;
313 u32 timeout = 5000;
314 u32 saved_config = 0;
315 u32 reg;
316
317 int cmd_status = 0;
318 int ret = -EINVAL;
319
320 /*
321 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or
322 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an
323 * endpoint command.
324 *
325 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY
326 * settings. Restore them after the command is completed.
327 *
328 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2
329 */
330 if (dwc->gadget->speed <= USB_SPEED_HIGH ||
331 DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER) {
332 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
333 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
334 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY;
335 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
336 }
337
338 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) {
339 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM;
340 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM;
341 }
342
343 if (saved_config)
344 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
345 }
346
347 /*
348 * For some commands such as Update Transfer command, DEPCMDPARn
349 * registers are reserved. Since the driver often sends Update Transfer
350 * command, don't write to DEPCMDPARn to avoid register write delays and
351 * improve performance.
352 */
353 if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) {
354 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
355 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
356 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
357 }
358
359 /*
360 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
361 * not relying on XferNotReady, we can make use of a special "No
362 * Response Update Transfer" command where we should clear both CmdAct
363 * and CmdIOC bits.
364 *
365 * With this, we don't need to wait for command completion and can
366 * straight away issue further commands to the endpoint.
367 *
368 * NOTICE: We're making an assumption that control endpoints will never
369 * make use of Update Transfer command. This is a safe assumption
370 * because we can never have more than one request at a time with
371 * Control Endpoints. If anybody changes that assumption, this chunk
372 * needs to be updated accordingly.
373 */
374 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
375 !usb_endpoint_xfer_isoc(desc))
376 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
377 else
378 cmd |= DWC3_DEPCMD_CMDACT;
379
380 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
381
382 if (!(cmd & DWC3_DEPCMD_CMDACT) ||
383 (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_ENDTRANSFER &&
384 !(cmd & DWC3_DEPCMD_CMDIOC))) {
385 ret = 0;
386 goto skip_status;
387 }
388
389 do {
390 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
391 if (!(reg & DWC3_DEPCMD_CMDACT)) {
392 cmd_status = DWC3_DEPCMD_STATUS(reg);
393
394 switch (cmd_status) {
395 case 0:
396 ret = 0;
397 break;
398 case DEPEVT_TRANSFER_NO_RESOURCE:
399 dev_WARN(dwc->dev, "No resource for %s\n",
400 dep->name);
401 ret = -EINVAL;
402 break;
403 case DEPEVT_TRANSFER_BUS_EXPIRY:
404 /*
405 * SW issues START TRANSFER command to
406 * isochronous ep with future frame interval. If
407 * future interval time has already passed when
408 * core receives the command, it will respond
409 * with an error status of 'Bus Expiry'.
410 *
411 * Instead of always returning -EINVAL, let's
412 * give a hint to the gadget driver that this is
413 * the case by returning -EAGAIN.
414 */
415 ret = -EAGAIN;
416 break;
417 default:
418 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
419 }
420
421 break;
422 }
423 } while (--timeout);
424
425 if (timeout == 0) {
426 ret = -ETIMEDOUT;
427 cmd_status = -ETIMEDOUT;
428 }
429
430 skip_status:
431 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
432
433 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
434 if (ret == 0)
435 dep->flags |= DWC3_EP_TRANSFER_STARTED;
436
437 if (ret != -ETIMEDOUT)
438 dwc3_gadget_ep_get_transfer_index(dep);
439 }
440
441 if (saved_config) {
442 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
443 reg |= saved_config;
444 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
445 }
446
447 return ret;
448 }
449
dwc3_send_clear_stall_ep_cmd(struct dwc3_ep * dep)450 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
451 {
452 struct dwc3 *dwc = dep->dwc;
453 struct dwc3_gadget_ep_cmd_params params;
454 u32 cmd = DWC3_DEPCMD_CLEARSTALL;
455
456 /*
457 * As of core revision 2.60a the recommended programming model
458 * is to set the ClearPendIN bit when issuing a Clear Stall EP
459 * command for IN endpoints. This is to prevent an issue where
460 * some (non-compliant) hosts may not send ACK TPs for pending
461 * IN transfers due to a mishandled error condition. Synopsys
462 * STAR 9000614252.
463 */
464 if (dep->direction &&
465 !DWC3_VER_IS_PRIOR(DWC3, 260A) &&
466 (dwc->gadget->speed >= USB_SPEED_SUPER))
467 cmd |= DWC3_DEPCMD_CLEARPENDIN;
468
469 memset(¶ms, 0, sizeof(params));
470
471 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
472 }
473
dwc3_trb_dma_offset(struct dwc3_ep * dep,struct dwc3_trb * trb)474 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
475 struct dwc3_trb *trb)
476 {
477 u32 offset = (char *) trb - (char *) dep->trb_pool;
478
479 return dep->trb_pool_dma + offset;
480 }
481
dwc3_alloc_trb_pool(struct dwc3_ep * dep)482 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
483 {
484 struct dwc3 *dwc = dep->dwc;
485
486 if (dep->trb_pool)
487 return 0;
488
489 dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
490 sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
491 &dep->trb_pool_dma, GFP_KERNEL);
492 if (!dep->trb_pool) {
493 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
494 dep->name);
495 return -ENOMEM;
496 }
497
498 return 0;
499 }
500
dwc3_free_trb_pool(struct dwc3_ep * dep)501 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
502 {
503 struct dwc3 *dwc = dep->dwc;
504
505 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
506 dep->trb_pool, dep->trb_pool_dma);
507
508 dep->trb_pool = NULL;
509 dep->trb_pool_dma = 0;
510 }
511
dwc3_gadget_set_xfer_resource(struct dwc3_ep * dep)512 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
513 {
514 struct dwc3_gadget_ep_cmd_params params;
515
516 memset(¶ms, 0x00, sizeof(params));
517
518 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
519
520 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
521 ¶ms);
522 }
523
524 /**
525 * dwc3_gadget_start_config - configure ep resources
526 * @dep: endpoint that is being enabled
527 *
528 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
529 * completion, it will set Transfer Resource for all available endpoints.
530 *
531 * The assignment of transfer resources cannot perfectly follow the data book
532 * due to the fact that the controller driver does not have all knowledge of the
533 * configuration in advance. It is given this information piecemeal by the
534 * composite gadget framework after every SET_CONFIGURATION and
535 * SET_INTERFACE. Trying to follow the databook programming model in this
536 * scenario can cause errors. For two reasons:
537 *
538 * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
539 * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
540 * incorrect in the scenario of multiple interfaces.
541 *
542 * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
543 * endpoint on alt setting (8.1.6).
544 *
545 * The following simplified method is used instead:
546 *
547 * All hardware endpoints can be assigned a transfer resource and this setting
548 * will stay persistent until either a core reset or hibernation. So whenever we
549 * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
550 * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
551 * guaranteed that there are as many transfer resources as endpoints.
552 *
553 * This function is called for each endpoint when it is being enabled but is
554 * triggered only when called for EP0-out, which always happens first, and which
555 * should only happen in one of the above conditions.
556 */
dwc3_gadget_start_config(struct dwc3_ep * dep)557 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
558 {
559 struct dwc3_gadget_ep_cmd_params params;
560 struct dwc3 *dwc;
561 u32 cmd;
562 int i;
563 int ret;
564
565 if (dep->number)
566 return 0;
567
568 memset(¶ms, 0x00, sizeof(params));
569 cmd = DWC3_DEPCMD_DEPSTARTCFG;
570 dwc = dep->dwc;
571
572 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
573 if (ret)
574 return ret;
575
576 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
577 struct dwc3_ep *dep = dwc->eps[i];
578
579 if (!dep)
580 continue;
581
582 ret = dwc3_gadget_set_xfer_resource(dep);
583 if (ret)
584 return ret;
585 }
586
587 return 0;
588 }
589
dwc3_gadget_set_ep_config(struct dwc3_ep * dep,unsigned int action)590 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
591 {
592 const struct usb_ss_ep_comp_descriptor *comp_desc;
593 const struct usb_endpoint_descriptor *desc;
594 struct dwc3_gadget_ep_cmd_params params;
595 struct dwc3 *dwc = dep->dwc;
596
597 comp_desc = dep->endpoint.comp_desc;
598 desc = dep->endpoint.desc;
599
600 memset(¶ms, 0x00, sizeof(params));
601
602 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
603 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
604
605 /* Burst size is only needed in SuperSpeed mode */
606 if (dwc->gadget->speed >= USB_SPEED_SUPER) {
607 u32 burst = dep->endpoint.maxburst;
608
609 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
610 }
611
612 params.param0 |= action;
613 if (action == DWC3_DEPCFG_ACTION_RESTORE)
614 params.param2 |= dep->saved_state;
615
616 if (usb_endpoint_xfer_control(desc))
617 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
618
619 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
620 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
621
622 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
623 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
624 | DWC3_DEPCFG_XFER_COMPLETE_EN
625 | DWC3_DEPCFG_STREAM_EVENT_EN;
626 dep->stream_capable = true;
627 }
628
629 if (!usb_endpoint_xfer_control(desc))
630 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
631
632 /*
633 * We are doing 1:1 mapping for endpoints, meaning
634 * Physical Endpoints 2 maps to Logical Endpoint 2 and
635 * so on. We consider the direction bit as part of the physical
636 * endpoint number. So USB endpoint 0x81 is 0x03.
637 */
638 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
639
640 /*
641 * We must use the lower 16 TX FIFOs even though
642 * HW might have more
643 */
644 if (dep->direction)
645 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
646
647 if (desc->bInterval) {
648 u8 bInterval_m1;
649
650 /*
651 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13.
652 *
653 * NOTE: The programming guide incorrectly stated bInterval_m1
654 * must be set to 0 when operating in fullspeed. Internally the
655 * controller does not have this limitation. See DWC_usb3x
656 * programming guide section 3.2.2.1.
657 */
658 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);
659
660 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT &&
661 dwc->gadget->speed == USB_SPEED_FULL)
662 dep->interval = desc->bInterval;
663 else
664 dep->interval = 1 << (desc->bInterval - 1);
665
666 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1);
667 }
668
669 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms);
670 }
671
672 /**
673 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
674 * @dwc: pointer to the DWC3 context
675 * @mult: multiplier to be used when calculating the fifo_size
676 *
677 * Calculates the size value based on the equation below:
678 *
679 * DWC3 revision 280A and prior:
680 * fifo_size = mult * (max_packet / mdwidth) + 1;
681 *
682 * DWC3 revision 290A and onwards:
683 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
684 *
685 * The max packet size is set to 1024, as the txfifo requirements mainly apply
686 * to super speed USB use cases. However, it is safe to overestimate the fifo
687 * allocations for other scenarios, i.e. high speed USB.
688 */
dwc3_gadget_calc_tx_fifo_size(struct dwc3 * dwc,int mult)689 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult)
690 {
691 int max_packet = 1024;
692 int fifo_size;
693 int mdwidth;
694
695 mdwidth = dwc3_mdwidth(dwc);
696
697 /* MDWIDTH is represented in bits, we need it in bytes */
698 mdwidth >>= 3;
699
700 if (DWC3_VER_IS_PRIOR(DWC3, 290A))
701 fifo_size = mult * (max_packet / mdwidth) + 1;
702 else
703 fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1;
704 return fifo_size;
705 }
706
707 /**
708 * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
709 * @dwc: pointer to the DWC3 context
710 *
711 * Iterates through all the endpoint registers and clears the previous txfifo
712 * allocations.
713 */
dwc3_gadget_clear_tx_fifos(struct dwc3 * dwc)714 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc)
715 {
716 struct dwc3_ep *dep;
717 int fifo_depth;
718 int size;
719 int num;
720
721 if (!dwc->do_fifo_resize)
722 return;
723
724 /* Read ep0IN related TXFIFO size */
725 dep = dwc->eps[1];
726 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
727 if (DWC3_IP_IS(DWC3))
728 fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size);
729 else
730 fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size);
731
732 dwc->last_fifo_depth = fifo_depth;
733 /* Clear existing TXFIFO for all IN eps except ep0 */
734 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM);
735 num += 2) {
736 dep = dwc->eps[num];
737 /* Don't change TXFRAMNUM on usb31 version */
738 size = DWC3_IP_IS(DWC3) ? 0 :
739 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &
740 DWC31_GTXFIFOSIZ_TXFRAMNUM;
741
742 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);
743 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;
744 }
745 dwc->num_ep_resized = 0;
746 }
747
748 /*
749 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
750 * @dwc: pointer to our context structure
751 *
752 * This function will a best effort FIFO allocation in order
753 * to improve FIFO usage and throughput, while still allowing
754 * us to enable as many endpoints as possible.
755 *
756 * Keep in mind that this operation will be highly dependent
757 * on the configured size for RAM1 - which contains TxFifo -,
758 * the amount of endpoints enabled on coreConsultant tool, and
759 * the width of the Master Bus.
760 *
761 * In general, FIFO depths are represented with the following equation:
762 *
763 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
764 *
765 * In conjunction with dwc3_gadget_check_config(), this resizing logic will
766 * ensure that all endpoints will have enough internal memory for one max
767 * packet per endpoint.
768 */
dwc3_gadget_resize_tx_fifos(struct dwc3_ep * dep)769 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep)
770 {
771 struct dwc3 *dwc = dep->dwc;
772 int fifo_0_start;
773 int ram1_depth;
774 int fifo_size;
775 int min_depth;
776 int num_in_ep;
777 int remaining;
778 int num_fifos = 1;
779 int fifo;
780 int tmp;
781
782 if (!dwc->do_fifo_resize)
783 return 0;
784
785 /* resize IN endpoints except ep0 */
786 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)
787 return 0;
788
789 /* bail if already resized */
790 if (dep->flags & DWC3_EP_TXFIFO_RESIZED)
791 return 0;
792
793 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
794
795 if ((dep->endpoint.maxburst > 1 &&
796 usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||
797 usb_endpoint_xfer_isoc(dep->endpoint.desc))
798 num_fifos = 3;
799
800 if (dep->endpoint.maxburst > 6 &&
801 (usb_endpoint_xfer_bulk(dep->endpoint.desc) ||
802 usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))
803 num_fifos = dwc->tx_fifo_resize_max_num;
804
805 /* FIFO size for a single buffer */
806 fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1);
807
808 /* Calculate the number of remaining EPs w/o any FIFO */
809 num_in_ep = dwc->max_cfg_eps;
810 num_in_ep -= dwc->num_ep_resized;
811
812 /* Reserve at least one FIFO for the number of IN EPs */
813 min_depth = num_in_ep * (fifo + 1);
814 remaining = ram1_depth - min_depth - dwc->last_fifo_depth;
815 remaining = max_t(int, 0, remaining);
816 /*
817 * We've already reserved 1 FIFO per EP, so check what we can fit in
818 * addition to it. If there is not enough remaining space, allocate
819 * all the remaining space to the EP.
820 */
821 fifo_size = (num_fifos - 1) * fifo;
822 if (remaining < fifo_size)
823 fifo_size = remaining;
824
825 fifo_size += fifo;
826 /* Last increment according to the TX FIFO size equation */
827 fifo_size++;
828
829 /* Check if TXFIFOs start at non-zero addr */
830 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));
831 fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp);
832
833 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));
834 if (DWC3_IP_IS(DWC3))
835 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
836 else
837 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
838
839 /* Check fifo size allocation doesn't exceed available RAM size. */
840 if (dwc->last_fifo_depth >= ram1_depth) {
841 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",
842 dwc->last_fifo_depth, ram1_depth,
843 dep->endpoint.name, fifo_size);
844 if (DWC3_IP_IS(DWC3))
845 fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);
846 else
847 fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);
848
849 dwc->last_fifo_depth -= fifo_size;
850 return -ENOMEM;
851 }
852
853 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);
854 dep->flags |= DWC3_EP_TXFIFO_RESIZED;
855 dwc->num_ep_resized++;
856
857 return 0;
858 }
859
860 /**
861 * __dwc3_gadget_ep_enable - initializes a hw endpoint
862 * @dep: endpoint to be initialized
863 * @action: one of INIT, MODIFY or RESTORE
864 *
865 * Caller should take care of locking. Execute all necessary commands to
866 * initialize a HW endpoint so it can be used by a gadget driver.
867 */
__dwc3_gadget_ep_enable(struct dwc3_ep * dep,unsigned int action)868 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
869 {
870 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
871 struct dwc3 *dwc = dep->dwc;
872
873 u32 reg;
874 int ret;
875
876 if (!(dep->flags & DWC3_EP_ENABLED)) {
877 ret = dwc3_gadget_resize_tx_fifos(dep);
878 if (ret)
879 return ret;
880
881 ret = dwc3_gadget_start_config(dep);
882 if (ret)
883 return ret;
884 }
885
886 ret = dwc3_gadget_set_ep_config(dep, action);
887 if (ret)
888 return ret;
889
890 if (!(dep->flags & DWC3_EP_ENABLED)) {
891 struct dwc3_trb *trb_st_hw;
892 struct dwc3_trb *trb_link;
893
894 dep->type = usb_endpoint_type(desc);
895 dep->flags |= DWC3_EP_ENABLED;
896
897 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
898 reg |= DWC3_DALEPENA_EP(dep->number);
899 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
900
901 dep->trb_dequeue = 0;
902 dep->trb_enqueue = 0;
903
904 if (usb_endpoint_xfer_control(desc))
905 goto out;
906
907 /* Initialize the TRB ring */
908 memset(dep->trb_pool, 0,
909 sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
910
911 /* Link TRB. The HWO bit is never reset */
912 trb_st_hw = &dep->trb_pool[0];
913
914 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
915 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
916 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
917 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
918 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
919 }
920
921 /*
922 * Issue StartTransfer here with no-op TRB so we can always rely on No
923 * Response Update Transfer command.
924 */
925 if (usb_endpoint_xfer_bulk(desc) ||
926 usb_endpoint_xfer_int(desc)) {
927 struct dwc3_gadget_ep_cmd_params params;
928 struct dwc3_trb *trb;
929 dma_addr_t trb_dma;
930 u32 cmd;
931
932 memset(¶ms, 0, sizeof(params));
933 trb = &dep->trb_pool[0];
934 trb_dma = dwc3_trb_dma_offset(dep, trb);
935
936 params.param0 = upper_32_bits(trb_dma);
937 params.param1 = lower_32_bits(trb_dma);
938
939 cmd = DWC3_DEPCMD_STARTTRANSFER;
940
941 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
942 if (ret < 0)
943 return ret;
944
945 if (dep->stream_capable) {
946 /*
947 * For streams, at start, there maybe a race where the
948 * host primes the endpoint before the function driver
949 * queues a request to initiate a stream. In that case,
950 * the controller will not see the prime to generate the
951 * ERDY and start stream. To workaround this, issue a
952 * no-op TRB as normal, but end it immediately. As a
953 * result, when the function driver queues the request,
954 * the next START_TRANSFER command will cause the
955 * controller to generate an ERDY to initiate the
956 * stream.
957 */
958 dwc3_stop_active_transfer(dep, true, true);
959
960 /*
961 * All stream eps will reinitiate stream on NoStream
962 * rejection until we can determine that the host can
963 * prime after the first transfer.
964 *
965 * However, if the controller is capable of
966 * TXF_FLUSH_BYPASS, then IN direction endpoints will
967 * automatically restart the stream without the driver
968 * initiation.
969 */
970 if (!dep->direction ||
971 !(dwc->hwparams.hwparams9 &
972 DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS))
973 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;
974 }
975 }
976
977 out:
978 trace_dwc3_gadget_ep_enable(dep);
979
980 return 0;
981 }
982
dwc3_remove_requests(struct dwc3 * dwc,struct dwc3_ep * dep,int status)983 void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep, int status)
984 {
985 struct dwc3_request *req;
986
987 dwc3_stop_active_transfer(dep, true, false);
988
989 /* If endxfer is delayed, avoid unmapping requests */
990 if (dep->flags & DWC3_EP_DELAY_STOP)
991 return;
992
993 /* - giveback all requests to gadget driver */
994 while (!list_empty(&dep->started_list)) {
995 req = next_request(&dep->started_list);
996
997 dwc3_gadget_giveback(dep, req, status);
998 }
999
1000 while (!list_empty(&dep->pending_list)) {
1001 req = next_request(&dep->pending_list);
1002
1003 dwc3_gadget_giveback(dep, req, status);
1004 }
1005
1006 while (!list_empty(&dep->cancelled_list)) {
1007 req = next_request(&dep->cancelled_list);
1008
1009 dwc3_gadget_giveback(dep, req, status);
1010 }
1011 }
1012
1013 /**
1014 * __dwc3_gadget_ep_disable - disables a hw endpoint
1015 * @dep: the endpoint to disable
1016 *
1017 * This function undoes what __dwc3_gadget_ep_enable did and also removes
1018 * requests which are currently being processed by the hardware and those which
1019 * are not yet scheduled.
1020 *
1021 * Caller should take care of locking.
1022 */
__dwc3_gadget_ep_disable(struct dwc3_ep * dep)1023 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
1024 {
1025 struct dwc3 *dwc = dep->dwc;
1026 u32 reg;
1027 u32 mask;
1028
1029 trace_dwc3_gadget_ep_disable(dep);
1030
1031 /* make sure HW endpoint isn't stalled */
1032 if (dep->flags & DWC3_EP_STALL)
1033 __dwc3_gadget_ep_set_halt(dep, 0, false);
1034
1035 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
1036 reg &= ~DWC3_DALEPENA_EP(dep->number);
1037 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
1038
1039 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
1040
1041 dep->stream_capable = false;
1042 dep->type = 0;
1043 mask = DWC3_EP_TXFIFO_RESIZED;
1044 /*
1045 * dwc3_remove_requests() can exit early if DWC3 EP delayed stop is
1046 * set. Do not clear DEP flags, so that the end transfer command will
1047 * be reattempted during the next SETUP stage.
1048 */
1049 if (dep->flags & DWC3_EP_DELAY_STOP)
1050 mask |= (DWC3_EP_DELAY_STOP | DWC3_EP_TRANSFER_STARTED);
1051 dep->flags &= mask;
1052
1053 /* Clear out the ep descriptors for non-ep0 */
1054 if (dep->number > 1) {
1055 dep->endpoint.comp_desc = NULL;
1056 dep->endpoint.desc = NULL;
1057 }
1058
1059 return 0;
1060 }
1061
1062 /* -------------------------------------------------------------------------- */
1063
dwc3_gadget_ep0_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)1064 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
1065 const struct usb_endpoint_descriptor *desc)
1066 {
1067 return -EINVAL;
1068 }
1069
dwc3_gadget_ep0_disable(struct usb_ep * ep)1070 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
1071 {
1072 return -EINVAL;
1073 }
1074
1075 /* -------------------------------------------------------------------------- */
1076
dwc3_gadget_ep_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)1077 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
1078 const struct usb_endpoint_descriptor *desc)
1079 {
1080 struct dwc3_ep *dep;
1081 struct dwc3 *dwc;
1082 unsigned long flags;
1083 int ret;
1084
1085 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
1086 pr_debug("dwc3: invalid parameters\n");
1087 return -EINVAL;
1088 }
1089
1090 if (!desc->wMaxPacketSize) {
1091 pr_debug("dwc3: missing wMaxPacketSize\n");
1092 return -EINVAL;
1093 }
1094
1095 dep = to_dwc3_ep(ep);
1096 dwc = dep->dwc;
1097
1098 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
1099 "%s is already enabled\n",
1100 dep->name))
1101 return 0;
1102
1103 spin_lock_irqsave(&dwc->lock, flags);
1104 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1105 spin_unlock_irqrestore(&dwc->lock, flags);
1106
1107 return ret;
1108 }
1109
dwc3_gadget_ep_disable(struct usb_ep * ep)1110 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
1111 {
1112 struct dwc3_ep *dep;
1113 struct dwc3 *dwc;
1114 unsigned long flags;
1115 int ret;
1116
1117 if (!ep) {
1118 pr_debug("dwc3: invalid parameters\n");
1119 return -EINVAL;
1120 }
1121
1122 dep = to_dwc3_ep(ep);
1123 dwc = dep->dwc;
1124
1125 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
1126 "%s is already disabled\n",
1127 dep->name))
1128 return 0;
1129
1130 spin_lock_irqsave(&dwc->lock, flags);
1131 ret = __dwc3_gadget_ep_disable(dep);
1132 spin_unlock_irqrestore(&dwc->lock, flags);
1133
1134 return ret;
1135 }
1136
dwc3_gadget_ep_alloc_request(struct usb_ep * ep,gfp_t gfp_flags)1137 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
1138 gfp_t gfp_flags)
1139 {
1140 struct dwc3_request *req;
1141 struct dwc3_ep *dep = to_dwc3_ep(ep);
1142
1143 req = kzalloc(sizeof(*req), gfp_flags);
1144 if (!req)
1145 return NULL;
1146
1147 req->direction = dep->direction;
1148 req->epnum = dep->number;
1149 req->dep = dep;
1150 req->status = DWC3_REQUEST_STATUS_UNKNOWN;
1151
1152 trace_dwc3_alloc_request(req);
1153
1154 return &req->request;
1155 }
1156
dwc3_gadget_ep_free_request(struct usb_ep * ep,struct usb_request * request)1157 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
1158 struct usb_request *request)
1159 {
1160 struct dwc3_request *req = to_dwc3_request(request);
1161
1162 trace_dwc3_free_request(req);
1163 kfree(req);
1164 }
1165
1166 /**
1167 * dwc3_ep_prev_trb - returns the previous TRB in the ring
1168 * @dep: The endpoint with the TRB ring
1169 * @index: The index of the current TRB in the ring
1170 *
1171 * Returns the TRB prior to the one pointed to by the index. If the
1172 * index is 0, we will wrap backwards, skip the link TRB, and return
1173 * the one just before that.
1174 */
dwc3_ep_prev_trb(struct dwc3_ep * dep,u8 index)1175 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1176 {
1177 u8 tmp = index;
1178
1179 if (!tmp)
1180 tmp = DWC3_TRB_NUM - 1;
1181
1182 return &dep->trb_pool[tmp - 1];
1183 }
1184
dwc3_calc_trbs_left(struct dwc3_ep * dep)1185 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1186 {
1187 u8 trbs_left;
1188
1189 /*
1190 * If the enqueue & dequeue are equal then the TRB ring is either full
1191 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs
1192 * pending to be processed by the driver.
1193 */
1194 if (dep->trb_enqueue == dep->trb_dequeue) {
1195 /*
1196 * If there is any request remained in the started_list at
1197 * this point, that means there is no TRB available.
1198 */
1199 if (!list_empty(&dep->started_list))
1200 return 0;
1201
1202 return DWC3_TRB_NUM - 1;
1203 }
1204
1205 trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1206 trbs_left &= (DWC3_TRB_NUM - 1);
1207
1208 if (dep->trb_dequeue < dep->trb_enqueue)
1209 trbs_left--;
1210
1211 return trbs_left;
1212 }
1213
1214 /**
1215 * dwc3_prepare_one_trb - setup one TRB from one request
1216 * @dep: endpoint for which this request is prepared
1217 * @req: dwc3_request pointer
1218 * @trb_length: buffer size of the TRB
1219 * @chain: should this TRB be chained to the next?
1220 * @node: only for isochronous endpoints. First TRB needs different type.
1221 * @use_bounce_buffer: set to use bounce buffer
1222 * @must_interrupt: set to interrupt on TRB completion
1223 */
dwc3_prepare_one_trb(struct dwc3_ep * dep,struct dwc3_request * req,unsigned int trb_length,unsigned int chain,unsigned int node,bool use_bounce_buffer,bool must_interrupt)1224 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1225 struct dwc3_request *req, unsigned int trb_length,
1226 unsigned int chain, unsigned int node, bool use_bounce_buffer,
1227 bool must_interrupt)
1228 {
1229 struct dwc3_trb *trb;
1230 dma_addr_t dma;
1231 unsigned int stream_id = req->request.stream_id;
1232 unsigned int short_not_ok = req->request.short_not_ok;
1233 unsigned int no_interrupt = req->request.no_interrupt;
1234 unsigned int is_last = req->request.is_last;
1235 struct dwc3 *dwc = dep->dwc;
1236 struct usb_gadget *gadget = dwc->gadget;
1237 enum usb_device_speed speed = gadget->speed;
1238
1239 if (use_bounce_buffer)
1240 dma = dep->dwc->bounce_addr;
1241 else if (req->request.num_sgs > 0)
1242 dma = sg_dma_address(req->start_sg);
1243 else
1244 dma = req->request.dma;
1245
1246 trb = &dep->trb_pool[dep->trb_enqueue];
1247
1248 if (!req->trb) {
1249 dwc3_gadget_move_started_request(req);
1250 req->trb = trb;
1251 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1252 }
1253
1254 req->num_trbs++;
1255
1256 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);
1257 trb->bpl = lower_32_bits(dma);
1258 trb->bph = upper_32_bits(dma);
1259
1260 switch (usb_endpoint_type(dep->endpoint.desc)) {
1261 case USB_ENDPOINT_XFER_CONTROL:
1262 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
1263 break;
1264
1265 case USB_ENDPOINT_XFER_ISOC:
1266 if (!node) {
1267 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
1268
1269 /*
1270 * USB Specification 2.0 Section 5.9.2 states that: "If
1271 * there is only a single transaction in the microframe,
1272 * only a DATA0 data packet PID is used. If there are
1273 * two transactions per microframe, DATA1 is used for
1274 * the first transaction data packet and DATA0 is used
1275 * for the second transaction data packet. If there are
1276 * three transactions per microframe, DATA2 is used for
1277 * the first transaction data packet, DATA1 is used for
1278 * the second, and DATA0 is used for the third."
1279 *
1280 * IOW, we should satisfy the following cases:
1281 *
1282 * 1) length <= maxpacket
1283 * - DATA0
1284 *
1285 * 2) maxpacket < length <= (2 * maxpacket)
1286 * - DATA1, DATA0
1287 *
1288 * 3) (2 * maxpacket) < length <= (3 * maxpacket)
1289 * - DATA2, DATA1, DATA0
1290 */
1291 if (speed == USB_SPEED_HIGH) {
1292 struct usb_ep *ep = &dep->endpoint;
1293 unsigned int mult = 2;
1294 unsigned int maxp = usb_endpoint_maxp(ep->desc);
1295
1296 if (req->request.length <= (2 * maxp))
1297 mult--;
1298
1299 if (req->request.length <= maxp)
1300 mult--;
1301
1302 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
1303 }
1304 } else {
1305 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
1306 }
1307
1308 if (!no_interrupt && !chain)
1309 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1310 break;
1311
1312 case USB_ENDPOINT_XFER_BULK:
1313 case USB_ENDPOINT_XFER_INT:
1314 trb->ctrl = DWC3_TRBCTL_NORMAL;
1315 break;
1316 default:
1317 /*
1318 * This is only possible with faulty memory because we
1319 * checked it already :)
1320 */
1321 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
1322 usb_endpoint_type(dep->endpoint.desc));
1323 }
1324
1325 /*
1326 * Enable Continue on Short Packet
1327 * when endpoint is not a stream capable
1328 */
1329 if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1330 if (!dep->stream_capable)
1331 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1332
1333 if (short_not_ok)
1334 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1335 }
1336
1337 /* All TRBs setup for MST must set CSP=1 when LST=0 */
1338 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))
1339 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1340
1341 if ((!no_interrupt && !chain) || must_interrupt)
1342 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1343
1344 if (chain)
1345 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1346 else if (dep->stream_capable && is_last &&
1347 !DWC3_MST_CAPABLE(&dwc->hwparams))
1348 trb->ctrl |= DWC3_TRB_CTRL_LST;
1349
1350 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1351 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1352
1353 /*
1354 * As per data book 4.2.3.2TRB Control Bit Rules section
1355 *
1356 * The controller autonomously checks the HWO field of a TRB to determine if the
1357 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB
1358 * is valid before setting the HWO field to '1'. In most systems, this means that
1359 * software must update the fourth DWORD of a TRB last.
1360 *
1361 * However there is a possibility of CPU re-ordering here which can cause
1362 * controller to observe the HWO bit set prematurely.
1363 * Add a write memory barrier to prevent CPU re-ordering.
1364 */
1365 wmb();
1366 trb->ctrl |= DWC3_TRB_CTRL_HWO;
1367
1368 dwc3_ep_inc_enq(dep);
1369
1370 trace_dwc3_prepare_trb(dep, trb);
1371 }
1372
dwc3_needs_extra_trb(struct dwc3_ep * dep,struct dwc3_request * req)1373 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req)
1374 {
1375 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1376 unsigned int rem = req->request.length % maxp;
1377
1378 if ((req->request.length && req->request.zero && !rem &&
1379 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||
1380 (!req->direction && rem))
1381 return true;
1382
1383 return false;
1384 }
1385
1386 /**
1387 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1388 * @dep: The endpoint that the request belongs to
1389 * @req: The request to prepare
1390 * @entry_length: The last SG entry size
1391 * @node: Indicates whether this is not the first entry (for isoc only)
1392 *
1393 * Return the number of TRBs prepared.
1394 */
dwc3_prepare_last_sg(struct dwc3_ep * dep,struct dwc3_request * req,unsigned int entry_length,unsigned int node)1395 static int dwc3_prepare_last_sg(struct dwc3_ep *dep,
1396 struct dwc3_request *req, unsigned int entry_length,
1397 unsigned int node)
1398 {
1399 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1400 unsigned int rem = req->request.length % maxp;
1401 unsigned int num_trbs = 1;
1402
1403 if (dwc3_needs_extra_trb(dep, req))
1404 num_trbs++;
1405
1406 if (dwc3_calc_trbs_left(dep) < num_trbs)
1407 return 0;
1408
1409 req->needs_extra_trb = num_trbs > 1;
1410
1411 /* Prepare a normal TRB */
1412 if (req->direction || req->request.length)
1413 dwc3_prepare_one_trb(dep, req, entry_length,
1414 req->needs_extra_trb, node, false, false);
1415
1416 /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */
1417 if ((!req->direction && !req->request.length) || req->needs_extra_trb)
1418 dwc3_prepare_one_trb(dep, req,
1419 req->direction ? 0 : maxp - rem,
1420 false, 1, true, false);
1421
1422 return num_trbs;
1423 }
1424
dwc3_prepare_trbs_sg(struct dwc3_ep * dep,struct dwc3_request * req)1425 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep,
1426 struct dwc3_request *req)
1427 {
1428 struct scatterlist *sg = req->start_sg;
1429 struct scatterlist *s;
1430 int i;
1431 unsigned int length = req->request.length;
1432 unsigned int remaining = req->request.num_mapped_sgs
1433 - req->num_queued_sgs;
1434 unsigned int num_trbs = req->num_trbs;
1435 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req);
1436
1437 /*
1438 * If we resume preparing the request, then get the remaining length of
1439 * the request and resume where we left off.
1440 */
1441 for_each_sg(req->request.sg, s, req->num_queued_sgs, i)
1442 length -= sg_dma_len(s);
1443
1444 for_each_sg(sg, s, remaining, i) {
1445 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep);
1446 unsigned int trb_length;
1447 bool must_interrupt = false;
1448 bool last_sg = false;
1449
1450 trb_length = min_t(unsigned int, length, sg_dma_len(s));
1451
1452 length -= trb_length;
1453
1454 /*
1455 * IOMMU driver is coalescing the list of sgs which shares a
1456 * page boundary into one and giving it to USB driver. With
1457 * this the number of sgs mapped is not equal to the number of
1458 * sgs passed. So mark the chain bit to false if it isthe last
1459 * mapped sg.
1460 */
1461 if ((i == remaining - 1) || !length)
1462 last_sg = true;
1463
1464 if (!num_trbs_left)
1465 break;
1466
1467 if (last_sg) {
1468 if (!dwc3_prepare_last_sg(dep, req, trb_length, i))
1469 break;
1470 } else {
1471 /*
1472 * Look ahead to check if we have enough TRBs for the
1473 * next SG entry. If not, set interrupt on this TRB to
1474 * resume preparing the next SG entry when more TRBs are
1475 * free.
1476 */
1477 if (num_trbs_left == 1 || (needs_extra_trb &&
1478 num_trbs_left <= 2 &&
1479 sg_dma_len(sg_next(s)) >= length)) {
1480 struct dwc3_request *r;
1481
1482 /* Check if previous requests already set IOC */
1483 list_for_each_entry(r, &dep->started_list, list) {
1484 if (r != req && !r->request.no_interrupt)
1485 break;
1486
1487 if (r == req)
1488 must_interrupt = true;
1489 }
1490 }
1491
1492 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false,
1493 must_interrupt);
1494 }
1495
1496 /*
1497 * There can be a situation where all sgs in sglist are not
1498 * queued because of insufficient trb number. To handle this
1499 * case, update start_sg to next sg to be queued, so that
1500 * we have free trbs we can continue queuing from where we
1501 * previously stopped
1502 */
1503 if (!last_sg)
1504 req->start_sg = sg_next(s);
1505
1506 req->num_queued_sgs++;
1507 req->num_pending_sgs--;
1508
1509 /*
1510 * The number of pending SG entries may not correspond to the
1511 * number of mapped SG entries. If all the data are queued, then
1512 * don't include unused SG entries.
1513 */
1514 if (length == 0) {
1515 req->num_pending_sgs = 0;
1516 break;
1517 }
1518
1519 if (must_interrupt)
1520 break;
1521 }
1522
1523 return req->num_trbs - num_trbs;
1524 }
1525
dwc3_prepare_trbs_linear(struct dwc3_ep * dep,struct dwc3_request * req)1526 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep,
1527 struct dwc3_request *req)
1528 {
1529 return dwc3_prepare_last_sg(dep, req, req->request.length, 0);
1530 }
1531
1532 /*
1533 * dwc3_prepare_trbs - setup TRBs from requests
1534 * @dep: endpoint for which requests are being prepared
1535 *
1536 * The function goes through the requests list and sets up TRBs for the
1537 * transfers. The function returns once there are no more TRBs available or
1538 * it runs out of requests.
1539 *
1540 * Returns the number of TRBs prepared or negative errno.
1541 */
dwc3_prepare_trbs(struct dwc3_ep * dep)1542 static int dwc3_prepare_trbs(struct dwc3_ep *dep)
1543 {
1544 struct dwc3_request *req, *n;
1545 int ret = 0;
1546
1547 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1548
1549 /*
1550 * We can get in a situation where there's a request in the started list
1551 * but there weren't enough TRBs to fully kick it in the first time
1552 * around, so it has been waiting for more TRBs to be freed up.
1553 *
1554 * In that case, we should check if we have a request with pending_sgs
1555 * in the started list and prepare TRBs for that request first,
1556 * otherwise we will prepare TRBs completely out of order and that will
1557 * break things.
1558 */
1559 list_for_each_entry(req, &dep->started_list, list) {
1560 if (req->num_pending_sgs > 0) {
1561 ret = dwc3_prepare_trbs_sg(dep, req);
1562 if (!ret || req->num_pending_sgs)
1563 return ret;
1564 }
1565
1566 if (!dwc3_calc_trbs_left(dep))
1567 return ret;
1568
1569 /*
1570 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1571 * burst capability may try to read and use TRBs beyond the
1572 * active transfer instead of stopping.
1573 */
1574 if (dep->stream_capable && req->request.is_last &&
1575 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1576 return ret;
1577 }
1578
1579 list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1580 struct dwc3 *dwc = dep->dwc;
1581
1582 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1583 dep->direction);
1584 if (ret)
1585 return ret;
1586
1587 req->sg = req->request.sg;
1588 req->start_sg = req->sg;
1589 req->num_queued_sgs = 0;
1590 req->num_pending_sgs = req->request.num_mapped_sgs;
1591
1592 if (req->num_pending_sgs > 0) {
1593 ret = dwc3_prepare_trbs_sg(dep, req);
1594 if (req->num_pending_sgs)
1595 return ret;
1596 } else {
1597 ret = dwc3_prepare_trbs_linear(dep, req);
1598 }
1599
1600 if (!ret || !dwc3_calc_trbs_left(dep))
1601 return ret;
1602
1603 /*
1604 * Don't prepare beyond a transfer. In DWC_usb32, its transfer
1605 * burst capability may try to read and use TRBs beyond the
1606 * active transfer instead of stopping.
1607 */
1608 if (dep->stream_capable && req->request.is_last &&
1609 !DWC3_MST_CAPABLE(&dwc->hwparams))
1610 return ret;
1611 }
1612
1613 return ret;
1614 }
1615
1616 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep);
1617
__dwc3_gadget_kick_transfer(struct dwc3_ep * dep)1618 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1619 {
1620 struct dwc3_gadget_ep_cmd_params params;
1621 struct dwc3_request *req;
1622 int starting;
1623 int ret;
1624 u32 cmd;
1625
1626 /*
1627 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0).
1628 * This happens when we need to stop and restart a transfer such as in
1629 * the case of reinitiating a stream or retrying an isoc transfer.
1630 */
1631 ret = dwc3_prepare_trbs(dep);
1632 if (ret < 0)
1633 return ret;
1634
1635 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1636
1637 /*
1638 * If there's no new TRB prepared and we don't need to restart a
1639 * transfer, there's no need to update the transfer.
1640 */
1641 if (!ret && !starting)
1642 return ret;
1643
1644 req = next_request(&dep->started_list);
1645 if (!req) {
1646 dep->flags |= DWC3_EP_PENDING_REQUEST;
1647 return 0;
1648 }
1649
1650 memset(¶ms, 0, sizeof(params));
1651
1652 if (starting) {
1653 params.param0 = upper_32_bits(req->trb_dma);
1654 params.param1 = lower_32_bits(req->trb_dma);
1655 cmd = DWC3_DEPCMD_STARTTRANSFER;
1656
1657 if (dep->stream_capable)
1658 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);
1659
1660 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1661 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1662 } else {
1663 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1664 DWC3_DEPCMD_PARAM(dep->resource_index);
1665 }
1666
1667 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1668 if (ret < 0) {
1669 struct dwc3_request *tmp;
1670
1671 if (ret == -EAGAIN)
1672 return ret;
1673
1674 dwc3_stop_active_transfer(dep, true, true);
1675
1676 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
1677 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED);
1678
1679 /* If ep isn't started, then there's no end transfer pending */
1680 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1681 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
1682
1683 return ret;
1684 }
1685
1686 if (dep->stream_capable && req->request.is_last &&
1687 !DWC3_MST_CAPABLE(&dep->dwc->hwparams))
1688 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;
1689
1690 return 0;
1691 }
1692
__dwc3_gadget_get_frame(struct dwc3 * dwc)1693 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1694 {
1695 u32 reg;
1696
1697 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1698 return DWC3_DSTS_SOFFN(reg);
1699 }
1700
1701 /**
1702 * __dwc3_stop_active_transfer - stop the current active transfer
1703 * @dep: isoc endpoint
1704 * @force: set forcerm bit in the command
1705 * @interrupt: command complete interrupt after End Transfer command
1706 *
1707 * When setting force, the ForceRM bit will be set. In that case
1708 * the controller won't update the TRB progress on command
1709 * completion. It also won't clear the HWO bit in the TRB.
1710 * The command will also not complete immediately in that case.
1711 */
__dwc3_stop_active_transfer(struct dwc3_ep * dep,bool force,bool interrupt)1712 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt)
1713 {
1714 struct dwc3_gadget_ep_cmd_params params;
1715 u32 cmd;
1716 int ret;
1717
1718 cmd = DWC3_DEPCMD_ENDTRANSFER;
1719 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1720 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0;
1721 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1722 memset(¶ms, 0, sizeof(params));
1723 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1724 /*
1725 * If the End Transfer command was timed out while the device is
1726 * not in SETUP phase, it's possible that an incoming Setup packet
1727 * may prevent the command's completion. Let's retry when the
1728 * ep0state returns to EP0_SETUP_PHASE.
1729 */
1730 if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {
1731 dep->flags |= DWC3_EP_DELAY_STOP;
1732 return 0;
1733 }
1734 WARN_ON_ONCE(ret);
1735 dep->resource_index = 0;
1736
1737 if (!interrupt) {
1738 mdelay(1);
1739 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
1740 } else if (!ret) {
1741 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
1742 }
1743
1744 dep->flags &= ~DWC3_EP_DELAY_STOP;
1745 return ret;
1746 }
1747
1748 /**
1749 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1750 * @dep: isoc endpoint
1751 *
1752 * This function tests for the correct combination of BIT[15:14] from the 16-bit
1753 * microframe number reported by the XferNotReady event for the future frame
1754 * number to start the isoc transfer.
1755 *
1756 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1757 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1758 * XferNotReady event are invalid. The driver uses this number to schedule the
1759 * isochronous transfer and passes it to the START TRANSFER command. Because
1760 * this number is invalid, the command may fail. If BIT[15:14] matches the
1761 * internal 16-bit microframe, the START TRANSFER command will pass and the
1762 * transfer will start at the scheduled time, if it is off by 1, the command
1763 * will still pass, but the transfer will start 2 seconds in the future. For all
1764 * other conditions, the START TRANSFER command will fail with bus-expiry.
1765 *
1766 * In order to workaround this issue, we can test for the correct combination of
1767 * BIT[15:14] by sending START TRANSFER commands with different values of
1768 * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart
1769 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1770 * As the result, within the 4 possible combinations for BIT[15:14], there will
1771 * be 2 successful and 2 failure START COMMAND status. One of the 2 successful
1772 * command status will result in a 2-second delay start. The smaller BIT[15:14]
1773 * value is the correct combination.
1774 *
1775 * Since there are only 4 outcomes and the results are ordered, we can simply
1776 * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to
1777 * deduce the smaller successful combination.
1778 *
1779 * Let test0 = test status for combination 'b00 and test1 = test status for 'b01
1780 * of BIT[15:14]. The correct combination is as follow:
1781 *
1782 * if test0 fails and test1 passes, BIT[15:14] is 'b01
1783 * if test0 fails and test1 fails, BIT[15:14] is 'b10
1784 * if test0 passes and test1 fails, BIT[15:14] is 'b11
1785 * if test0 passes and test1 passes, BIT[15:14] is 'b00
1786 *
1787 * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN
1788 * endpoints.
1789 */
dwc3_gadget_start_isoc_quirk(struct dwc3_ep * dep)1790 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep)
1791 {
1792 int cmd_status = 0;
1793 bool test0;
1794 bool test1;
1795
1796 while (dep->combo_num < 2) {
1797 struct dwc3_gadget_ep_cmd_params params;
1798 u32 test_frame_number;
1799 u32 cmd;
1800
1801 /*
1802 * Check if we can start isoc transfer on the next interval or
1803 * 4 uframes in the future with BIT[15:14] as dep->combo_num
1804 */
1805 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;
1806 test_frame_number |= dep->combo_num << 14;
1807 test_frame_number += max_t(u32, 4, dep->interval);
1808
1809 params.param0 = upper_32_bits(dep->dwc->bounce_addr);
1810 params.param1 = lower_32_bits(dep->dwc->bounce_addr);
1811
1812 cmd = DWC3_DEPCMD_STARTTRANSFER;
1813 cmd |= DWC3_DEPCMD_PARAM(test_frame_number);
1814 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms);
1815
1816 /* Redo if some other failure beside bus-expiry is received */
1817 if (cmd_status && cmd_status != -EAGAIN) {
1818 dep->start_cmd_status = 0;
1819 dep->combo_num = 0;
1820 return 0;
1821 }
1822
1823 /* Store the first test status */
1824 if (dep->combo_num == 0)
1825 dep->start_cmd_status = cmd_status;
1826
1827 dep->combo_num++;
1828
1829 /*
1830 * End the transfer if the START_TRANSFER command is successful
1831 * to wait for the next XferNotReady to test the command again
1832 */
1833 if (cmd_status == 0) {
1834 dwc3_stop_active_transfer(dep, true, true);
1835 return 0;
1836 }
1837 }
1838
1839 /* test0 and test1 are both completed at this point */
1840 test0 = (dep->start_cmd_status == 0);
1841 test1 = (cmd_status == 0);
1842
1843 if (!test0 && test1)
1844 dep->combo_num = 1;
1845 else if (!test0 && !test1)
1846 dep->combo_num = 2;
1847 else if (test0 && !test1)
1848 dep->combo_num = 3;
1849 else if (test0 && test1)
1850 dep->combo_num = 0;
1851
1852 dep->frame_number &= DWC3_FRNUMBER_MASK;
1853 dep->frame_number |= dep->combo_num << 14;
1854 dep->frame_number += max_t(u32, 4, dep->interval);
1855
1856 /* Reinitialize test variables */
1857 dep->start_cmd_status = 0;
1858 dep->combo_num = 0;
1859
1860 return __dwc3_gadget_kick_transfer(dep);
1861 }
1862
__dwc3_gadget_start_isoc(struct dwc3_ep * dep)1863 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1864 {
1865 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
1866 struct dwc3 *dwc = dep->dwc;
1867 int ret;
1868 int i;
1869
1870 if (list_empty(&dep->pending_list) &&
1871 list_empty(&dep->started_list)) {
1872 dep->flags |= DWC3_EP_PENDING_REQUEST;
1873 return -EAGAIN;
1874 }
1875
1876 if (!dwc->dis_start_transfer_quirk &&
1877 (DWC3_VER_IS_PRIOR(DWC31, 170A) ||
1878 DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) {
1879 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)
1880 return dwc3_gadget_start_isoc_quirk(dep);
1881 }
1882
1883 if (desc->bInterval <= 14 &&
1884 dwc->gadget->speed >= USB_SPEED_HIGH) {
1885 u32 frame = __dwc3_gadget_get_frame(dwc);
1886 bool rollover = frame <
1887 (dep->frame_number & DWC3_FRNUMBER_MASK);
1888
1889 /*
1890 * frame_number is set from XferNotReady and may be already
1891 * out of date. DSTS only provides the lower 14 bit of the
1892 * current frame number. So add the upper two bits of
1893 * frame_number and handle a possible rollover.
1894 * This will provide the correct frame_number unless more than
1895 * rollover has happened since XferNotReady.
1896 */
1897
1898 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |
1899 frame;
1900 if (rollover)
1901 dep->frame_number += BIT(14);
1902 }
1903
1904 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) {
1905 int future_interval = i + 1;
1906
1907 /* Give the controller at least 500us to schedule transfers */
1908 if (desc->bInterval < 3)
1909 future_interval += 3 - desc->bInterval;
1910
1911 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);
1912
1913 ret = __dwc3_gadget_kick_transfer(dep);
1914 if (ret != -EAGAIN)
1915 break;
1916 }
1917
1918 /*
1919 * After a number of unsuccessful start attempts due to bus-expiry
1920 * status, issue END_TRANSFER command and retry on the next XferNotReady
1921 * event.
1922 */
1923 if (ret == -EAGAIN)
1924 ret = __dwc3_stop_active_transfer(dep, false, true);
1925
1926 return ret;
1927 }
1928
__dwc3_gadget_ep_queue(struct dwc3_ep * dep,struct dwc3_request * req)1929 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1930 {
1931 struct dwc3 *dwc = dep->dwc;
1932
1933 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {
1934 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",
1935 dep->name);
1936 return -ESHUTDOWN;
1937 }
1938
1939 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1940 &req->request, req->dep->name))
1941 return -EINVAL;
1942
1943 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,
1944 "%s: request %pK already in flight\n",
1945 dep->name, &req->request))
1946 return -EINVAL;
1947
1948 pm_runtime_get(dwc->dev);
1949
1950 req->request.actual = 0;
1951 req->request.status = -EINPROGRESS;
1952
1953 trace_dwc3_ep_queue(req);
1954
1955 list_add_tail(&req->list, &dep->pending_list);
1956 req->status = DWC3_REQUEST_STATUS_QUEUED;
1957
1958 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)
1959 return 0;
1960
1961 /*
1962 * Start the transfer only after the END_TRANSFER is completed
1963 * and endpoint STALL is cleared.
1964 */
1965 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
1966 (dep->flags & DWC3_EP_WEDGE) ||
1967 (dep->flags & DWC3_EP_DELAY_STOP) ||
1968 (dep->flags & DWC3_EP_STALL)) {
1969 dep->flags |= DWC3_EP_DELAY_START;
1970 return 0;
1971 }
1972
1973 /*
1974 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1975 * wait for a XferNotReady event so we will know what's the current
1976 * (micro-)frame number.
1977 *
1978 * Without this trick, we are very, very likely gonna get Bus Expiry
1979 * errors which will force us issue EndTransfer command.
1980 */
1981 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1982 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1983 if ((dep->flags & DWC3_EP_PENDING_REQUEST))
1984 return __dwc3_gadget_start_isoc(dep);
1985
1986 return 0;
1987 }
1988 }
1989
1990 __dwc3_gadget_kick_transfer(dep);
1991
1992 return 0;
1993 }
1994
dwc3_gadget_ep_queue(struct usb_ep * ep,struct usb_request * request,gfp_t gfp_flags)1995 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1996 gfp_t gfp_flags)
1997 {
1998 struct dwc3_request *req = to_dwc3_request(request);
1999 struct dwc3_ep *dep = to_dwc3_ep(ep);
2000 struct dwc3 *dwc = dep->dwc;
2001
2002 unsigned long flags;
2003
2004 int ret;
2005
2006 spin_lock_irqsave(&dwc->lock, flags);
2007 ret = __dwc3_gadget_ep_queue(dep, req);
2008 spin_unlock_irqrestore(&dwc->lock, flags);
2009
2010 return ret;
2011 }
2012
dwc3_gadget_ep_skip_trbs(struct dwc3_ep * dep,struct dwc3_request * req)2013 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req)
2014 {
2015 int i;
2016
2017 /* If req->trb is not set, then the request has not started */
2018 if (!req->trb)
2019 return;
2020
2021 /*
2022 * If request was already started, this means we had to
2023 * stop the transfer. With that we also need to ignore
2024 * all TRBs used by the request, however TRBs can only
2025 * be modified after completion of END_TRANSFER
2026 * command. So what we do here is that we wait for
2027 * END_TRANSFER completion and only after that, we jump
2028 * over TRBs by clearing HWO and incrementing dequeue
2029 * pointer.
2030 */
2031 for (i = 0; i < req->num_trbs; i++) {
2032 struct dwc3_trb *trb;
2033
2034 trb = &dep->trb_pool[dep->trb_dequeue];
2035 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2036 dwc3_ep_inc_deq(dep);
2037 }
2038
2039 req->num_trbs = 0;
2040 }
2041
dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep * dep)2042 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep)
2043 {
2044 struct dwc3_request *req;
2045 struct dwc3 *dwc = dep->dwc;
2046
2047 while (!list_empty(&dep->cancelled_list)) {
2048 req = next_request(&dep->cancelled_list);
2049 dwc3_gadget_ep_skip_trbs(dep, req);
2050 switch (req->status) {
2051 case DWC3_REQUEST_STATUS_DISCONNECTED:
2052 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
2053 break;
2054 case DWC3_REQUEST_STATUS_DEQUEUED:
2055 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2056 break;
2057 case DWC3_REQUEST_STATUS_STALLED:
2058 dwc3_gadget_giveback(dep, req, -EPIPE);
2059 break;
2060 default:
2061 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);
2062 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2063 break;
2064 }
2065 /*
2066 * The endpoint is disabled, let the dwc3_remove_requests()
2067 * handle the cleanup.
2068 */
2069 if (!dep->endpoint.desc)
2070 break;
2071 }
2072 }
2073
dwc3_gadget_ep_dequeue(struct usb_ep * ep,struct usb_request * request)2074 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
2075 struct usb_request *request)
2076 {
2077 struct dwc3_request *req = to_dwc3_request(request);
2078 struct dwc3_request *r = NULL;
2079
2080 struct dwc3_ep *dep = to_dwc3_ep(ep);
2081 struct dwc3 *dwc = dep->dwc;
2082
2083 unsigned long flags;
2084 int ret = 0;
2085
2086 trace_dwc3_ep_dequeue(req);
2087
2088 spin_lock_irqsave(&dwc->lock, flags);
2089
2090 list_for_each_entry(r, &dep->cancelled_list, list) {
2091 if (r == req)
2092 goto out;
2093 }
2094
2095 list_for_each_entry(r, &dep->pending_list, list) {
2096 if (r == req) {
2097 /*
2098 * Explicitly check for EP0/1 as dequeue for those
2099 * EPs need to be handled differently. Control EP
2100 * only deals with one USB req, and giveback will
2101 * occur during dwc3_ep0_stall_and_restart(). EP0
2102 * requests are never added to started_list.
2103 */
2104 if (dep->number > 1)
2105 dwc3_gadget_giveback(dep, req, -ECONNRESET);
2106 else
2107 dwc3_ep0_reset_state(dwc);
2108 goto out;
2109 }
2110 }
2111
2112 list_for_each_entry(r, &dep->started_list, list) {
2113 if (r == req) {
2114 struct dwc3_request *t;
2115
2116 /* wait until it is processed */
2117 dwc3_stop_active_transfer(dep, true, true);
2118
2119 /*
2120 * Remove any started request if the transfer is
2121 * cancelled.
2122 */
2123 list_for_each_entry_safe(r, t, &dep->started_list, list)
2124 dwc3_gadget_move_cancelled_request(r,
2125 DWC3_REQUEST_STATUS_DEQUEUED);
2126
2127 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
2128
2129 goto out;
2130 }
2131 }
2132
2133 dev_err(dwc->dev, "request %pK was not queued to %s\n",
2134 request, ep->name);
2135 ret = -EINVAL;
2136 out:
2137 spin_unlock_irqrestore(&dwc->lock, flags);
2138
2139 return ret;
2140 }
2141
__dwc3_gadget_ep_set_halt(struct dwc3_ep * dep,int value,int protocol)2142 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
2143 {
2144 struct dwc3_gadget_ep_cmd_params params;
2145 struct dwc3 *dwc = dep->dwc;
2146 struct dwc3_request *req;
2147 struct dwc3_request *tmp;
2148 int ret;
2149
2150 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2151 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
2152 return -EINVAL;
2153 }
2154
2155 memset(¶ms, 0x00, sizeof(params));
2156
2157 if (value) {
2158 struct dwc3_trb *trb;
2159
2160 unsigned int transfer_in_flight;
2161 unsigned int started;
2162
2163 if (dep->number > 1)
2164 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
2165 else
2166 trb = &dwc->ep0_trb[dep->trb_enqueue];
2167
2168 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
2169 started = !list_empty(&dep->started_list);
2170
2171 if (!protocol && ((dep->direction && transfer_in_flight) ||
2172 (!dep->direction && started))) {
2173 return -EAGAIN;
2174 }
2175
2176 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
2177 ¶ms);
2178 if (ret)
2179 dev_err(dwc->dev, "failed to set STALL on %s\n",
2180 dep->name);
2181 else
2182 dep->flags |= DWC3_EP_STALL;
2183 } else {
2184 /*
2185 * Don't issue CLEAR_STALL command to control endpoints. The
2186 * controller automatically clears the STALL when it receives
2187 * the SETUP token.
2188 */
2189 if (dep->number <= 1) {
2190 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2191 return 0;
2192 }
2193
2194 dwc3_stop_active_transfer(dep, true, true);
2195
2196 list_for_each_entry_safe(req, tmp, &dep->started_list, list)
2197 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED);
2198
2199 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||
2200 (dep->flags & DWC3_EP_DELAY_STOP)) {
2201 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;
2202 if (protocol)
2203 dwc->clear_stall_protocol = dep->number;
2204
2205 return 0;
2206 }
2207
2208 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
2209
2210 ret = dwc3_send_clear_stall_ep_cmd(dep);
2211 if (ret) {
2212 dev_err(dwc->dev, "failed to clear STALL on %s\n",
2213 dep->name);
2214 return ret;
2215 }
2216
2217 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
2218
2219 if ((dep->flags & DWC3_EP_DELAY_START) &&
2220 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
2221 __dwc3_gadget_kick_transfer(dep);
2222
2223 dep->flags &= ~DWC3_EP_DELAY_START;
2224 }
2225
2226 return ret;
2227 }
2228
dwc3_gadget_ep_set_halt(struct usb_ep * ep,int value)2229 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2230 {
2231 struct dwc3_ep *dep = to_dwc3_ep(ep);
2232 struct dwc3 *dwc = dep->dwc;
2233
2234 unsigned long flags;
2235
2236 int ret;
2237
2238 spin_lock_irqsave(&dwc->lock, flags);
2239 ret = __dwc3_gadget_ep_set_halt(dep, value, false);
2240 spin_unlock_irqrestore(&dwc->lock, flags);
2241
2242 return ret;
2243 }
2244
dwc3_gadget_ep_set_wedge(struct usb_ep * ep)2245 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
2246 {
2247 struct dwc3_ep *dep = to_dwc3_ep(ep);
2248 struct dwc3 *dwc = dep->dwc;
2249 unsigned long flags;
2250 int ret;
2251
2252 spin_lock_irqsave(&dwc->lock, flags);
2253 dep->flags |= DWC3_EP_WEDGE;
2254
2255 if (dep->number == 0 || dep->number == 1)
2256 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
2257 else
2258 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
2259 spin_unlock_irqrestore(&dwc->lock, flags);
2260
2261 return ret;
2262 }
2263
2264 /* -------------------------------------------------------------------------- */
2265
2266 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
2267 .bLength = USB_DT_ENDPOINT_SIZE,
2268 .bDescriptorType = USB_DT_ENDPOINT,
2269 .bmAttributes = USB_ENDPOINT_XFER_CONTROL,
2270 };
2271
2272 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
2273 .enable = dwc3_gadget_ep0_enable,
2274 .disable = dwc3_gadget_ep0_disable,
2275 .alloc_request = dwc3_gadget_ep_alloc_request,
2276 .free_request = dwc3_gadget_ep_free_request,
2277 .queue = dwc3_gadget_ep0_queue,
2278 .dequeue = dwc3_gadget_ep_dequeue,
2279 .set_halt = dwc3_gadget_ep0_set_halt,
2280 .set_wedge = dwc3_gadget_ep_set_wedge,
2281 };
2282
2283 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
2284 .enable = dwc3_gadget_ep_enable,
2285 .disable = dwc3_gadget_ep_disable,
2286 .alloc_request = dwc3_gadget_ep_alloc_request,
2287 .free_request = dwc3_gadget_ep_free_request,
2288 .queue = dwc3_gadget_ep_queue,
2289 .dequeue = dwc3_gadget_ep_dequeue,
2290 .set_halt = dwc3_gadget_ep_set_halt,
2291 .set_wedge = dwc3_gadget_ep_set_wedge,
2292 };
2293
2294 /* -------------------------------------------------------------------------- */
2295
dwc3_gadget_enable_linksts_evts(struct dwc3 * dwc,bool set)2296 static void dwc3_gadget_enable_linksts_evts(struct dwc3 *dwc, bool set)
2297 {
2298 u32 reg;
2299
2300 if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2301 return;
2302
2303 reg = dwc3_readl(dwc->regs, DWC3_DEVTEN);
2304 if (set)
2305 reg |= DWC3_DEVTEN_ULSTCNGEN;
2306 else
2307 reg &= ~DWC3_DEVTEN_ULSTCNGEN;
2308
2309 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2310 }
2311
dwc3_gadget_get_frame(struct usb_gadget * g)2312 static int dwc3_gadget_get_frame(struct usb_gadget *g)
2313 {
2314 struct dwc3 *dwc = gadget_to_dwc(g);
2315
2316 return __dwc3_gadget_get_frame(dwc);
2317 }
2318
__dwc3_gadget_wakeup(struct dwc3 * dwc,bool async)2319 static int __dwc3_gadget_wakeup(struct dwc3 *dwc, bool async)
2320 {
2321 int retries;
2322
2323 int ret;
2324 u32 reg;
2325
2326 u8 link_state;
2327
2328 /*
2329 * According to the Databook Remote wakeup request should
2330 * be issued only when the device is in early suspend state.
2331 *
2332 * We can check that via USB Link State bits in DSTS register.
2333 */
2334 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2335
2336 link_state = DWC3_DSTS_USBLNKST(reg);
2337
2338 switch (link_state) {
2339 case DWC3_LINK_STATE_RESET:
2340 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */
2341 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */
2342 case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */
2343 case DWC3_LINK_STATE_U1:
2344 case DWC3_LINK_STATE_RESUME:
2345 break;
2346 default:
2347 return -EINVAL;
2348 }
2349
2350 if (async)
2351 dwc3_gadget_enable_linksts_evts(dwc, true);
2352
2353 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
2354 if (ret < 0) {
2355 dev_err(dwc->dev, "failed to put link in Recovery\n");
2356 dwc3_gadget_enable_linksts_evts(dwc, false);
2357 return ret;
2358 }
2359
2360 /* Recent versions do this automatically */
2361 if (DWC3_VER_IS_PRIOR(DWC3, 194A)) {
2362 /* write zeroes to Link Change Request */
2363 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2364 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
2365 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2366 }
2367
2368 /*
2369 * Since link status change events are enabled we will receive
2370 * an U0 event when wakeup is successful. So bail out.
2371 */
2372 if (async)
2373 return 0;
2374
2375 /* poll until Link State changes to ON */
2376 retries = 20000;
2377
2378 while (retries--) {
2379 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2380
2381 /* in HS, means ON */
2382 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
2383 break;
2384 }
2385
2386 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
2387 dev_err(dwc->dev, "failed to send remote wakeup\n");
2388 return -EINVAL;
2389 }
2390
2391 return 0;
2392 }
2393
dwc3_gadget_wakeup(struct usb_gadget * g)2394 static int dwc3_gadget_wakeup(struct usb_gadget *g)
2395 {
2396 struct dwc3 *dwc = gadget_to_dwc(g);
2397 unsigned long flags;
2398 int ret;
2399
2400 if (!dwc->wakeup_configured) {
2401 dev_err(dwc->dev, "remote wakeup not configured\n");
2402 return -EINVAL;
2403 }
2404
2405 spin_lock_irqsave(&dwc->lock, flags);
2406 if (!dwc->gadget->wakeup_armed) {
2407 dev_err(dwc->dev, "not armed for remote wakeup\n");
2408 spin_unlock_irqrestore(&dwc->lock, flags);
2409 return -EINVAL;
2410 }
2411 ret = __dwc3_gadget_wakeup(dwc, true);
2412
2413 spin_unlock_irqrestore(&dwc->lock, flags);
2414
2415 return ret;
2416 }
2417
2418 static void dwc3_resume_gadget(struct dwc3 *dwc);
2419
dwc3_gadget_func_wakeup(struct usb_gadget * g,int intf_id)2420 static int dwc3_gadget_func_wakeup(struct usb_gadget *g, int intf_id)
2421 {
2422 struct dwc3 *dwc = gadget_to_dwc(g);
2423 unsigned long flags;
2424 int ret;
2425 int link_state;
2426
2427 if (!dwc->wakeup_configured) {
2428 dev_err(dwc->dev, "remote wakeup not configured\n");
2429 return -EINVAL;
2430 }
2431
2432 spin_lock_irqsave(&dwc->lock, flags);
2433 /*
2434 * If the link is in U3, signal for remote wakeup and wait for the
2435 * link to transition to U0 before sending device notification.
2436 */
2437 link_state = dwc3_gadget_get_link_state(dwc);
2438 if (link_state == DWC3_LINK_STATE_U3) {
2439 ret = __dwc3_gadget_wakeup(dwc, false);
2440 if (ret) {
2441 spin_unlock_irqrestore(&dwc->lock, flags);
2442 return -EINVAL;
2443 }
2444 dwc3_resume_gadget(dwc);
2445 dwc->suspended = false;
2446 dwc->link_state = DWC3_LINK_STATE_U0;
2447 }
2448
2449 ret = dwc3_send_gadget_generic_command(dwc, DWC3_DGCMD_DEV_NOTIFICATION,
2450 DWC3_DGCMDPAR_DN_FUNC_WAKE |
2451 DWC3_DGCMDPAR_INTF_SEL(intf_id));
2452 if (ret)
2453 dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);
2454
2455 spin_unlock_irqrestore(&dwc->lock, flags);
2456
2457 return ret;
2458 }
2459
dwc3_gadget_set_remote_wakeup(struct usb_gadget * g,int set)2460 static int dwc3_gadget_set_remote_wakeup(struct usb_gadget *g, int set)
2461 {
2462 struct dwc3 *dwc = gadget_to_dwc(g);
2463 unsigned long flags;
2464
2465 spin_lock_irqsave(&dwc->lock, flags);
2466 dwc->wakeup_configured = !!set;
2467 spin_unlock_irqrestore(&dwc->lock, flags);
2468
2469 return 0;
2470 }
2471
dwc3_gadget_set_selfpowered(struct usb_gadget * g,int is_selfpowered)2472 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
2473 int is_selfpowered)
2474 {
2475 struct dwc3 *dwc = gadget_to_dwc(g);
2476 unsigned long flags;
2477
2478 spin_lock_irqsave(&dwc->lock, flags);
2479 g->is_selfpowered = !!is_selfpowered;
2480 spin_unlock_irqrestore(&dwc->lock, flags);
2481
2482 return 0;
2483 }
2484
dwc3_stop_active_transfers(struct dwc3 * dwc)2485 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
2486 {
2487 u32 epnum;
2488
2489 for (epnum = 2; epnum < dwc->num_eps; epnum++) {
2490 struct dwc3_ep *dep;
2491
2492 dep = dwc->eps[epnum];
2493 if (!dep)
2494 continue;
2495
2496 dwc3_remove_requests(dwc, dep, -ESHUTDOWN);
2497 }
2498 }
2499
__dwc3_gadget_set_ssp_rate(struct dwc3 * dwc)2500 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc)
2501 {
2502 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate;
2503 u32 reg;
2504
2505 if (ssp_rate == USB_SSP_GEN_UNKNOWN)
2506 ssp_rate = dwc->max_ssp_rate;
2507
2508 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2509 reg &= ~DWC3_DCFG_SPEED_MASK;
2510 reg &= ~DWC3_DCFG_NUMLANES(~0);
2511
2512 if (ssp_rate == USB_SSP_GEN_1x2)
2513 reg |= DWC3_DCFG_SUPERSPEED;
2514 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)
2515 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2516
2517 if (ssp_rate != USB_SSP_GEN_2x1 &&
2518 dwc->max_ssp_rate != USB_SSP_GEN_2x1)
2519 reg |= DWC3_DCFG_NUMLANES(1);
2520
2521 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2522 }
2523
__dwc3_gadget_set_speed(struct dwc3 * dwc)2524 static void __dwc3_gadget_set_speed(struct dwc3 *dwc)
2525 {
2526 enum usb_device_speed speed;
2527 u32 reg;
2528
2529 speed = dwc->gadget_max_speed;
2530 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)
2531 speed = dwc->maximum_speed;
2532
2533 if (speed == USB_SPEED_SUPER_PLUS &&
2534 DWC3_IP_IS(DWC32)) {
2535 __dwc3_gadget_set_ssp_rate(dwc);
2536 return;
2537 }
2538
2539 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2540 reg &= ~(DWC3_DCFG_SPEED_MASK);
2541
2542 /*
2543 * WORKAROUND: DWC3 revision < 2.20a have an issue
2544 * which would cause metastability state on Run/Stop
2545 * bit if we try to force the IP to USB2-only mode.
2546 *
2547 * Because of that, we cannot configure the IP to any
2548 * speed other than the SuperSpeed
2549 *
2550 * Refers to:
2551 *
2552 * STAR#9000525659: Clock Domain Crossing on DCTL in
2553 * USB 2.0 Mode
2554 */
2555 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
2556 !dwc->dis_metastability_quirk) {
2557 reg |= DWC3_DCFG_SUPERSPEED;
2558 } else {
2559 switch (speed) {
2560 case USB_SPEED_FULL:
2561 reg |= DWC3_DCFG_FULLSPEED;
2562 break;
2563 case USB_SPEED_HIGH:
2564 reg |= DWC3_DCFG_HIGHSPEED;
2565 break;
2566 case USB_SPEED_SUPER:
2567 reg |= DWC3_DCFG_SUPERSPEED;
2568 break;
2569 case USB_SPEED_SUPER_PLUS:
2570 if (DWC3_IP_IS(DWC3))
2571 reg |= DWC3_DCFG_SUPERSPEED;
2572 else
2573 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2574 break;
2575 default:
2576 dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2577
2578 if (DWC3_IP_IS(DWC3))
2579 reg |= DWC3_DCFG_SUPERSPEED;
2580 else
2581 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2582 }
2583 }
2584
2585 if (DWC3_IP_IS(DWC32) &&
2586 speed > USB_SPEED_UNKNOWN &&
2587 speed < USB_SPEED_SUPER_PLUS)
2588 reg &= ~DWC3_DCFG_NUMLANES(~0);
2589
2590 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2591 }
2592
dwc3_gadget_run_stop(struct dwc3 * dwc,int is_on)2593 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on)
2594 {
2595 u32 reg;
2596 u32 timeout = 2000;
2597
2598 if (pm_runtime_suspended(dwc->dev))
2599 return 0;
2600
2601 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2602 if (is_on) {
2603 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) {
2604 reg &= ~DWC3_DCTL_TRGTULST_MASK;
2605 reg |= DWC3_DCTL_TRGTULST_RX_DET;
2606 }
2607
2608 if (!DWC3_VER_IS_PRIOR(DWC3, 194A))
2609 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2610 reg |= DWC3_DCTL_RUN_STOP;
2611
2612 __dwc3_gadget_set_speed(dwc);
2613 dwc->pullups_connected = true;
2614 } else {
2615 reg &= ~DWC3_DCTL_RUN_STOP;
2616
2617 dwc->pullups_connected = false;
2618 }
2619
2620 dwc3_gadget_dctl_write_safe(dwc, reg);
2621
2622 do {
2623 usleep_range(1000, 2000);
2624 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2625 reg &= DWC3_DSTS_DEVCTRLHLT;
2626 } while (--timeout && !(!is_on ^ !reg));
2627
2628 if (!timeout)
2629 return -ETIMEDOUT;
2630
2631 return 0;
2632 }
2633
2634 static void dwc3_gadget_disable_irq(struct dwc3 *dwc);
2635 static void __dwc3_gadget_stop(struct dwc3 *dwc);
2636 static int __dwc3_gadget_start(struct dwc3 *dwc);
2637
dwc3_gadget_soft_disconnect(struct dwc3 * dwc)2638 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc)
2639 {
2640 unsigned long flags;
2641 int ret;
2642
2643 spin_lock_irqsave(&dwc->lock, flags);
2644 if (!dwc->pullups_connected) {
2645 spin_unlock_irqrestore(&dwc->lock, flags);
2646 return 0;
2647 }
2648
2649 dwc->connected = false;
2650
2651 /*
2652 * Attempt to end pending SETUP status phase, and not wait for the
2653 * function to do so.
2654 */
2655 if (dwc->delayed_status)
2656 dwc3_ep0_send_delayed_status(dwc);
2657
2658 /*
2659 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a
2660 * Section 4.1.8 Table 4-7, it states that for a device-initiated
2661 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER
2662 * command for any active transfers" before clearing the RunStop
2663 * bit.
2664 */
2665 dwc3_stop_active_transfers(dwc);
2666 spin_unlock_irqrestore(&dwc->lock, flags);
2667
2668 /*
2669 * Per databook, when we want to stop the gadget, if a control transfer
2670 * is still in process, complete it and get the core into setup phase.
2671 * In case the host is unresponsive to a SETUP transaction, forcefully
2672 * stall the transfer, and move back to the SETUP phase, so that any
2673 * pending endxfers can be executed.
2674 */
2675 if (dwc->ep0state != EP0_SETUP_PHASE) {
2676 reinit_completion(&dwc->ep0_in_setup);
2677
2678 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
2679 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
2680 if (ret == 0) {
2681 dev_warn(dwc->dev, "wait for SETUP phase timed out\n");
2682 spin_lock_irqsave(&dwc->lock, flags);
2683 dwc3_ep0_reset_state(dwc);
2684 spin_unlock_irqrestore(&dwc->lock, flags);
2685 }
2686 }
2687
2688 /*
2689 * Note: if the GEVNTCOUNT indicates events in the event buffer, the
2690 * driver needs to acknowledge them before the controller can halt.
2691 * Simply let the interrupt handler acknowledges and handle the
2692 * remaining event generated by the controller while polling for
2693 * DSTS.DEVCTLHLT.
2694 */
2695 ret = dwc3_gadget_run_stop(dwc, false);
2696
2697 /*
2698 * Stop the gadget after controller is halted, so that if needed, the
2699 * events to update EP0 state can still occur while the run/stop
2700 * routine polls for the halted state. DEVTEN is cleared as part of
2701 * gadget stop.
2702 */
2703 spin_lock_irqsave(&dwc->lock, flags);
2704 __dwc3_gadget_stop(dwc);
2705 spin_unlock_irqrestore(&dwc->lock, flags);
2706
2707 return ret;
2708 }
2709
dwc3_gadget_soft_connect(struct dwc3 * dwc)2710 static int dwc3_gadget_soft_connect(struct dwc3 *dwc)
2711 {
2712 int ret;
2713
2714 /*
2715 * In the Synopsys DWC_usb31 1.90a programming guide section
2716 * 4.1.9, it specifies that for a reconnect after a
2717 * device-initiated disconnect requires a core soft reset
2718 * (DCTL.CSftRst) before enabling the run/stop bit.
2719 */
2720 ret = dwc3_core_soft_reset(dwc);
2721 if (ret)
2722 return ret;
2723
2724 dwc3_event_buffers_setup(dwc);
2725 __dwc3_gadget_start(dwc);
2726 return dwc3_gadget_run_stop(dwc, true);
2727 }
2728
dwc3_gadget_pullup(struct usb_gadget * g,int is_on)2729 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
2730 {
2731 struct dwc3 *dwc = gadget_to_dwc(g);
2732 int ret;
2733
2734 is_on = !!is_on;
2735
2736 dwc->softconnect = is_on;
2737
2738 /*
2739 * Avoid issuing a runtime resume if the device is already in the
2740 * suspended state during gadget disconnect. DWC3 gadget was already
2741 * halted/stopped during runtime suspend.
2742 */
2743 if (!is_on) {
2744 pm_runtime_barrier(dwc->dev);
2745 if (pm_runtime_suspended(dwc->dev))
2746 return 0;
2747 }
2748
2749 /*
2750 * Check the return value for successful resume, or error. For a
2751 * successful resume, the DWC3 runtime PM resume routine will handle
2752 * the run stop sequence, so avoid duplicate operations here.
2753 */
2754 ret = pm_runtime_get_sync(dwc->dev);
2755 if (!ret || ret < 0) {
2756 pm_runtime_put(dwc->dev);
2757 if (ret < 0)
2758 pm_runtime_set_suspended(dwc->dev);
2759 return ret;
2760 }
2761
2762 if (dwc->pullups_connected == is_on) {
2763 pm_runtime_put(dwc->dev);
2764 return 0;
2765 }
2766
2767 synchronize_irq(dwc->irq_gadget);
2768
2769 if (!is_on)
2770 ret = dwc3_gadget_soft_disconnect(dwc);
2771 else
2772 ret = dwc3_gadget_soft_connect(dwc);
2773
2774 pm_runtime_put(dwc->dev);
2775
2776 return ret;
2777 }
2778
dwc3_gadget_enable_irq(struct dwc3 * dwc)2779 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
2780 {
2781 u32 reg;
2782
2783 /* Enable all but Start and End of Frame IRQs */
2784 reg = (DWC3_DEVTEN_EVNTOVERFLOWEN |
2785 DWC3_DEVTEN_CMDCMPLTEN |
2786 DWC3_DEVTEN_ERRTICERREN |
2787 DWC3_DEVTEN_WKUPEVTEN |
2788 DWC3_DEVTEN_CONNECTDONEEN |
2789 DWC3_DEVTEN_USBRSTEN |
2790 DWC3_DEVTEN_DISCONNEVTEN);
2791
2792 if (DWC3_VER_IS_PRIOR(DWC3, 250A))
2793 reg |= DWC3_DEVTEN_ULSTCNGEN;
2794
2795 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */
2796 if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
2797 reg |= DWC3_DEVTEN_U3L2L1SUSPEN;
2798
2799 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
2800 }
2801
dwc3_gadget_disable_irq(struct dwc3 * dwc)2802 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
2803 {
2804 /* mask all interrupts */
2805 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
2806 }
2807
2808 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
2809 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
2810
2811 /**
2812 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2813 * @dwc: pointer to our context structure
2814 *
2815 * The following looks like complex but it's actually very simple. In order to
2816 * calculate the number of packets we can burst at once on OUT transfers, we're
2817 * gonna use RxFIFO size.
2818 *
2819 * To calculate RxFIFO size we need two numbers:
2820 * MDWIDTH = size, in bits, of the internal memory bus
2821 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
2822 *
2823 * Given these two numbers, the formula is simple:
2824 *
2825 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2826 *
2827 * 24 bytes is for 3x SETUP packets
2828 * 16 bytes is a clock domain crossing tolerance
2829 *
2830 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
2831 */
dwc3_gadget_setup_nump(struct dwc3 * dwc)2832 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
2833 {
2834 u32 ram2_depth;
2835 u32 mdwidth;
2836 u32 nump;
2837 u32 reg;
2838
2839 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
2840 mdwidth = dwc3_mdwidth(dwc);
2841
2842 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
2843 nump = min_t(u32, nump, 16);
2844
2845 /* update NumP */
2846 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2847 reg &= ~DWC3_DCFG_NUMP_MASK;
2848 reg |= nump << DWC3_DCFG_NUMP_SHIFT;
2849 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2850 }
2851
__dwc3_gadget_start(struct dwc3 * dwc)2852 static int __dwc3_gadget_start(struct dwc3 *dwc)
2853 {
2854 struct dwc3_ep *dep;
2855 int ret = 0;
2856 u32 reg;
2857
2858 /*
2859 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
2860 * the core supports IMOD, disable it.
2861 */
2862 if (dwc->imod_interval) {
2863 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
2864 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
2865 } else if (dwc3_has_imod(dwc)) {
2866 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
2867 }
2868
2869 /*
2870 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
2871 * field instead of letting dwc3 itself calculate that automatically.
2872 *
2873 * This way, we maximize the chances that we'll be able to get several
2874 * bursts of data without going through any sort of endpoint throttling.
2875 */
2876 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
2877 if (DWC3_IP_IS(DWC3))
2878 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
2879 else
2880 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
2881
2882 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
2883
2884 dwc3_gadget_setup_nump(dwc);
2885
2886 /*
2887 * Currently the controller handles single stream only. So, Ignore
2888 * Packet Pending bit for stream selection and don't search for another
2889 * stream if the host sends Data Packet with PP=0 (for OUT direction) or
2890 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves
2891 * the stream performance.
2892 */
2893 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2894 reg |= DWC3_DCFG_IGNSTRMPP;
2895 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2896
2897 /* Enable MST by default if the device is capable of MST */
2898 if (DWC3_MST_CAPABLE(&dwc->hwparams)) {
2899 reg = dwc3_readl(dwc->regs, DWC3_DCFG1);
2900 reg &= ~DWC3_DCFG1_DIS_MST_ENH;
2901 dwc3_writel(dwc->regs, DWC3_DCFG1, reg);
2902 }
2903
2904 /* Start with SuperSpeed Default */
2905 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2906
2907 dep = dwc->eps[0];
2908 dep->flags = 0;
2909 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2910 if (ret) {
2911 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2912 goto err0;
2913 }
2914
2915 dep = dwc->eps[1];
2916 dep->flags = 0;
2917 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
2918 if (ret) {
2919 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2920 goto err1;
2921 }
2922
2923 /* begin to receive SETUP packets */
2924 dwc->ep0state = EP0_SETUP_PHASE;
2925 dwc->ep0_bounced = false;
2926 dwc->link_state = DWC3_LINK_STATE_SS_DIS;
2927 dwc->delayed_status = false;
2928 dwc3_ep0_out_start(dwc);
2929
2930 dwc3_gadget_enable_irq(dwc);
2931 dwc3_enable_susphy(dwc, true);
2932
2933 return 0;
2934
2935 err1:
2936 __dwc3_gadget_ep_disable(dwc->eps[0]);
2937
2938 err0:
2939 return ret;
2940 }
2941
dwc3_gadget_start(struct usb_gadget * g,struct usb_gadget_driver * driver)2942 static int dwc3_gadget_start(struct usb_gadget *g,
2943 struct usb_gadget_driver *driver)
2944 {
2945 struct dwc3 *dwc = gadget_to_dwc(g);
2946 unsigned long flags;
2947 int ret;
2948 int irq;
2949
2950 irq = dwc->irq_gadget;
2951 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
2952 IRQF_SHARED, "dwc3", dwc->ev_buf);
2953 if (ret) {
2954 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
2955 irq, ret);
2956 return ret;
2957 }
2958
2959 spin_lock_irqsave(&dwc->lock, flags);
2960 dwc->gadget_driver = driver;
2961 spin_unlock_irqrestore(&dwc->lock, flags);
2962
2963 if (dwc->sys_wakeup)
2964 device_wakeup_enable(dwc->sysdev);
2965
2966 return 0;
2967 }
2968
__dwc3_gadget_stop(struct dwc3 * dwc)2969 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2970 {
2971 dwc3_gadget_disable_irq(dwc);
2972 __dwc3_gadget_ep_disable(dwc->eps[0]);
2973 __dwc3_gadget_ep_disable(dwc->eps[1]);
2974 }
2975
dwc3_gadget_stop(struct usb_gadget * g)2976 static int dwc3_gadget_stop(struct usb_gadget *g)
2977 {
2978 struct dwc3 *dwc = gadget_to_dwc(g);
2979 unsigned long flags;
2980
2981 if (dwc->sys_wakeup)
2982 device_wakeup_disable(dwc->sysdev);
2983
2984 spin_lock_irqsave(&dwc->lock, flags);
2985 dwc->gadget_driver = NULL;
2986 dwc->max_cfg_eps = 0;
2987 spin_unlock_irqrestore(&dwc->lock, flags);
2988
2989 free_irq(dwc->irq_gadget, dwc->ev_buf);
2990
2991 return 0;
2992 }
2993
dwc3_gadget_config_params(struct usb_gadget * g,struct usb_dcd_config_params * params)2994 static void dwc3_gadget_config_params(struct usb_gadget *g,
2995 struct usb_dcd_config_params *params)
2996 {
2997 struct dwc3 *dwc = gadget_to_dwc(g);
2998
2999 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;
3000 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;
3001
3002 /* Recommended BESL */
3003 if (!dwc->dis_enblslpm_quirk) {
3004 /*
3005 * If the recommended BESL baseline is 0 or if the BESL deep is
3006 * less than 2, Microsoft's Windows 10 host usb stack will issue
3007 * a usb reset immediately after it receives the extended BOS
3008 * descriptor and the enumeration will fail. To maintain
3009 * compatibility with the Windows' usb stack, let's set the
3010 * recommended BESL baseline to 1 and clamp the BESL deep to be
3011 * within 2 to 15.
3012 */
3013 params->besl_baseline = 1;
3014 if (dwc->is_utmi_l1_suspend)
3015 params->besl_deep =
3016 clamp_t(u8, dwc->hird_threshold, 2, 15);
3017 }
3018
3019 /* U1 Device exit Latency */
3020 if (dwc->dis_u1_entry_quirk)
3021 params->bU1devExitLat = 0;
3022 else
3023 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;
3024
3025 /* U2 Device exit Latency */
3026 if (dwc->dis_u2_entry_quirk)
3027 params->bU2DevExitLat = 0;
3028 else
3029 params->bU2DevExitLat =
3030 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT);
3031 }
3032
dwc3_gadget_set_speed(struct usb_gadget * g,enum usb_device_speed speed)3033 static void dwc3_gadget_set_speed(struct usb_gadget *g,
3034 enum usb_device_speed speed)
3035 {
3036 struct dwc3 *dwc = gadget_to_dwc(g);
3037 unsigned long flags;
3038
3039 spin_lock_irqsave(&dwc->lock, flags);
3040 dwc->gadget_max_speed = speed;
3041 spin_unlock_irqrestore(&dwc->lock, flags);
3042 }
3043
dwc3_gadget_set_ssp_rate(struct usb_gadget * g,enum usb_ssp_rate rate)3044 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g,
3045 enum usb_ssp_rate rate)
3046 {
3047 struct dwc3 *dwc = gadget_to_dwc(g);
3048 unsigned long flags;
3049
3050 spin_lock_irqsave(&dwc->lock, flags);
3051 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;
3052 dwc->gadget_ssp_rate = rate;
3053 spin_unlock_irqrestore(&dwc->lock, flags);
3054 }
3055
dwc3_gadget_vbus_draw(struct usb_gadget * g,unsigned int mA)3056 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA)
3057 {
3058 struct dwc3 *dwc = gadget_to_dwc(g);
3059 union power_supply_propval val = {0};
3060 int ret;
3061
3062 if (dwc->usb2_phy)
3063 return usb_phy_set_power(dwc->usb2_phy, mA);
3064
3065 if (!dwc->usb_psy)
3066 return -EOPNOTSUPP;
3067
3068 val.intval = 1000 * mA;
3069 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);
3070
3071 return ret;
3072 }
3073
3074 /**
3075 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3076 * @g: pointer to the USB gadget
3077 *
3078 * Used to record the maximum number of endpoints being used in a USB composite
3079 * device. (across all configurations) This is to be used in the calculation
3080 * of the TXFIFO sizes when resizing internal memory for individual endpoints.
3081 * It will help ensured that the resizing logic reserves enough space for at
3082 * least one max packet.
3083 */
dwc3_gadget_check_config(struct usb_gadget * g)3084 static int dwc3_gadget_check_config(struct usb_gadget *g)
3085 {
3086 struct dwc3 *dwc = gadget_to_dwc(g);
3087 struct usb_ep *ep;
3088 int fifo_size = 0;
3089 int ram1_depth;
3090 int ep_num = 0;
3091
3092 if (!dwc->do_fifo_resize)
3093 return 0;
3094
3095 list_for_each_entry(ep, &g->ep_list, ep_list) {
3096 /* Only interested in the IN endpoints */
3097 if (ep->claimed && (ep->address & USB_DIR_IN))
3098 ep_num++;
3099 }
3100
3101 if (ep_num <= dwc->max_cfg_eps)
3102 return 0;
3103
3104 /* Update the max number of eps in the composition */
3105 dwc->max_cfg_eps = ep_num;
3106
3107 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);
3108 /* Based on the equation, increment by one for every ep */
3109 fifo_size += dwc->max_cfg_eps;
3110
3111 /* Check if we can fit a single fifo per endpoint */
3112 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);
3113 if (fifo_size > ram1_depth)
3114 return -ENOMEM;
3115
3116 return 0;
3117 }
3118
dwc3_gadget_async_callbacks(struct usb_gadget * g,bool enable)3119 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable)
3120 {
3121 struct dwc3 *dwc = gadget_to_dwc(g);
3122 unsigned long flags;
3123
3124 spin_lock_irqsave(&dwc->lock, flags);
3125 dwc->async_callbacks = enable;
3126 spin_unlock_irqrestore(&dwc->lock, flags);
3127 }
3128
3129 static const struct usb_gadget_ops dwc3_gadget_ops = {
3130 .get_frame = dwc3_gadget_get_frame,
3131 .wakeup = dwc3_gadget_wakeup,
3132 .func_wakeup = dwc3_gadget_func_wakeup,
3133 .set_remote_wakeup = dwc3_gadget_set_remote_wakeup,
3134 .set_selfpowered = dwc3_gadget_set_selfpowered,
3135 .pullup = dwc3_gadget_pullup,
3136 .udc_start = dwc3_gadget_start,
3137 .udc_stop = dwc3_gadget_stop,
3138 .udc_set_speed = dwc3_gadget_set_speed,
3139 .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate,
3140 .get_config_params = dwc3_gadget_config_params,
3141 .vbus_draw = dwc3_gadget_vbus_draw,
3142 .check_config = dwc3_gadget_check_config,
3143 .udc_async_callbacks = dwc3_gadget_async_callbacks,
3144 };
3145
3146 /* -------------------------------------------------------------------------- */
3147
dwc3_gadget_init_control_endpoint(struct dwc3_ep * dep)3148 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
3149 {
3150 struct dwc3 *dwc = dep->dwc;
3151
3152 usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
3153 dep->endpoint.maxburst = 1;
3154 dep->endpoint.ops = &dwc3_gadget_ep0_ops;
3155 if (!dep->direction)
3156 dwc->gadget->ep0 = &dep->endpoint;
3157
3158 dep->endpoint.caps.type_control = true;
3159
3160 return 0;
3161 }
3162
dwc3_gadget_init_in_endpoint(struct dwc3_ep * dep)3163 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
3164 {
3165 struct dwc3 *dwc = dep->dwc;
3166 u32 mdwidth;
3167 int size;
3168 int maxpacket;
3169
3170 mdwidth = dwc3_mdwidth(dwc);
3171
3172 /* MDWIDTH is represented in bits, we need it in bytes */
3173 mdwidth /= 8;
3174
3175 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
3176 if (DWC3_IP_IS(DWC3))
3177 size = DWC3_GTXFIFOSIZ_TXFDEP(size);
3178 else
3179 size = DWC31_GTXFIFOSIZ_TXFDEP(size);
3180
3181 /*
3182 * maxpacket size is determined as part of the following, after assuming
3183 * a mult value of one maxpacket:
3184 * DWC3 revision 280A and prior:
3185 * fifo_size = mult * (max_packet / mdwidth) + 1;
3186 * maxpacket = mdwidth * (fifo_size - 1);
3187 *
3188 * DWC3 revision 290A and onwards:
3189 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1
3190 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;
3191 */
3192 if (DWC3_VER_IS_PRIOR(DWC3, 290A))
3193 maxpacket = mdwidth * (size - 1);
3194 else
3195 maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;
3196
3197 /* Functionally, space for one max packet is sufficient */
3198 size = min_t(int, maxpacket, 1024);
3199 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3200
3201 dep->endpoint.max_streams = 16;
3202 dep->endpoint.ops = &dwc3_gadget_ep_ops;
3203 list_add_tail(&dep->endpoint.ep_list,
3204 &dwc->gadget->ep_list);
3205 dep->endpoint.caps.type_iso = true;
3206 dep->endpoint.caps.type_bulk = true;
3207 dep->endpoint.caps.type_int = true;
3208
3209 return dwc3_alloc_trb_pool(dep);
3210 }
3211
dwc3_gadget_init_out_endpoint(struct dwc3_ep * dep)3212 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
3213 {
3214 struct dwc3 *dwc = dep->dwc;
3215 u32 mdwidth;
3216 int size;
3217
3218 mdwidth = dwc3_mdwidth(dwc);
3219
3220 /* MDWIDTH is represented in bits, convert to bytes */
3221 mdwidth /= 8;
3222
3223 /* All OUT endpoints share a single RxFIFO space */
3224 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));
3225 if (DWC3_IP_IS(DWC3))
3226 size = DWC3_GRXFIFOSIZ_RXFDEP(size);
3227 else
3228 size = DWC31_GRXFIFOSIZ_RXFDEP(size);
3229
3230 /* FIFO depth is in MDWDITH bytes */
3231 size *= mdwidth;
3232
3233 /*
3234 * To meet performance requirement, a minimum recommended RxFIFO size
3235 * is defined as follow:
3236 * RxFIFO size >= (3 x MaxPacketSize) +
3237 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin)
3238 *
3239 * Then calculate the max packet limit as below.
3240 */
3241 size -= (3 * 8) + 16;
3242 if (size < 0)
3243 size = 0;
3244 else
3245 size /= 3;
3246
3247 usb_ep_set_maxpacket_limit(&dep->endpoint, size);
3248 dep->endpoint.max_streams = 16;
3249 dep->endpoint.ops = &dwc3_gadget_ep_ops;
3250 list_add_tail(&dep->endpoint.ep_list,
3251 &dwc->gadget->ep_list);
3252 dep->endpoint.caps.type_iso = true;
3253 dep->endpoint.caps.type_bulk = true;
3254 dep->endpoint.caps.type_int = true;
3255
3256 return dwc3_alloc_trb_pool(dep);
3257 }
3258
dwc3_gadget_init_endpoint(struct dwc3 * dwc,u8 epnum)3259 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
3260 {
3261 struct dwc3_ep *dep;
3262 bool direction = epnum & 1;
3263 int ret;
3264 u8 num = epnum >> 1;
3265
3266 dep = kzalloc(sizeof(*dep), GFP_KERNEL);
3267 if (!dep)
3268 return -ENOMEM;
3269
3270 dep->dwc = dwc;
3271 dep->number = epnum;
3272 dep->direction = direction;
3273 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
3274 dwc->eps[epnum] = dep;
3275 dep->combo_num = 0;
3276 dep->start_cmd_status = 0;
3277
3278 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
3279 direction ? "in" : "out");
3280
3281 dep->endpoint.name = dep->name;
3282
3283 if (!(dep->number > 1)) {
3284 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
3285 dep->endpoint.comp_desc = NULL;
3286 }
3287
3288 if (num == 0)
3289 ret = dwc3_gadget_init_control_endpoint(dep);
3290 else if (direction)
3291 ret = dwc3_gadget_init_in_endpoint(dep);
3292 else
3293 ret = dwc3_gadget_init_out_endpoint(dep);
3294
3295 if (ret)
3296 return ret;
3297
3298 dep->endpoint.caps.dir_in = direction;
3299 dep->endpoint.caps.dir_out = !direction;
3300
3301 INIT_LIST_HEAD(&dep->pending_list);
3302 INIT_LIST_HEAD(&dep->started_list);
3303 INIT_LIST_HEAD(&dep->cancelled_list);
3304
3305 dwc3_debugfs_create_endpoint_dir(dep);
3306
3307 return 0;
3308 }
3309
dwc3_gadget_init_endpoints(struct dwc3 * dwc,u8 total)3310 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
3311 {
3312 u8 epnum;
3313
3314 INIT_LIST_HEAD(&dwc->gadget->ep_list);
3315
3316 for (epnum = 0; epnum < total; epnum++) {
3317 int ret;
3318
3319 ret = dwc3_gadget_init_endpoint(dwc, epnum);
3320 if (ret)
3321 return ret;
3322 }
3323
3324 return 0;
3325 }
3326
dwc3_gadget_free_endpoints(struct dwc3 * dwc)3327 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
3328 {
3329 struct dwc3_ep *dep;
3330 u8 epnum;
3331
3332 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3333 dep = dwc->eps[epnum];
3334 if (!dep)
3335 continue;
3336 /*
3337 * Physical endpoints 0 and 1 are special; they form the
3338 * bi-directional USB endpoint 0.
3339 *
3340 * For those two physical endpoints, we don't allocate a TRB
3341 * pool nor do we add them the endpoints list. Due to that, we
3342 * shouldn't do these two operations otherwise we would end up
3343 * with all sorts of bugs when removing dwc3.ko.
3344 */
3345 if (epnum != 0 && epnum != 1) {
3346 dwc3_free_trb_pool(dep);
3347 list_del(&dep->endpoint.ep_list);
3348 }
3349
3350 dwc3_debugfs_remove_endpoint_dir(dep);
3351 kfree(dep);
3352 }
3353 }
3354
3355 /* -------------------------------------------------------------------------- */
3356
dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep * dep,struct dwc3_request * req,struct dwc3_trb * trb,const struct dwc3_event_depevt * event,int status,int chain)3357 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
3358 struct dwc3_request *req, struct dwc3_trb *trb,
3359 const struct dwc3_event_depevt *event, int status, int chain)
3360 {
3361 unsigned int count;
3362
3363 dwc3_ep_inc_deq(dep);
3364
3365 trace_dwc3_complete_trb(dep, trb);
3366 req->num_trbs--;
3367
3368 /*
3369 * If we're in the middle of series of chained TRBs and we
3370 * receive a short transfer along the way, DWC3 will skip
3371 * through all TRBs including the last TRB in the chain (the
3372 * where CHN bit is zero. DWC3 will also avoid clearing HWO
3373 * bit and SW has to do it manually.
3374 *
3375 * We're going to do that here to avoid problems of HW trying
3376 * to use bogus TRBs for transfers.
3377 */
3378 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
3379 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3380
3381 /*
3382 * For isochronous transfers, the first TRB in a service interval must
3383 * have the Isoc-First type. Track and report its interval frame number.
3384 */
3385 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3386 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {
3387 unsigned int frame_number;
3388
3389 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);
3390 frame_number &= ~(dep->interval - 1);
3391 req->request.frame_number = frame_number;
3392 }
3393
3394 /*
3395 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If
3396 * this TRB points to the bounce buffer address, it's a MPS alignment
3397 * TRB. Don't add it to req->remaining calculation.
3398 */
3399 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&
3400 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {
3401 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
3402 return 1;
3403 }
3404
3405 count = trb->size & DWC3_TRB_SIZE_MASK;
3406 req->remaining += count;
3407
3408 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
3409 return 1;
3410
3411 if (event->status & DEPEVT_STATUS_SHORT && !chain)
3412 return 1;
3413
3414 if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&
3415 DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)
3416 return 1;
3417
3418 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||
3419 (trb->ctrl & DWC3_TRB_CTRL_LST))
3420 return 1;
3421
3422 return 0;
3423 }
3424
dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep * dep,struct dwc3_request * req,const struct dwc3_event_depevt * event,int status)3425 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
3426 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3427 int status)
3428 {
3429 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3430 struct scatterlist *sg = req->sg;
3431 struct scatterlist *s;
3432 unsigned int num_queued = req->num_queued_sgs;
3433 unsigned int i;
3434 int ret = 0;
3435
3436 for_each_sg(sg, s, num_queued, i) {
3437 trb = &dep->trb_pool[dep->trb_dequeue];
3438
3439 req->sg = sg_next(s);
3440 req->num_queued_sgs--;
3441
3442 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
3443 trb, event, status, true);
3444 if (ret)
3445 break;
3446 }
3447
3448 return ret;
3449 }
3450
dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep * dep,struct dwc3_request * req,const struct dwc3_event_depevt * event,int status)3451 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
3452 struct dwc3_request *req, const struct dwc3_event_depevt *event,
3453 int status)
3454 {
3455 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
3456
3457 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
3458 event, status, false);
3459 }
3460
dwc3_gadget_ep_request_completed(struct dwc3_request * req)3461 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
3462 {
3463 return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;
3464 }
3465
dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,struct dwc3_request * req,int status)3466 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
3467 const struct dwc3_event_depevt *event,
3468 struct dwc3_request *req, int status)
3469 {
3470 int request_status;
3471 int ret;
3472
3473 if (req->request.num_mapped_sgs)
3474 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
3475 status);
3476 else
3477 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3478 status);
3479
3480 req->request.actual = req->request.length - req->remaining;
3481
3482 if (!dwc3_gadget_ep_request_completed(req))
3483 goto out;
3484
3485 if (req->needs_extra_trb) {
3486 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
3487 status);
3488 req->needs_extra_trb = false;
3489 }
3490
3491 /*
3492 * The event status only reflects the status of the TRB with IOC set.
3493 * For the requests that don't set interrupt on completion, the driver
3494 * needs to check and return the status of the completed TRBs associated
3495 * with the request. Use the status of the last TRB of the request.
3496 */
3497 if (req->request.no_interrupt) {
3498 struct dwc3_trb *trb;
3499
3500 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);
3501 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {
3502 case DWC3_TRBSTS_MISSED_ISOC:
3503 /* Isoc endpoint only */
3504 request_status = -EXDEV;
3505 break;
3506 case DWC3_TRB_STS_XFER_IN_PROG:
3507 /* Applicable when End Transfer with ForceRM=0 */
3508 case DWC3_TRBSTS_SETUP_PENDING:
3509 /* Control endpoint only */
3510 case DWC3_TRBSTS_OK:
3511 default:
3512 request_status = 0;
3513 break;
3514 }
3515 } else {
3516 request_status = status;
3517 }
3518
3519 dwc3_gadget_giveback(dep, req, request_status);
3520
3521 out:
3522 return ret;
3523 }
3524
dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,int status)3525 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
3526 const struct dwc3_event_depevt *event, int status)
3527 {
3528 struct dwc3_request *req;
3529
3530 while (!list_empty(&dep->started_list)) {
3531 int ret;
3532
3533 req = next_request(&dep->started_list);
3534 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
3535 req, status);
3536 if (ret)
3537 break;
3538 /*
3539 * The endpoint is disabled, let the dwc3_remove_requests()
3540 * handle the cleanup.
3541 */
3542 if (!dep->endpoint.desc)
3543 break;
3544 }
3545 }
3546
dwc3_gadget_ep_should_continue(struct dwc3_ep * dep)3547 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep)
3548 {
3549 struct dwc3_request *req;
3550 struct dwc3 *dwc = dep->dwc;
3551
3552 if (!dep->endpoint.desc || !dwc->pullups_connected ||
3553 !dwc->connected)
3554 return false;
3555
3556 if (!list_empty(&dep->pending_list))
3557 return true;
3558
3559 /*
3560 * We only need to check the first entry of the started list. We can
3561 * assume the completed requests are removed from the started list.
3562 */
3563 req = next_request(&dep->started_list);
3564 if (!req)
3565 return false;
3566
3567 return !dwc3_gadget_ep_request_completed(req);
3568 }
3569
dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3570 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
3571 const struct dwc3_event_depevt *event)
3572 {
3573 dep->frame_number = event->parameters;
3574 }
3575
dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event,int status)3576 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep,
3577 const struct dwc3_event_depevt *event, int status)
3578 {
3579 struct dwc3 *dwc = dep->dwc;
3580 bool no_started_trb = true;
3581
3582 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
3583
3584 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3585 goto out;
3586
3587 if (!dep->endpoint.desc)
3588 return no_started_trb;
3589
3590 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
3591 list_empty(&dep->started_list) &&
3592 (list_empty(&dep->pending_list) || status == -EXDEV))
3593 dwc3_stop_active_transfer(dep, true, true);
3594 else if (dwc3_gadget_ep_should_continue(dep))
3595 if (__dwc3_gadget_kick_transfer(dep) == 0)
3596 no_started_trb = false;
3597
3598 out:
3599 /*
3600 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
3601 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
3602 */
3603 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
3604 u32 reg;
3605 int i;
3606
3607 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
3608 dep = dwc->eps[i];
3609
3610 if (!(dep->flags & DWC3_EP_ENABLED))
3611 continue;
3612
3613 if (!list_empty(&dep->started_list))
3614 return no_started_trb;
3615 }
3616
3617 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3618 reg |= dwc->u1u2;
3619 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3620
3621 dwc->u1u2 = 0;
3622 }
3623
3624 return no_started_trb;
3625 }
3626
dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3627 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
3628 const struct dwc3_event_depevt *event)
3629 {
3630 int status = 0;
3631
3632 if (!dep->endpoint.desc)
3633 return;
3634
3635 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
3636 dwc3_gadget_endpoint_frame_from_event(dep, event);
3637
3638 if (event->status & DEPEVT_STATUS_BUSERR)
3639 status = -ECONNRESET;
3640
3641 if (event->status & DEPEVT_STATUS_MISSED_ISOC)
3642 status = -EXDEV;
3643
3644 dwc3_gadget_endpoint_trbs_complete(dep, event, status);
3645 }
3646
dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3647 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep,
3648 const struct dwc3_event_depevt *event)
3649 {
3650 int status = 0;
3651
3652 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3653
3654 if (event->status & DEPEVT_STATUS_BUSERR)
3655 status = -ECONNRESET;
3656
3657 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status))
3658 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;
3659 }
3660
dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3661 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
3662 const struct dwc3_event_depevt *event)
3663 {
3664 dwc3_gadget_endpoint_frame_from_event(dep, event);
3665
3666 /*
3667 * The XferNotReady event is generated only once before the endpoint
3668 * starts. It will be generated again when END_TRANSFER command is
3669 * issued. For some controller versions, the XferNotReady event may be
3670 * generated while the END_TRANSFER command is still in process. Ignore
3671 * it and wait for the next XferNotReady event after the command is
3672 * completed.
3673 */
3674 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)
3675 return;
3676
3677 (void) __dwc3_gadget_start_isoc(dep);
3678 }
3679
dwc3_gadget_endpoint_command_complete(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3680 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep,
3681 const struct dwc3_event_depevt *event)
3682 {
3683 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
3684
3685 if (cmd != DWC3_DEPCMD_ENDTRANSFER)
3686 return;
3687
3688 /*
3689 * The END_TRANSFER command will cause the controller to generate a
3690 * NoStream Event, and it's not due to the host DP NoStream rejection.
3691 * Ignore the next NoStream event.
3692 */
3693 if (dep->stream_capable)
3694 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;
3695
3696 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
3697 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
3698 dwc3_gadget_ep_cleanup_cancelled_requests(dep);
3699
3700 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {
3701 struct dwc3 *dwc = dep->dwc;
3702
3703 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;
3704 if (dwc3_send_clear_stall_ep_cmd(dep)) {
3705 struct usb_ep *ep0 = &dwc->eps[0]->endpoint;
3706
3707 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);
3708 if (dwc->delayed_status)
3709 __dwc3_gadget_ep0_set_halt(ep0, 1);
3710 return;
3711 }
3712
3713 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
3714 if (dwc->clear_stall_protocol == dep->number)
3715 dwc3_ep0_send_delayed_status(dwc);
3716 }
3717
3718 if ((dep->flags & DWC3_EP_DELAY_START) &&
3719 !usb_endpoint_xfer_isoc(dep->endpoint.desc))
3720 __dwc3_gadget_kick_transfer(dep);
3721
3722 dep->flags &= ~DWC3_EP_DELAY_START;
3723 }
3724
dwc3_gadget_endpoint_stream_event(struct dwc3_ep * dep,const struct dwc3_event_depevt * event)3725 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep,
3726 const struct dwc3_event_depevt *event)
3727 {
3728 struct dwc3 *dwc = dep->dwc;
3729
3730 if (event->status == DEPEVT_STREAMEVT_FOUND) {
3731 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3732 goto out;
3733 }
3734
3735 /* Note: NoStream rejection event param value is 0 and not 0xFFFF */
3736 switch (event->parameters) {
3737 case DEPEVT_STREAM_PRIME:
3738 /*
3739 * If the host can properly transition the endpoint state from
3740 * idle to prime after a NoStream rejection, there's no need to
3741 * force restarting the endpoint to reinitiate the stream. To
3742 * simplify the check, assume the host follows the USB spec if
3743 * it primed the endpoint more than once.
3744 */
3745 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {
3746 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)
3747 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;
3748 else
3749 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;
3750 }
3751
3752 break;
3753 case DEPEVT_STREAM_NOSTREAM:
3754 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||
3755 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||
3756 (!DWC3_MST_CAPABLE(&dwc->hwparams) &&
3757 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))
3758 break;
3759
3760 /*
3761 * If the host rejects a stream due to no active stream, by the
3762 * USB and xHCI spec, the endpoint will be put back to idle
3763 * state. When the host is ready (buffer added/updated), it will
3764 * prime the endpoint to inform the usb device controller. This
3765 * triggers the device controller to issue ERDY to restart the
3766 * stream. However, some hosts don't follow this and keep the
3767 * endpoint in the idle state. No prime will come despite host
3768 * streams are updated, and the device controller will not be
3769 * triggered to generate ERDY to move the next stream data. To
3770 * workaround this and maintain compatibility with various
3771 * hosts, force to reinitiate the stream until the host is ready
3772 * instead of waiting for the host to prime the endpoint.
3773 */
3774 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) {
3775 unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME;
3776
3777 dwc3_send_gadget_generic_command(dwc, cmd, dep->number);
3778 } else {
3779 dep->flags |= DWC3_EP_DELAY_START;
3780 dwc3_stop_active_transfer(dep, true, true);
3781 return;
3782 }
3783 break;
3784 }
3785
3786 out:
3787 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;
3788 }
3789
dwc3_endpoint_interrupt(struct dwc3 * dwc,const struct dwc3_event_depevt * event)3790 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
3791 const struct dwc3_event_depevt *event)
3792 {
3793 struct dwc3_ep *dep;
3794 u8 epnum = event->endpoint_number;
3795
3796 dep = dwc->eps[epnum];
3797
3798 if (!(dep->flags & DWC3_EP_ENABLED)) {
3799 if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))
3800 return;
3801
3802 /* Handle only EPCMDCMPLT when EP disabled */
3803 if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&
3804 !(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))
3805 return;
3806 }
3807
3808 if (epnum == 0 || epnum == 1) {
3809 dwc3_ep0_interrupt(dwc, event);
3810 return;
3811 }
3812
3813 switch (event->endpoint_event) {
3814 case DWC3_DEPEVT_XFERINPROGRESS:
3815 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
3816 break;
3817 case DWC3_DEPEVT_XFERNOTREADY:
3818 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
3819 break;
3820 case DWC3_DEPEVT_EPCMDCMPLT:
3821 dwc3_gadget_endpoint_command_complete(dep, event);
3822 break;
3823 case DWC3_DEPEVT_XFERCOMPLETE:
3824 dwc3_gadget_endpoint_transfer_complete(dep, event);
3825 break;
3826 case DWC3_DEPEVT_STREAMEVT:
3827 dwc3_gadget_endpoint_stream_event(dep, event);
3828 break;
3829 case DWC3_DEPEVT_RXTXFIFOEVT:
3830 break;
3831 default:
3832 dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event);
3833 break;
3834 }
3835 }
3836
dwc3_disconnect_gadget(struct dwc3 * dwc)3837 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
3838 {
3839 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {
3840 spin_unlock(&dwc->lock);
3841 dwc->gadget_driver->disconnect(dwc->gadget);
3842 spin_lock(&dwc->lock);
3843 }
3844 }
3845
dwc3_suspend_gadget(struct dwc3 * dwc)3846 static void dwc3_suspend_gadget(struct dwc3 *dwc)
3847 {
3848 if (dwc->async_callbacks && dwc->gadget_driver->suspend) {
3849 spin_unlock(&dwc->lock);
3850 dwc->gadget_driver->suspend(dwc->gadget);
3851 spin_lock(&dwc->lock);
3852 }
3853 }
3854
dwc3_resume_gadget(struct dwc3 * dwc)3855 static void dwc3_resume_gadget(struct dwc3 *dwc)
3856 {
3857 if (dwc->async_callbacks && dwc->gadget_driver->resume) {
3858 spin_unlock(&dwc->lock);
3859 dwc->gadget_driver->resume(dwc->gadget);
3860 spin_lock(&dwc->lock);
3861 }
3862 }
3863
dwc3_reset_gadget(struct dwc3 * dwc)3864 static void dwc3_reset_gadget(struct dwc3 *dwc)
3865 {
3866 if (!dwc->gadget_driver)
3867 return;
3868
3869 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {
3870 spin_unlock(&dwc->lock);
3871 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);
3872 spin_lock(&dwc->lock);
3873 }
3874 }
3875
dwc3_stop_active_transfer(struct dwc3_ep * dep,bool force,bool interrupt)3876 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force,
3877 bool interrupt)
3878 {
3879 struct dwc3 *dwc = dep->dwc;
3880
3881 /*
3882 * Only issue End Transfer command to the control endpoint of a started
3883 * Data Phase. Typically we should only do so in error cases such as
3884 * invalid/unexpected direction as described in the control transfer
3885 * flow of the programming guide.
3886 */
3887 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)
3888 return;
3889
3890 if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))
3891 return;
3892
3893 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||
3894 (dep->flags & DWC3_EP_END_TRANSFER_PENDING))
3895 return;
3896
3897 /*
3898 * If a Setup packet is received but yet to DMA out, the controller will
3899 * not process the End Transfer command of any endpoint. Polling of its
3900 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a
3901 * timeout. Delay issuing the End Transfer command until the Setup TRB is
3902 * prepared.
3903 */
3904 if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {
3905 dep->flags |= DWC3_EP_DELAY_STOP;
3906 return;
3907 }
3908
3909 /*
3910 * NOTICE: We are violating what the Databook says about the
3911 * EndTransfer command. Ideally we would _always_ wait for the
3912 * EndTransfer Command Completion IRQ, but that's causing too
3913 * much trouble synchronizing between us and gadget driver.
3914 *
3915 * We have discussed this with the IP Provider and it was
3916 * suggested to giveback all requests here.
3917 *
3918 * Note also that a similar handling was tested by Synopsys
3919 * (thanks a lot Paul) and nothing bad has come out of it.
3920 * In short, what we're doing is issuing EndTransfer with
3921 * CMDIOC bit set and delay kicking transfer until the
3922 * EndTransfer command had completed.
3923 *
3924 * As of IP version 3.10a of the DWC_usb3 IP, the controller
3925 * supports a mode to work around the above limitation. The
3926 * software can poll the CMDACT bit in the DEPCMD register
3927 * after issuing a EndTransfer command. This mode is enabled
3928 * by writing GUCTL2[14]. This polling is already done in the
3929 * dwc3_send_gadget_ep_cmd() function so if the mode is
3930 * enabled, the EndTransfer command will have completed upon
3931 * returning from this function.
3932 *
3933 * This mode is NOT available on the DWC_usb31 IP. In this
3934 * case, if the IOC bit is not set, then delay by 1ms
3935 * after issuing the EndTransfer command. This allows for the
3936 * controller to handle the command completely before DWC3
3937 * remove requests attempts to unmap USB request buffers.
3938 */
3939
3940 __dwc3_stop_active_transfer(dep, force, interrupt);
3941 }
3942
dwc3_clear_stall_all_ep(struct dwc3 * dwc)3943 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
3944 {
3945 u32 epnum;
3946
3947 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
3948 struct dwc3_ep *dep;
3949 int ret;
3950
3951 dep = dwc->eps[epnum];
3952 if (!dep)
3953 continue;
3954
3955 if (!(dep->flags & DWC3_EP_STALL))
3956 continue;
3957
3958 dep->flags &= ~DWC3_EP_STALL;
3959
3960 ret = dwc3_send_clear_stall_ep_cmd(dep);
3961 WARN_ON_ONCE(ret);
3962 }
3963 }
3964
dwc3_gadget_disconnect_interrupt(struct dwc3 * dwc)3965 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
3966 {
3967 int reg;
3968
3969 dwc->suspended = false;
3970
3971 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET);
3972
3973 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3974 reg &= ~DWC3_DCTL_INITU1ENA;
3975 reg &= ~DWC3_DCTL_INITU2ENA;
3976 dwc3_gadget_dctl_write_safe(dwc, reg);
3977
3978 dwc->connected = false;
3979
3980 dwc3_disconnect_gadget(dwc);
3981
3982 dwc->gadget->speed = USB_SPEED_UNKNOWN;
3983 dwc->setup_packet_pending = false;
3984 dwc->gadget->wakeup_armed = false;
3985 dwc3_gadget_enable_linksts_evts(dwc, false);
3986 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);
3987
3988 dwc3_ep0_reset_state(dwc);
3989
3990 /*
3991 * Request PM idle to address condition where usage count is
3992 * already decremented to zero, but waiting for the disconnect
3993 * interrupt to set dwc->connected to FALSE.
3994 */
3995 pm_request_idle(dwc->dev);
3996 }
3997
dwc3_gadget_reset_interrupt(struct dwc3 * dwc)3998 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
3999 {
4000 u32 reg;
4001
4002 dwc->suspended = false;
4003
4004 /*
4005 * Ideally, dwc3_reset_gadget() would trigger the function
4006 * drivers to stop any active transfers through ep disable.
4007 * However, for functions which defer ep disable, such as mass
4008 * storage, we will need to rely on the call to stop active
4009 * transfers here, and avoid allowing of request queuing.
4010 */
4011 dwc->connected = false;
4012
4013 /*
4014 * WORKAROUND: DWC3 revisions <1.88a have an issue which
4015 * would cause a missing Disconnect Event if there's a
4016 * pending Setup Packet in the FIFO.
4017 *
4018 * There's no suggested workaround on the official Bug
4019 * report, which states that "unless the driver/application
4020 * is doing any special handling of a disconnect event,
4021 * there is no functional issue".
4022 *
4023 * Unfortunately, it turns out that we _do_ some special
4024 * handling of a disconnect event, namely complete all
4025 * pending transfers, notify gadget driver of the
4026 * disconnection, and so on.
4027 *
4028 * Our suggested workaround is to follow the Disconnect
4029 * Event steps here, instead, based on a setup_packet_pending
4030 * flag. Such flag gets set whenever we have a SETUP_PENDING
4031 * status for EP0 TRBs and gets cleared on XferComplete for the
4032 * same endpoint.
4033 *
4034 * Refers to:
4035 *
4036 * STAR#9000466709: RTL: Device : Disconnect event not
4037 * generated if setup packet pending in FIFO
4038 */
4039 if (DWC3_VER_IS_PRIOR(DWC3, 188A)) {
4040 if (dwc->setup_packet_pending)
4041 dwc3_gadget_disconnect_interrupt(dwc);
4042 }
4043
4044 dwc3_reset_gadget(dwc);
4045
4046 /*
4047 * From SNPS databook section 8.1.2, the EP0 should be in setup
4048 * phase. So ensure that EP0 is in setup phase by issuing a stall
4049 * and restart if EP0 is not in setup phase.
4050 */
4051 dwc3_ep0_reset_state(dwc);
4052
4053 /*
4054 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a
4055 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW
4056 * needs to ensure that it sends "a DEPENDXFER command for any active
4057 * transfers."
4058 */
4059 dwc3_stop_active_transfers(dwc);
4060 dwc->connected = true;
4061
4062 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4063 reg &= ~DWC3_DCTL_TSTCTRL_MASK;
4064 dwc3_gadget_dctl_write_safe(dwc, reg);
4065 dwc->test_mode = false;
4066 dwc->gadget->wakeup_armed = false;
4067 dwc3_gadget_enable_linksts_evts(dwc, false);
4068 dwc3_clear_stall_all_ep(dwc);
4069
4070 /* Reset device address to zero */
4071 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4072 reg &= ~(DWC3_DCFG_DEVADDR_MASK);
4073 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4074 }
4075
dwc3_gadget_conndone_interrupt(struct dwc3 * dwc)4076 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
4077 {
4078 struct dwc3_ep *dep;
4079 int ret;
4080 u32 reg;
4081 u8 lanes = 1;
4082 u8 speed;
4083
4084 if (!dwc->softconnect)
4085 return;
4086
4087 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
4088 speed = reg & DWC3_DSTS_CONNECTSPD;
4089 dwc->speed = speed;
4090
4091 if (DWC3_IP_IS(DWC32))
4092 lanes = DWC3_DSTS_CONNLANES(reg) + 1;
4093
4094 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4095
4096 /*
4097 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
4098 * each time on Connect Done.
4099 *
4100 * Currently we always use the reset value. If any platform
4101 * wants to set this to a different value, we need to add a
4102 * setting and update GCTL.RAMCLKSEL here.
4103 */
4104
4105 switch (speed) {
4106 case DWC3_DSTS_SUPERSPEED_PLUS:
4107 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4108 dwc->gadget->ep0->maxpacket = 512;
4109 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4110
4111 if (lanes > 1)
4112 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;
4113 else
4114 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;
4115 break;
4116 case DWC3_DSTS_SUPERSPEED:
4117 /*
4118 * WORKAROUND: DWC3 revisions <1.90a have an issue which
4119 * would cause a missing USB3 Reset event.
4120 *
4121 * In such situations, we should force a USB3 Reset
4122 * event by calling our dwc3_gadget_reset_interrupt()
4123 * routine.
4124 *
4125 * Refers to:
4126 *
4127 * STAR#9000483510: RTL: SS : USB3 reset event may
4128 * not be generated always when the link enters poll
4129 */
4130 if (DWC3_VER_IS_PRIOR(DWC3, 190A))
4131 dwc3_gadget_reset_interrupt(dwc);
4132
4133 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
4134 dwc->gadget->ep0->maxpacket = 512;
4135 dwc->gadget->speed = USB_SPEED_SUPER;
4136
4137 if (lanes > 1) {
4138 dwc->gadget->speed = USB_SPEED_SUPER_PLUS;
4139 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;
4140 }
4141 break;
4142 case DWC3_DSTS_HIGHSPEED:
4143 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4144 dwc->gadget->ep0->maxpacket = 64;
4145 dwc->gadget->speed = USB_SPEED_HIGH;
4146 break;
4147 case DWC3_DSTS_FULLSPEED:
4148 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
4149 dwc->gadget->ep0->maxpacket = 64;
4150 dwc->gadget->speed = USB_SPEED_FULL;
4151 break;
4152 }
4153
4154 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;
4155
4156 /* Enable USB2 LPM Capability */
4157
4158 if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) &&
4159 !dwc->usb2_gadget_lpm_disable &&
4160 (speed != DWC3_DSTS_SUPERSPEED) &&
4161 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
4162 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4163 reg |= DWC3_DCFG_LPM_CAP;
4164 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4165
4166 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4167 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
4168
4169 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |
4170 (dwc->is_utmi_l1_suspend << 4));
4171
4172 /*
4173 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
4174 * DCFG.LPMCap is set, core responses with an ACK and the
4175 * BESL value in the LPM token is less than or equal to LPM
4176 * NYET threshold.
4177 */
4178 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,
4179 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
4180
4181 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A))
4182 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);
4183
4184 dwc3_gadget_dctl_write_safe(dwc, reg);
4185 } else {
4186 if (dwc->usb2_gadget_lpm_disable) {
4187 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
4188 reg &= ~DWC3_DCFG_LPM_CAP;
4189 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
4190 }
4191
4192 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4193 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
4194 dwc3_gadget_dctl_write_safe(dwc, reg);
4195 }
4196
4197 dep = dwc->eps[0];
4198 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4199 if (ret) {
4200 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4201 return;
4202 }
4203
4204 dep = dwc->eps[1];
4205 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
4206 if (ret) {
4207 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
4208 return;
4209 }
4210
4211 /*
4212 * Configure PHY via GUSB3PIPECTLn if required.
4213 *
4214 * Update GTXFIFOSIZn
4215 *
4216 * In both cases reset values should be sufficient.
4217 */
4218 }
4219
dwc3_gadget_wakeup_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4220 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc, unsigned int evtinfo)
4221 {
4222 dwc->suspended = false;
4223
4224 /*
4225 * TODO take core out of low power mode when that's
4226 * implemented.
4227 */
4228
4229 if (dwc->async_callbacks && dwc->gadget_driver->resume) {
4230 spin_unlock(&dwc->lock);
4231 dwc->gadget_driver->resume(dwc->gadget);
4232 spin_lock(&dwc->lock);
4233 }
4234
4235 dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;
4236 }
4237
dwc3_gadget_linksts_change_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4238 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
4239 unsigned int evtinfo)
4240 {
4241 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4242 unsigned int pwropt;
4243
4244 /*
4245 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
4246 * Hibernation mode enabled which would show up when device detects
4247 * host-initiated U3 exit.
4248 *
4249 * In that case, device will generate a Link State Change Interrupt
4250 * from U3 to RESUME which is only necessary if Hibernation is
4251 * configured in.
4252 *
4253 * There are no functional changes due to such spurious event and we
4254 * just need to ignore it.
4255 *
4256 * Refers to:
4257 *
4258 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
4259 * operational mode
4260 */
4261 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
4262 if (DWC3_VER_IS_PRIOR(DWC3, 250A) &&
4263 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
4264 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
4265 (next == DWC3_LINK_STATE_RESUME)) {
4266 return;
4267 }
4268 }
4269
4270 /*
4271 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
4272 * on the link partner, the USB session might do multiple entry/exit
4273 * of low power states before a transfer takes place.
4274 *
4275 * Due to this problem, we might experience lower throughput. The
4276 * suggested workaround is to disable DCTL[12:9] bits if we're
4277 * transitioning from U1/U2 to U0 and enable those bits again
4278 * after a transfer completes and there are no pending transfers
4279 * on any of the enabled endpoints.
4280 *
4281 * This is the first half of that workaround.
4282 *
4283 * Refers to:
4284 *
4285 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
4286 * core send LGO_Ux entering U0
4287 */
4288 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) {
4289 if (next == DWC3_LINK_STATE_U0) {
4290 u32 u1u2;
4291 u32 reg;
4292
4293 switch (dwc->link_state) {
4294 case DWC3_LINK_STATE_U1:
4295 case DWC3_LINK_STATE_U2:
4296 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
4297 u1u2 = reg & (DWC3_DCTL_INITU2ENA
4298 | DWC3_DCTL_ACCEPTU2ENA
4299 | DWC3_DCTL_INITU1ENA
4300 | DWC3_DCTL_ACCEPTU1ENA);
4301
4302 if (!dwc->u1u2)
4303 dwc->u1u2 = reg & u1u2;
4304
4305 reg &= ~u1u2;
4306
4307 dwc3_gadget_dctl_write_safe(dwc, reg);
4308 break;
4309 default:
4310 /* do nothing */
4311 break;
4312 }
4313 }
4314 }
4315
4316 switch (next) {
4317 case DWC3_LINK_STATE_U0:
4318 if (dwc->gadget->wakeup_armed) {
4319 dwc3_gadget_enable_linksts_evts(dwc, false);
4320 dwc3_resume_gadget(dwc);
4321 dwc->suspended = false;
4322 }
4323 break;
4324 case DWC3_LINK_STATE_U1:
4325 if (dwc->speed == USB_SPEED_SUPER)
4326 dwc3_suspend_gadget(dwc);
4327 break;
4328 case DWC3_LINK_STATE_U2:
4329 case DWC3_LINK_STATE_U3:
4330 dwc3_suspend_gadget(dwc);
4331 break;
4332 case DWC3_LINK_STATE_RESUME:
4333 dwc3_resume_gadget(dwc);
4334 break;
4335 default:
4336 /* do nothing */
4337 break;
4338 }
4339
4340 dwc->link_state = next;
4341 }
4342
dwc3_gadget_suspend_interrupt(struct dwc3 * dwc,unsigned int evtinfo)4343 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
4344 unsigned int evtinfo)
4345 {
4346 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
4347
4348 if (!dwc->suspended && next == DWC3_LINK_STATE_U3) {
4349 dwc->suspended = true;
4350 dwc3_suspend_gadget(dwc);
4351 }
4352
4353 dwc->link_state = next;
4354 }
4355
dwc3_gadget_interrupt(struct dwc3 * dwc,const struct dwc3_event_devt * event)4356 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
4357 const struct dwc3_event_devt *event)
4358 {
4359 switch (event->type) {
4360 case DWC3_DEVICE_EVENT_DISCONNECT:
4361 dwc3_gadget_disconnect_interrupt(dwc);
4362 break;
4363 case DWC3_DEVICE_EVENT_RESET:
4364 dwc3_gadget_reset_interrupt(dwc);
4365 break;
4366 case DWC3_DEVICE_EVENT_CONNECT_DONE:
4367 dwc3_gadget_conndone_interrupt(dwc);
4368 break;
4369 case DWC3_DEVICE_EVENT_WAKEUP:
4370 dwc3_gadget_wakeup_interrupt(dwc, event->event_info);
4371 break;
4372 case DWC3_DEVICE_EVENT_HIBER_REQ:
4373 dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");
4374 break;
4375 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
4376 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
4377 break;
4378 case DWC3_DEVICE_EVENT_SUSPEND:
4379 /* It changed to be suspend event for version 2.30a and above */
4380 if (!DWC3_VER_IS_PRIOR(DWC3, 230A))
4381 dwc3_gadget_suspend_interrupt(dwc, event->event_info);
4382 break;
4383 case DWC3_DEVICE_EVENT_SOF:
4384 case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
4385 case DWC3_DEVICE_EVENT_CMD_CMPL:
4386 case DWC3_DEVICE_EVENT_OVERFLOW:
4387 break;
4388 default:
4389 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
4390 }
4391 }
4392
dwc3_process_event_entry(struct dwc3 * dwc,const union dwc3_event * event)4393 static void dwc3_process_event_entry(struct dwc3 *dwc,
4394 const union dwc3_event *event)
4395 {
4396 trace_dwc3_event(event->raw, dwc);
4397
4398 if (!event->type.is_devspec)
4399 dwc3_endpoint_interrupt(dwc, &event->depevt);
4400 else if (event->type.type == DWC3_EVENT_TYPE_DEV)
4401 dwc3_gadget_interrupt(dwc, &event->devt);
4402 else
4403 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
4404 }
4405
dwc3_process_event_buf(struct dwc3_event_buffer * evt)4406 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
4407 {
4408 struct dwc3 *dwc = evt->dwc;
4409 irqreturn_t ret = IRQ_NONE;
4410 int left;
4411
4412 left = evt->count;
4413
4414 if (!(evt->flags & DWC3_EVENT_PENDING))
4415 return IRQ_NONE;
4416
4417 while (left > 0) {
4418 union dwc3_event event;
4419
4420 event.raw = *(u32 *) (evt->cache + evt->lpos);
4421
4422 dwc3_process_event_entry(dwc, &event);
4423
4424 /*
4425 * FIXME we wrap around correctly to the next entry as
4426 * almost all entries are 4 bytes in size. There is one
4427 * entry which has 12 bytes which is a regular entry
4428 * followed by 8 bytes data. ATM I don't know how
4429 * things are organized if we get next to the a
4430 * boundary so I worry about that once we try to handle
4431 * that.
4432 */
4433 evt->lpos = (evt->lpos + 4) % evt->length;
4434 left -= 4;
4435 }
4436
4437 evt->count = 0;
4438 ret = IRQ_HANDLED;
4439
4440 /* Unmask interrupt */
4441 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4442 DWC3_GEVNTSIZ_SIZE(evt->length));
4443
4444 if (dwc->imod_interval) {
4445 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
4446 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
4447 }
4448
4449 /* Keep the clearing of DWC3_EVENT_PENDING at the end */
4450 evt->flags &= ~DWC3_EVENT_PENDING;
4451
4452 return ret;
4453 }
4454
dwc3_thread_interrupt(int irq,void * _evt)4455 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
4456 {
4457 struct dwc3_event_buffer *evt = _evt;
4458 struct dwc3 *dwc = evt->dwc;
4459 unsigned long flags;
4460 irqreturn_t ret = IRQ_NONE;
4461
4462 local_bh_disable();
4463 spin_lock_irqsave(&dwc->lock, flags);
4464 ret = dwc3_process_event_buf(evt);
4465 spin_unlock_irqrestore(&dwc->lock, flags);
4466 local_bh_enable();
4467
4468 return ret;
4469 }
4470
dwc3_check_event_buf(struct dwc3_event_buffer * evt)4471 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
4472 {
4473 struct dwc3 *dwc = evt->dwc;
4474 u32 amount;
4475 u32 count;
4476
4477 if (pm_runtime_suspended(dwc->dev)) {
4478 dwc->pending_events = true;
4479 /*
4480 * Trigger runtime resume. The get() function will be balanced
4481 * after processing the pending events in dwc3_process_pending
4482 * events().
4483 */
4484 pm_runtime_get(dwc->dev);
4485 disable_irq_nosync(dwc->irq_gadget);
4486 return IRQ_HANDLED;
4487 }
4488
4489 /*
4490 * With PCIe legacy interrupt, test shows that top-half irq handler can
4491 * be called again after HW interrupt deassertion. Check if bottom-half
4492 * irq event handler completes before caching new event to prevent
4493 * losing events.
4494 */
4495 if (evt->flags & DWC3_EVENT_PENDING)
4496 return IRQ_HANDLED;
4497
4498 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
4499 count &= DWC3_GEVNTCOUNT_MASK;
4500 if (!count)
4501 return IRQ_NONE;
4502
4503 evt->count = count;
4504 evt->flags |= DWC3_EVENT_PENDING;
4505
4506 /* Mask interrupt */
4507 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),
4508 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));
4509
4510 amount = min(count, evt->length - evt->lpos);
4511 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
4512
4513 if (amount < count)
4514 memcpy(evt->cache, evt->buf, count - amount);
4515
4516 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
4517
4518 return IRQ_WAKE_THREAD;
4519 }
4520
dwc3_interrupt(int irq,void * _evt)4521 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
4522 {
4523 struct dwc3_event_buffer *evt = _evt;
4524
4525 return dwc3_check_event_buf(evt);
4526 }
4527
dwc3_gadget_get_irq(struct dwc3 * dwc)4528 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
4529 {
4530 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
4531 int irq;
4532
4533 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral");
4534 if (irq > 0)
4535 goto out;
4536
4537 if (irq == -EPROBE_DEFER)
4538 goto out;
4539
4540 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3");
4541 if (irq > 0)
4542 goto out;
4543
4544 if (irq == -EPROBE_DEFER)
4545 goto out;
4546
4547 irq = platform_get_irq(dwc3_pdev, 0);
4548
4549 out:
4550 return irq;
4551 }
4552
dwc_gadget_release(struct device * dev)4553 static void dwc_gadget_release(struct device *dev)
4554 {
4555 struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev);
4556
4557 kfree(gadget);
4558 }
4559
4560 /**
4561 * dwc3_gadget_init - initializes gadget related registers
4562 * @dwc: pointer to our controller context structure
4563 *
4564 * Returns 0 on success otherwise negative errno.
4565 */
dwc3_gadget_init(struct dwc3 * dwc)4566 int dwc3_gadget_init(struct dwc3 *dwc)
4567 {
4568 int ret;
4569 int irq;
4570 struct device *dev;
4571
4572 irq = dwc3_gadget_get_irq(dwc);
4573 if (irq < 0) {
4574 ret = irq;
4575 goto err0;
4576 }
4577
4578 dwc->irq_gadget = irq;
4579
4580 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
4581 sizeof(*dwc->ep0_trb) * 2,
4582 &dwc->ep0_trb_addr, GFP_KERNEL);
4583 if (!dwc->ep0_trb) {
4584 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
4585 ret = -ENOMEM;
4586 goto err0;
4587 }
4588
4589 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
4590 if (!dwc->setup_buf) {
4591 ret = -ENOMEM;
4592 goto err1;
4593 }
4594
4595 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
4596 &dwc->bounce_addr, GFP_KERNEL);
4597 if (!dwc->bounce) {
4598 ret = -ENOMEM;
4599 goto err2;
4600 }
4601
4602 init_completion(&dwc->ep0_in_setup);
4603 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);
4604 if (!dwc->gadget) {
4605 ret = -ENOMEM;
4606 goto err3;
4607 }
4608
4609
4610 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);
4611 dev = &dwc->gadget->dev;
4612 dev->platform_data = dwc;
4613 dwc->gadget->ops = &dwc3_gadget_ops;
4614 dwc->gadget->speed = USB_SPEED_UNKNOWN;
4615 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;
4616 dwc->gadget->sg_supported = true;
4617 dwc->gadget->name = "dwc3-gadget";
4618 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable;
4619 dwc->gadget->wakeup_capable = true;
4620
4621 /*
4622 * FIXME We might be setting max_speed to <SUPER, however versions
4623 * <2.20a of dwc3 have an issue with metastability (documented
4624 * elsewhere in this driver) which tells us we can't set max speed to
4625 * anything lower than SUPER.
4626 *
4627 * Because gadget.max_speed is only used by composite.c and function
4628 * drivers (i.e. it won't go into dwc3's registers) we are allowing this
4629 * to happen so we avoid sending SuperSpeed Capability descriptor
4630 * together with our BOS descriptor as that could confuse host into
4631 * thinking we can handle super speed.
4632 *
4633 * Note that, in fact, we won't even support GetBOS requests when speed
4634 * is less than super speed because we don't have means, yet, to tell
4635 * composite.c that we are USB 2.0 + LPM ECN.
4636 */
4637 if (DWC3_VER_IS_PRIOR(DWC3, 220A) &&
4638 !dwc->dis_metastability_quirk)
4639 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
4640 dwc->revision);
4641
4642 dwc->gadget->max_speed = dwc->maximum_speed;
4643 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate;
4644
4645 /*
4646 * REVISIT: Here we should clear all pending IRQs to be
4647 * sure we're starting from a well known location.
4648 */
4649
4650 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
4651 if (ret)
4652 goto err4;
4653
4654 ret = usb_add_gadget(dwc->gadget);
4655 if (ret) {
4656 dev_err(dwc->dev, "failed to add gadget\n");
4657 goto err5;
4658 }
4659
4660 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)
4661 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);
4662 else
4663 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);
4664
4665 /* No system wakeup if no gadget driver bound */
4666 if (dwc->sys_wakeup)
4667 device_wakeup_disable(dwc->sysdev);
4668
4669 return 0;
4670
4671 err5:
4672 dwc3_gadget_free_endpoints(dwc);
4673 err4:
4674 usb_put_gadget(dwc->gadget);
4675 dwc->gadget = NULL;
4676 err3:
4677 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4678 dwc->bounce_addr);
4679
4680 err2:
4681 kfree(dwc->setup_buf);
4682
4683 err1:
4684 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4685 dwc->ep0_trb, dwc->ep0_trb_addr);
4686
4687 err0:
4688 return ret;
4689 }
4690
4691 /* -------------------------------------------------------------------------- */
4692
dwc3_gadget_exit(struct dwc3 * dwc)4693 void dwc3_gadget_exit(struct dwc3 *dwc)
4694 {
4695 if (!dwc->gadget)
4696 return;
4697
4698 dwc3_enable_susphy(dwc, false);
4699 usb_del_gadget(dwc->gadget);
4700 dwc3_gadget_free_endpoints(dwc);
4701 usb_put_gadget(dwc->gadget);
4702 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
4703 dwc->bounce_addr);
4704 kfree(dwc->setup_buf);
4705 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
4706 dwc->ep0_trb, dwc->ep0_trb_addr);
4707 }
4708
dwc3_gadget_suspend(struct dwc3 * dwc)4709 int dwc3_gadget_suspend(struct dwc3 *dwc)
4710 {
4711 unsigned long flags;
4712 int ret;
4713
4714 ret = dwc3_gadget_soft_disconnect(dwc);
4715 if (ret)
4716 goto err;
4717
4718 spin_lock_irqsave(&dwc->lock, flags);
4719 if (dwc->gadget_driver)
4720 dwc3_disconnect_gadget(dwc);
4721 spin_unlock_irqrestore(&dwc->lock, flags);
4722
4723 return 0;
4724
4725 err:
4726 /*
4727 * Attempt to reset the controller's state. Likely no
4728 * communication can be established until the host
4729 * performs a port reset.
4730 */
4731 if (dwc->softconnect)
4732 dwc3_gadget_soft_connect(dwc);
4733
4734 return ret;
4735 }
4736
dwc3_gadget_resume(struct dwc3 * dwc)4737 int dwc3_gadget_resume(struct dwc3 *dwc)
4738 {
4739 if (!dwc->gadget_driver || !dwc->softconnect)
4740 return 0;
4741
4742 return dwc3_gadget_soft_connect(dwc);
4743 }
4744
dwc3_gadget_process_pending_events(struct dwc3 * dwc)4745 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
4746 {
4747 if (dwc->pending_events) {
4748 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
4749 dwc3_thread_interrupt(dwc->irq_gadget, dwc->ev_buf);
4750 pm_runtime_put(dwc->dev);
4751 dwc->pending_events = false;
4752 enable_irq(dwc->irq_gadget);
4753 }
4754 }
4755