1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link 4 * 5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com 6 * 7 * Authors: Felipe Balbi <balbi@ti.com>, 8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de> 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/delay.h> 13 #include <linux/slab.h> 14 #include <linux/spinlock.h> 15 #include <linux/platform_device.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/list.h> 20 #include <linux/dma-mapping.h> 21 22 #include <linux/usb/ch9.h> 23 #include <linux/usb/gadget.h> 24 25 #include "debug.h" 26 #include "core.h" 27 #include "gadget.h" 28 #include "io.h" 29 30 #define DWC3_ALIGN_FRAME(d, n) (((d)->frame_number + ((d)->interval * (n))) \ 31 & ~((d)->interval - 1)) 32 33 /** 34 * dwc3_gadget_set_test_mode - enables usb2 test modes 35 * @dwc: pointer to our context structure 36 * @mode: the mode to set (J, K SE0 NAK, Force Enable) 37 * 38 * Caller should take care of locking. This function will return 0 on 39 * success or -EINVAL if wrong Test Selector is passed. 40 */ 41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode) 42 { 43 u32 reg; 44 45 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 46 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 47 48 switch (mode) { 49 case USB_TEST_J: 50 case USB_TEST_K: 51 case USB_TEST_SE0_NAK: 52 case USB_TEST_PACKET: 53 case USB_TEST_FORCE_ENABLE: 54 reg |= mode << 1; 55 break; 56 default: 57 return -EINVAL; 58 } 59 60 dwc3_gadget_dctl_write_safe(dwc, reg); 61 62 return 0; 63 } 64 65 /** 66 * dwc3_gadget_get_link_state - gets current state of usb link 67 * @dwc: pointer to our context structure 68 * 69 * Caller should take care of locking. This function will 70 * return the link state on success (>= 0) or -ETIMEDOUT. 71 */ 72 int dwc3_gadget_get_link_state(struct dwc3 *dwc) 73 { 74 u32 reg; 75 76 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 77 78 return DWC3_DSTS_USBLNKST(reg); 79 } 80 81 /** 82 * dwc3_gadget_set_link_state - sets usb link to a particular state 83 * @dwc: pointer to our context structure 84 * @state: the state to put link into 85 * 86 * Caller should take care of locking. This function will 87 * return 0 on success or -ETIMEDOUT. 88 */ 89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state) 90 { 91 int retries = 10000; 92 u32 reg; 93 94 /* 95 * Wait until device controller is ready. Only applies to 1.94a and 96 * later RTL. 97 */ 98 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) { 99 while (--retries) { 100 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 101 if (reg & DWC3_DSTS_DCNRD) 102 udelay(5); 103 else 104 break; 105 } 106 107 if (retries <= 0) 108 return -ETIMEDOUT; 109 } 110 111 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 113 114 /* set no action before sending new link state change */ 115 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 116 117 /* set requested state */ 118 reg |= DWC3_DCTL_ULSTCHNGREQ(state); 119 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 120 121 /* 122 * The following code is racy when called from dwc3_gadget_wakeup, 123 * and is not needed, at least on newer versions 124 */ 125 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) 126 return 0; 127 128 /* wait for a change in DSTS */ 129 retries = 10000; 130 while (--retries) { 131 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 132 133 if (DWC3_DSTS_USBLNKST(reg) == state) 134 return 0; 135 136 udelay(5); 137 } 138 139 return -ETIMEDOUT; 140 } 141 142 /** 143 * dwc3_ep_inc_trb - increment a trb index. 144 * @index: Pointer to the TRB index to increment. 145 * 146 * The index should never point to the link TRB. After incrementing, 147 * if it is point to the link TRB, wrap around to the beginning. The 148 * link TRB is always at the last TRB entry. 149 */ 150 static void dwc3_ep_inc_trb(u8 *index) 151 { 152 (*index)++; 153 if (*index == (DWC3_TRB_NUM - 1)) 154 *index = 0; 155 } 156 157 /** 158 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer 159 * @dep: The endpoint whose enqueue pointer we're incrementing 160 */ 161 static void dwc3_ep_inc_enq(struct dwc3_ep *dep) 162 { 163 dwc3_ep_inc_trb(&dep->trb_enqueue); 164 } 165 166 /** 167 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer 168 * @dep: The endpoint whose enqueue pointer we're incrementing 169 */ 170 static void dwc3_ep_inc_deq(struct dwc3_ep *dep) 171 { 172 dwc3_ep_inc_trb(&dep->trb_dequeue); 173 } 174 175 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep, 176 struct dwc3_request *req, int status) 177 { 178 struct dwc3 *dwc = dep->dwc; 179 180 list_del(&req->list); 181 req->remaining = 0; 182 req->needs_extra_trb = false; 183 184 if (req->request.status == -EINPROGRESS) 185 req->request.status = status; 186 187 if (req->trb) 188 usb_gadget_unmap_request_by_dev(dwc->sysdev, 189 &req->request, req->direction); 190 191 req->trb = NULL; 192 trace_dwc3_gadget_giveback(req); 193 194 if (dep->number > 1) 195 pm_runtime_put(dwc->dev); 196 } 197 198 /** 199 * dwc3_gadget_giveback - call struct usb_request's ->complete callback 200 * @dep: The endpoint to whom the request belongs to 201 * @req: The request we're giving back 202 * @status: completion code for the request 203 * 204 * Must be called with controller's lock held and interrupts disabled. This 205 * function will unmap @req and call its ->complete() callback to notify upper 206 * layers that it has completed. 207 */ 208 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, 209 int status) 210 { 211 struct dwc3 *dwc = dep->dwc; 212 213 dwc3_gadget_del_and_unmap_request(dep, req, status); 214 req->status = DWC3_REQUEST_STATUS_COMPLETED; 215 216 spin_unlock(&dwc->lock); 217 usb_gadget_giveback_request(&dep->endpoint, &req->request); 218 spin_lock(&dwc->lock); 219 } 220 221 /** 222 * dwc3_send_gadget_generic_command - issue a generic command for the controller 223 * @dwc: pointer to the controller context 224 * @cmd: the command to be issued 225 * @param: command parameter 226 * 227 * Caller should take care of locking. Issue @cmd with a given @param to @dwc 228 * and wait for its completion. 229 */ 230 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned int cmd, 231 u32 param) 232 { 233 u32 timeout = 500; 234 int status = 0; 235 int ret = 0; 236 u32 reg; 237 238 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); 239 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); 240 241 do { 242 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); 243 if (!(reg & DWC3_DGCMD_CMDACT)) { 244 status = DWC3_DGCMD_STATUS(reg); 245 if (status) 246 ret = -EINVAL; 247 break; 248 } 249 } while (--timeout); 250 251 if (!timeout) { 252 ret = -ETIMEDOUT; 253 status = -ETIMEDOUT; 254 } 255 256 trace_dwc3_gadget_generic_cmd(cmd, param, status); 257 258 return ret; 259 } 260 261 static int __dwc3_gadget_wakeup(struct dwc3 *dwc); 262 263 /** 264 * dwc3_send_gadget_ep_cmd - issue an endpoint command 265 * @dep: the endpoint to which the command is going to be issued 266 * @cmd: the command to be issued 267 * @params: parameters to the command 268 * 269 * Caller should handle locking. This function will issue @cmd with given 270 * @params to @dep and wait for its completion. 271 */ 272 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned int cmd, 273 struct dwc3_gadget_ep_cmd_params *params) 274 { 275 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 276 struct dwc3 *dwc = dep->dwc; 277 u32 timeout = 5000; 278 u32 saved_config = 0; 279 u32 reg; 280 281 int cmd_status = 0; 282 int ret = -EINVAL; 283 284 /* 285 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or 286 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an 287 * endpoint command. 288 * 289 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY 290 * settings. Restore them after the command is completed. 291 * 292 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2 293 */ 294 if (dwc->gadget->speed <= USB_SPEED_HIGH) { 295 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 296 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) { 297 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY; 298 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY; 299 } 300 301 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) { 302 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM; 303 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM; 304 } 305 306 if (saved_config) 307 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 308 } 309 310 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) { 311 int link_state; 312 313 /* 314 * Initiate remote wakeup if the link state is in U3 when 315 * operating in SS/SSP or L1/L2 when operating in HS/FS. If the 316 * link state is in U1/U2, no remote wakeup is needed. The Start 317 * Transfer command will initiate the link recovery. 318 */ 319 link_state = dwc3_gadget_get_link_state(dwc); 320 switch (link_state) { 321 case DWC3_LINK_STATE_U2: 322 if (dwc->gadget->speed >= USB_SPEED_SUPER) 323 break; 324 325 fallthrough; 326 case DWC3_LINK_STATE_U3: 327 ret = __dwc3_gadget_wakeup(dwc); 328 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n", 329 ret); 330 break; 331 } 332 } 333 334 /* 335 * For some commands such as Update Transfer command, DEPCMDPARn 336 * registers are reserved. Since the driver often sends Update Transfer 337 * command, don't write to DEPCMDPARn to avoid register write delays and 338 * improve performance. 339 */ 340 if (DWC3_DEPCMD_CMD(cmd) != DWC3_DEPCMD_UPDATETRANSFER) { 341 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); 342 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); 343 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); 344 } 345 346 /* 347 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're 348 * not relying on XferNotReady, we can make use of a special "No 349 * Response Update Transfer" command where we should clear both CmdAct 350 * and CmdIOC bits. 351 * 352 * With this, we don't need to wait for command completion and can 353 * straight away issue further commands to the endpoint. 354 * 355 * NOTICE: We're making an assumption that control endpoints will never 356 * make use of Update Transfer command. This is a safe assumption 357 * because we can never have more than one request at a time with 358 * Control Endpoints. If anybody changes that assumption, this chunk 359 * needs to be updated accordingly. 360 */ 361 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER && 362 !usb_endpoint_xfer_isoc(desc)) 363 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT); 364 else 365 cmd |= DWC3_DEPCMD_CMDACT; 366 367 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); 368 369 if (!(cmd & DWC3_DEPCMD_CMDACT)) { 370 ret = 0; 371 goto skip_status; 372 } 373 374 do { 375 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); 376 if (!(reg & DWC3_DEPCMD_CMDACT)) { 377 cmd_status = DWC3_DEPCMD_STATUS(reg); 378 379 switch (cmd_status) { 380 case 0: 381 ret = 0; 382 break; 383 case DEPEVT_TRANSFER_NO_RESOURCE: 384 dev_WARN(dwc->dev, "No resource for %s\n", 385 dep->name); 386 ret = -EINVAL; 387 break; 388 case DEPEVT_TRANSFER_BUS_EXPIRY: 389 /* 390 * SW issues START TRANSFER command to 391 * isochronous ep with future frame interval. If 392 * future interval time has already passed when 393 * core receives the command, it will respond 394 * with an error status of 'Bus Expiry'. 395 * 396 * Instead of always returning -EINVAL, let's 397 * give a hint to the gadget driver that this is 398 * the case by returning -EAGAIN. 399 */ 400 ret = -EAGAIN; 401 break; 402 default: 403 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); 404 } 405 406 break; 407 } 408 } while (--timeout); 409 410 if (timeout == 0) { 411 ret = -ETIMEDOUT; 412 cmd_status = -ETIMEDOUT; 413 } 414 415 skip_status: 416 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status); 417 418 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) { 419 if (ret == 0) 420 dep->flags |= DWC3_EP_TRANSFER_STARTED; 421 422 if (ret != -ETIMEDOUT) 423 dwc3_gadget_ep_get_transfer_index(dep); 424 } 425 426 if (saved_config) { 427 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 428 reg |= saved_config; 429 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 430 } 431 432 return ret; 433 } 434 435 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep) 436 { 437 struct dwc3 *dwc = dep->dwc; 438 struct dwc3_gadget_ep_cmd_params params; 439 u32 cmd = DWC3_DEPCMD_CLEARSTALL; 440 441 /* 442 * As of core revision 2.60a the recommended programming model 443 * is to set the ClearPendIN bit when issuing a Clear Stall EP 444 * command for IN endpoints. This is to prevent an issue where 445 * some (non-compliant) hosts may not send ACK TPs for pending 446 * IN transfers due to a mishandled error condition. Synopsys 447 * STAR 9000614252. 448 */ 449 if (dep->direction && 450 !DWC3_VER_IS_PRIOR(DWC3, 260A) && 451 (dwc->gadget->speed >= USB_SPEED_SUPER)) 452 cmd |= DWC3_DEPCMD_CLEARPENDIN; 453 454 memset(¶ms, 0, sizeof(params)); 455 456 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 457 } 458 459 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, 460 struct dwc3_trb *trb) 461 { 462 u32 offset = (char *) trb - (char *) dep->trb_pool; 463 464 return dep->trb_pool_dma + offset; 465 } 466 467 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep) 468 { 469 struct dwc3 *dwc = dep->dwc; 470 471 if (dep->trb_pool) 472 return 0; 473 474 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, 475 sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 476 &dep->trb_pool_dma, GFP_KERNEL); 477 if (!dep->trb_pool) { 478 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", 479 dep->name); 480 return -ENOMEM; 481 } 482 483 return 0; 484 } 485 486 static void dwc3_free_trb_pool(struct dwc3_ep *dep) 487 { 488 struct dwc3 *dwc = dep->dwc; 489 490 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 491 dep->trb_pool, dep->trb_pool_dma); 492 493 dep->trb_pool = NULL; 494 dep->trb_pool_dma = 0; 495 } 496 497 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep) 498 { 499 struct dwc3_gadget_ep_cmd_params params; 500 501 memset(¶ms, 0x00, sizeof(params)); 502 503 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1); 504 505 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE, 506 ¶ms); 507 } 508 509 /** 510 * dwc3_gadget_start_config - configure ep resources 511 * @dep: endpoint that is being enabled 512 * 513 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's 514 * completion, it will set Transfer Resource for all available endpoints. 515 * 516 * The assignment of transfer resources cannot perfectly follow the data book 517 * due to the fact that the controller driver does not have all knowledge of the 518 * configuration in advance. It is given this information piecemeal by the 519 * composite gadget framework after every SET_CONFIGURATION and 520 * SET_INTERFACE. Trying to follow the databook programming model in this 521 * scenario can cause errors. For two reasons: 522 * 523 * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every 524 * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is 525 * incorrect in the scenario of multiple interfaces. 526 * 527 * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new 528 * endpoint on alt setting (8.1.6). 529 * 530 * The following simplified method is used instead: 531 * 532 * All hardware endpoints can be assigned a transfer resource and this setting 533 * will stay persistent until either a core reset or hibernation. So whenever we 534 * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do 535 * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are 536 * guaranteed that there are as many transfer resources as endpoints. 537 * 538 * This function is called for each endpoint when it is being enabled but is 539 * triggered only when called for EP0-out, which always happens first, and which 540 * should only happen in one of the above conditions. 541 */ 542 static int dwc3_gadget_start_config(struct dwc3_ep *dep) 543 { 544 struct dwc3_gadget_ep_cmd_params params; 545 struct dwc3 *dwc; 546 u32 cmd; 547 int i; 548 int ret; 549 550 if (dep->number) 551 return 0; 552 553 memset(¶ms, 0x00, sizeof(params)); 554 cmd = DWC3_DEPCMD_DEPSTARTCFG; 555 dwc = dep->dwc; 556 557 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 558 if (ret) 559 return ret; 560 561 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 562 struct dwc3_ep *dep = dwc->eps[i]; 563 564 if (!dep) 565 continue; 566 567 ret = dwc3_gadget_set_xfer_resource(dep); 568 if (ret) 569 return ret; 570 } 571 572 return 0; 573 } 574 575 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) 576 { 577 const struct usb_ss_ep_comp_descriptor *comp_desc; 578 const struct usb_endpoint_descriptor *desc; 579 struct dwc3_gadget_ep_cmd_params params; 580 struct dwc3 *dwc = dep->dwc; 581 582 comp_desc = dep->endpoint.comp_desc; 583 desc = dep->endpoint.desc; 584 585 memset(¶ms, 0x00, sizeof(params)); 586 587 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc)) 588 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc)); 589 590 /* Burst size is only needed in SuperSpeed mode */ 591 if (dwc->gadget->speed >= USB_SPEED_SUPER) { 592 u32 burst = dep->endpoint.maxburst; 593 594 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); 595 } 596 597 params.param0 |= action; 598 if (action == DWC3_DEPCFG_ACTION_RESTORE) 599 params.param2 |= dep->saved_state; 600 601 if (usb_endpoint_xfer_control(desc)) 602 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN; 603 604 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) 605 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN; 606 607 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) { 608 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE 609 | DWC3_DEPCFG_XFER_COMPLETE_EN 610 | DWC3_DEPCFG_STREAM_EVENT_EN; 611 dep->stream_capable = true; 612 } 613 614 if (!usb_endpoint_xfer_control(desc)) 615 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN; 616 617 /* 618 * We are doing 1:1 mapping for endpoints, meaning 619 * Physical Endpoints 2 maps to Logical Endpoint 2 and 620 * so on. We consider the direction bit as part of the physical 621 * endpoint number. So USB endpoint 0x81 is 0x03. 622 */ 623 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); 624 625 /* 626 * We must use the lower 16 TX FIFOs even though 627 * HW might have more 628 */ 629 if (dep->direction) 630 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); 631 632 if (desc->bInterval) { 633 u8 bInterval_m1; 634 635 /* 636 * Valid range for DEPCFG.bInterval_m1 is from 0 to 13. 637 * 638 * NOTE: The programming guide incorrectly stated bInterval_m1 639 * must be set to 0 when operating in fullspeed. Internally the 640 * controller does not have this limitation. See DWC_usb3x 641 * programming guide section 3.2.2.1. 642 */ 643 bInterval_m1 = min_t(u8, desc->bInterval - 1, 13); 644 645 if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT && 646 dwc->gadget->speed == USB_SPEED_FULL) 647 dep->interval = desc->bInterval; 648 else 649 dep->interval = 1 << (desc->bInterval - 1); 650 651 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(bInterval_m1); 652 } 653 654 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms); 655 } 656 657 /** 658 * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value 659 * @dwc: pointer to the DWC3 context 660 * @mult: multiplier to be used when calculating the fifo_size 661 * 662 * Calculates the size value based on the equation below: 663 * 664 * DWC3 revision 280A and prior: 665 * fifo_size = mult * (max_packet / mdwidth) + 1; 666 * 667 * DWC3 revision 290A and onwards: 668 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 669 * 670 * The max packet size is set to 1024, as the txfifo requirements mainly apply 671 * to super speed USB use cases. However, it is safe to overestimate the fifo 672 * allocations for other scenarios, i.e. high speed USB. 673 */ 674 static int dwc3_gadget_calc_tx_fifo_size(struct dwc3 *dwc, int mult) 675 { 676 int max_packet = 1024; 677 int fifo_size; 678 int mdwidth; 679 680 mdwidth = dwc3_mdwidth(dwc); 681 682 /* MDWIDTH is represented in bits, we need it in bytes */ 683 mdwidth >>= 3; 684 685 if (DWC3_VER_IS_PRIOR(DWC3, 290A)) 686 fifo_size = mult * (max_packet / mdwidth) + 1; 687 else 688 fifo_size = mult * ((max_packet + mdwidth) / mdwidth) + 1; 689 return fifo_size; 690 } 691 692 /** 693 * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation 694 * @dwc: pointer to the DWC3 context 695 * 696 * Iterates through all the endpoint registers and clears the previous txfifo 697 * allocations. 698 */ 699 void dwc3_gadget_clear_tx_fifos(struct dwc3 *dwc) 700 { 701 struct dwc3_ep *dep; 702 int fifo_depth; 703 int size; 704 int num; 705 706 if (!dwc->do_fifo_resize) 707 return; 708 709 /* Read ep0IN related TXFIFO size */ 710 dep = dwc->eps[1]; 711 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); 712 if (DWC3_IP_IS(DWC3)) 713 fifo_depth = DWC3_GTXFIFOSIZ_TXFDEP(size); 714 else 715 fifo_depth = DWC31_GTXFIFOSIZ_TXFDEP(size); 716 717 dwc->last_fifo_depth = fifo_depth; 718 /* Clear existing TXFIFO for all IN eps except ep0 */ 719 for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); 720 num += 2) { 721 dep = dwc->eps[num]; 722 /* Don't change TXFRAMNUM on usb31 version */ 723 size = DWC3_IP_IS(DWC3) ? 0 : 724 dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) & 725 DWC31_GTXFIFOSIZ_TXFRAMNUM; 726 727 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size); 728 dep->flags &= ~DWC3_EP_TXFIFO_RESIZED; 729 } 730 dwc->num_ep_resized = 0; 731 } 732 733 /* 734 * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case 735 * @dwc: pointer to our context structure 736 * 737 * This function will a best effort FIFO allocation in order 738 * to improve FIFO usage and throughput, while still allowing 739 * us to enable as many endpoints as possible. 740 * 741 * Keep in mind that this operation will be highly dependent 742 * on the configured size for RAM1 - which contains TxFifo -, 743 * the amount of endpoints enabled on coreConsultant tool, and 744 * the width of the Master Bus. 745 * 746 * In general, FIFO depths are represented with the following equation: 747 * 748 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 749 * 750 * In conjunction with dwc3_gadget_check_config(), this resizing logic will 751 * ensure that all endpoints will have enough internal memory for one max 752 * packet per endpoint. 753 */ 754 static int dwc3_gadget_resize_tx_fifos(struct dwc3_ep *dep) 755 { 756 struct dwc3 *dwc = dep->dwc; 757 int fifo_0_start; 758 int ram1_depth; 759 int fifo_size; 760 int min_depth; 761 int num_in_ep; 762 int remaining; 763 int num_fifos = 1; 764 int fifo; 765 int tmp; 766 767 if (!dwc->do_fifo_resize) 768 return 0; 769 770 /* resize IN endpoints except ep0 */ 771 if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1) 772 return 0; 773 774 /* bail if already resized */ 775 if (dep->flags & DWC3_EP_TXFIFO_RESIZED) 776 return 0; 777 778 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); 779 780 if ((dep->endpoint.maxburst > 1 && 781 usb_endpoint_xfer_bulk(dep->endpoint.desc)) || 782 usb_endpoint_xfer_isoc(dep->endpoint.desc)) 783 num_fifos = 3; 784 785 if (dep->endpoint.maxburst > 6 && 786 (usb_endpoint_xfer_bulk(dep->endpoint.desc) || 787 usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31)) 788 num_fifos = dwc->tx_fifo_resize_max_num; 789 790 /* FIFO size for a single buffer */ 791 fifo = dwc3_gadget_calc_tx_fifo_size(dwc, 1); 792 793 /* Calculate the number of remaining EPs w/o any FIFO */ 794 num_in_ep = dwc->max_cfg_eps; 795 num_in_ep -= dwc->num_ep_resized; 796 797 /* Reserve at least one FIFO for the number of IN EPs */ 798 min_depth = num_in_ep * (fifo + 1); 799 remaining = ram1_depth - min_depth - dwc->last_fifo_depth; 800 remaining = max_t(int, 0, remaining); 801 /* 802 * We've already reserved 1 FIFO per EP, so check what we can fit in 803 * addition to it. If there is not enough remaining space, allocate 804 * all the remaining space to the EP. 805 */ 806 fifo_size = (num_fifos - 1) * fifo; 807 if (remaining < fifo_size) 808 fifo_size = remaining; 809 810 fifo_size += fifo; 811 /* Last increment according to the TX FIFO size equation */ 812 fifo_size++; 813 814 /* Check if TXFIFOs start at non-zero addr */ 815 tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0)); 816 fifo_0_start = DWC3_GTXFIFOSIZ_TXFSTADDR(tmp); 817 818 fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16)); 819 if (DWC3_IP_IS(DWC3)) 820 dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); 821 else 822 dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); 823 824 /* Check fifo size allocation doesn't exceed available RAM size. */ 825 if (dwc->last_fifo_depth >= ram1_depth) { 826 dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n", 827 dwc->last_fifo_depth, ram1_depth, 828 dep->endpoint.name, fifo_size); 829 if (DWC3_IP_IS(DWC3)) 830 fifo_size = DWC3_GTXFIFOSIZ_TXFDEP(fifo_size); 831 else 832 fifo_size = DWC31_GTXFIFOSIZ_TXFDEP(fifo_size); 833 834 dwc->last_fifo_depth -= fifo_size; 835 return -ENOMEM; 836 } 837 838 dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size); 839 dep->flags |= DWC3_EP_TXFIFO_RESIZED; 840 dwc->num_ep_resized++; 841 842 return 0; 843 } 844 845 /** 846 * __dwc3_gadget_ep_enable - initializes a hw endpoint 847 * @dep: endpoint to be initialized 848 * @action: one of INIT, MODIFY or RESTORE 849 * 850 * Caller should take care of locking. Execute all necessary commands to 851 * initialize a HW endpoint so it can be used by a gadget driver. 852 */ 853 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) 854 { 855 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 856 struct dwc3 *dwc = dep->dwc; 857 858 u32 reg; 859 int ret; 860 861 if (!(dep->flags & DWC3_EP_ENABLED)) { 862 ret = dwc3_gadget_resize_tx_fifos(dep); 863 if (ret) 864 return ret; 865 866 ret = dwc3_gadget_start_config(dep); 867 if (ret) 868 return ret; 869 } 870 871 ret = dwc3_gadget_set_ep_config(dep, action); 872 if (ret) 873 return ret; 874 875 if (!(dep->flags & DWC3_EP_ENABLED)) { 876 struct dwc3_trb *trb_st_hw; 877 struct dwc3_trb *trb_link; 878 879 dep->type = usb_endpoint_type(desc); 880 dep->flags |= DWC3_EP_ENABLED; 881 882 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 883 reg |= DWC3_DALEPENA_EP(dep->number); 884 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 885 886 dep->trb_dequeue = 0; 887 dep->trb_enqueue = 0; 888 889 if (usb_endpoint_xfer_control(desc)) 890 goto out; 891 892 /* Initialize the TRB ring */ 893 memset(dep->trb_pool, 0, 894 sizeof(struct dwc3_trb) * DWC3_TRB_NUM); 895 896 /* Link TRB. The HWO bit is never reset */ 897 trb_st_hw = &dep->trb_pool[0]; 898 899 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; 900 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 901 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 902 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; 903 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; 904 } 905 906 /* 907 * Issue StartTransfer here with no-op TRB so we can always rely on No 908 * Response Update Transfer command. 909 */ 910 if (usb_endpoint_xfer_bulk(desc) || 911 usb_endpoint_xfer_int(desc)) { 912 struct dwc3_gadget_ep_cmd_params params; 913 struct dwc3_trb *trb; 914 dma_addr_t trb_dma; 915 u32 cmd; 916 917 memset(¶ms, 0, sizeof(params)); 918 trb = &dep->trb_pool[0]; 919 trb_dma = dwc3_trb_dma_offset(dep, trb); 920 921 params.param0 = upper_32_bits(trb_dma); 922 params.param1 = lower_32_bits(trb_dma); 923 924 cmd = DWC3_DEPCMD_STARTTRANSFER; 925 926 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 927 if (ret < 0) 928 return ret; 929 930 if (dep->stream_capable) { 931 /* 932 * For streams, at start, there maybe a race where the 933 * host primes the endpoint before the function driver 934 * queues a request to initiate a stream. In that case, 935 * the controller will not see the prime to generate the 936 * ERDY and start stream. To workaround this, issue a 937 * no-op TRB as normal, but end it immediately. As a 938 * result, when the function driver queues the request, 939 * the next START_TRANSFER command will cause the 940 * controller to generate an ERDY to initiate the 941 * stream. 942 */ 943 dwc3_stop_active_transfer(dep, true, true); 944 945 /* 946 * All stream eps will reinitiate stream on NoStream 947 * rejection until we can determine that the host can 948 * prime after the first transfer. 949 * 950 * However, if the controller is capable of 951 * TXF_FLUSH_BYPASS, then IN direction endpoints will 952 * automatically restart the stream without the driver 953 * initiation. 954 */ 955 if (!dep->direction || 956 !(dwc->hwparams.hwparams9 & 957 DWC3_GHWPARAMS9_DEV_TXF_FLUSH_BYPASS)) 958 dep->flags |= DWC3_EP_FORCE_RESTART_STREAM; 959 } 960 } 961 962 out: 963 trace_dwc3_gadget_ep_enable(dep); 964 965 return 0; 966 } 967 968 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep) 969 { 970 struct dwc3_request *req; 971 972 dwc3_stop_active_transfer(dep, true, false); 973 974 /* - giveback all requests to gadget driver */ 975 while (!list_empty(&dep->started_list)) { 976 req = next_request(&dep->started_list); 977 978 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 979 } 980 981 while (!list_empty(&dep->pending_list)) { 982 req = next_request(&dep->pending_list); 983 984 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 985 } 986 987 while (!list_empty(&dep->cancelled_list)) { 988 req = next_request(&dep->cancelled_list); 989 990 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 991 } 992 } 993 994 /** 995 * __dwc3_gadget_ep_disable - disables a hw endpoint 996 * @dep: the endpoint to disable 997 * 998 * This function undoes what __dwc3_gadget_ep_enable did and also removes 999 * requests which are currently being processed by the hardware and those which 1000 * are not yet scheduled. 1001 * 1002 * Caller should take care of locking. 1003 */ 1004 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) 1005 { 1006 struct dwc3 *dwc = dep->dwc; 1007 u32 reg; 1008 1009 trace_dwc3_gadget_ep_disable(dep); 1010 1011 /* make sure HW endpoint isn't stalled */ 1012 if (dep->flags & DWC3_EP_STALL) 1013 __dwc3_gadget_ep_set_halt(dep, 0, false); 1014 1015 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 1016 reg &= ~DWC3_DALEPENA_EP(dep->number); 1017 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 1018 1019 /* Clear out the ep descriptors for non-ep0 */ 1020 if (dep->number > 1) { 1021 dep->endpoint.comp_desc = NULL; 1022 dep->endpoint.desc = NULL; 1023 } 1024 1025 dwc3_remove_requests(dwc, dep); 1026 1027 dep->stream_capable = false; 1028 dep->type = 0; 1029 dep->flags &= DWC3_EP_TXFIFO_RESIZED; 1030 1031 return 0; 1032 } 1033 1034 /* -------------------------------------------------------------------------- */ 1035 1036 static int dwc3_gadget_ep0_enable(struct usb_ep *ep, 1037 const struct usb_endpoint_descriptor *desc) 1038 { 1039 return -EINVAL; 1040 } 1041 1042 static int dwc3_gadget_ep0_disable(struct usb_ep *ep) 1043 { 1044 return -EINVAL; 1045 } 1046 1047 /* -------------------------------------------------------------------------- */ 1048 1049 static int dwc3_gadget_ep_enable(struct usb_ep *ep, 1050 const struct usb_endpoint_descriptor *desc) 1051 { 1052 struct dwc3_ep *dep; 1053 struct dwc3 *dwc; 1054 unsigned long flags; 1055 int ret; 1056 1057 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { 1058 pr_debug("dwc3: invalid parameters\n"); 1059 return -EINVAL; 1060 } 1061 1062 if (!desc->wMaxPacketSize) { 1063 pr_debug("dwc3: missing wMaxPacketSize\n"); 1064 return -EINVAL; 1065 } 1066 1067 dep = to_dwc3_ep(ep); 1068 dwc = dep->dwc; 1069 1070 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, 1071 "%s is already enabled\n", 1072 dep->name)) 1073 return 0; 1074 1075 spin_lock_irqsave(&dwc->lock, flags); 1076 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 1077 spin_unlock_irqrestore(&dwc->lock, flags); 1078 1079 return ret; 1080 } 1081 1082 static int dwc3_gadget_ep_disable(struct usb_ep *ep) 1083 { 1084 struct dwc3_ep *dep; 1085 struct dwc3 *dwc; 1086 unsigned long flags; 1087 int ret; 1088 1089 if (!ep) { 1090 pr_debug("dwc3: invalid parameters\n"); 1091 return -EINVAL; 1092 } 1093 1094 dep = to_dwc3_ep(ep); 1095 dwc = dep->dwc; 1096 1097 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), 1098 "%s is already disabled\n", 1099 dep->name)) 1100 return 0; 1101 1102 spin_lock_irqsave(&dwc->lock, flags); 1103 ret = __dwc3_gadget_ep_disable(dep); 1104 spin_unlock_irqrestore(&dwc->lock, flags); 1105 1106 return ret; 1107 } 1108 1109 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep, 1110 gfp_t gfp_flags) 1111 { 1112 struct dwc3_request *req; 1113 struct dwc3_ep *dep = to_dwc3_ep(ep); 1114 1115 req = kzalloc(sizeof(*req), gfp_flags); 1116 if (!req) 1117 return NULL; 1118 1119 req->direction = dep->direction; 1120 req->epnum = dep->number; 1121 req->dep = dep; 1122 req->status = DWC3_REQUEST_STATUS_UNKNOWN; 1123 1124 trace_dwc3_alloc_request(req); 1125 1126 return &req->request; 1127 } 1128 1129 static void dwc3_gadget_ep_free_request(struct usb_ep *ep, 1130 struct usb_request *request) 1131 { 1132 struct dwc3_request *req = to_dwc3_request(request); 1133 1134 trace_dwc3_free_request(req); 1135 kfree(req); 1136 } 1137 1138 /** 1139 * dwc3_ep_prev_trb - returns the previous TRB in the ring 1140 * @dep: The endpoint with the TRB ring 1141 * @index: The index of the current TRB in the ring 1142 * 1143 * Returns the TRB prior to the one pointed to by the index. If the 1144 * index is 0, we will wrap backwards, skip the link TRB, and return 1145 * the one just before that. 1146 */ 1147 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index) 1148 { 1149 u8 tmp = index; 1150 1151 if (!tmp) 1152 tmp = DWC3_TRB_NUM - 1; 1153 1154 return &dep->trb_pool[tmp - 1]; 1155 } 1156 1157 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep) 1158 { 1159 u8 trbs_left; 1160 1161 /* 1162 * If the enqueue & dequeue are equal then the TRB ring is either full 1163 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs 1164 * pending to be processed by the driver. 1165 */ 1166 if (dep->trb_enqueue == dep->trb_dequeue) { 1167 /* 1168 * If there is any request remained in the started_list at 1169 * this point, that means there is no TRB available. 1170 */ 1171 if (!list_empty(&dep->started_list)) 1172 return 0; 1173 1174 return DWC3_TRB_NUM - 1; 1175 } 1176 1177 trbs_left = dep->trb_dequeue - dep->trb_enqueue; 1178 trbs_left &= (DWC3_TRB_NUM - 1); 1179 1180 if (dep->trb_dequeue < dep->trb_enqueue) 1181 trbs_left--; 1182 1183 return trbs_left; 1184 } 1185 1186 /** 1187 * dwc3_prepare_one_trb - setup one TRB from one request 1188 * @dep: endpoint for which this request is prepared 1189 * @req: dwc3_request pointer 1190 * @trb_length: buffer size of the TRB 1191 * @chain: should this TRB be chained to the next? 1192 * @node: only for isochronous endpoints. First TRB needs different type. 1193 * @use_bounce_buffer: set to use bounce buffer 1194 * @must_interrupt: set to interrupt on TRB completion 1195 */ 1196 static void dwc3_prepare_one_trb(struct dwc3_ep *dep, 1197 struct dwc3_request *req, unsigned int trb_length, 1198 unsigned int chain, unsigned int node, bool use_bounce_buffer, 1199 bool must_interrupt) 1200 { 1201 struct dwc3_trb *trb; 1202 dma_addr_t dma; 1203 unsigned int stream_id = req->request.stream_id; 1204 unsigned int short_not_ok = req->request.short_not_ok; 1205 unsigned int no_interrupt = req->request.no_interrupt; 1206 unsigned int is_last = req->request.is_last; 1207 struct dwc3 *dwc = dep->dwc; 1208 struct usb_gadget *gadget = dwc->gadget; 1209 enum usb_device_speed speed = gadget->speed; 1210 1211 if (use_bounce_buffer) 1212 dma = dep->dwc->bounce_addr; 1213 else if (req->request.num_sgs > 0) 1214 dma = sg_dma_address(req->start_sg); 1215 else 1216 dma = req->request.dma; 1217 1218 trb = &dep->trb_pool[dep->trb_enqueue]; 1219 1220 if (!req->trb) { 1221 dwc3_gadget_move_started_request(req); 1222 req->trb = trb; 1223 req->trb_dma = dwc3_trb_dma_offset(dep, trb); 1224 } 1225 1226 req->num_trbs++; 1227 1228 trb->size = DWC3_TRB_SIZE_LENGTH(trb_length); 1229 trb->bpl = lower_32_bits(dma); 1230 trb->bph = upper_32_bits(dma); 1231 1232 switch (usb_endpoint_type(dep->endpoint.desc)) { 1233 case USB_ENDPOINT_XFER_CONTROL: 1234 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; 1235 break; 1236 1237 case USB_ENDPOINT_XFER_ISOC: 1238 if (!node) { 1239 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; 1240 1241 /* 1242 * USB Specification 2.0 Section 5.9.2 states that: "If 1243 * there is only a single transaction in the microframe, 1244 * only a DATA0 data packet PID is used. If there are 1245 * two transactions per microframe, DATA1 is used for 1246 * the first transaction data packet and DATA0 is used 1247 * for the second transaction data packet. If there are 1248 * three transactions per microframe, DATA2 is used for 1249 * the first transaction data packet, DATA1 is used for 1250 * the second, and DATA0 is used for the third." 1251 * 1252 * IOW, we should satisfy the following cases: 1253 * 1254 * 1) length <= maxpacket 1255 * - DATA0 1256 * 1257 * 2) maxpacket < length <= (2 * maxpacket) 1258 * - DATA1, DATA0 1259 * 1260 * 3) (2 * maxpacket) < length <= (3 * maxpacket) 1261 * - DATA2, DATA1, DATA0 1262 */ 1263 if (speed == USB_SPEED_HIGH) { 1264 struct usb_ep *ep = &dep->endpoint; 1265 unsigned int mult = 2; 1266 unsigned int maxp = usb_endpoint_maxp(ep->desc); 1267 1268 if (req->request.length <= (2 * maxp)) 1269 mult--; 1270 1271 if (req->request.length <= maxp) 1272 mult--; 1273 1274 trb->size |= DWC3_TRB_SIZE_PCM1(mult); 1275 } 1276 } else { 1277 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; 1278 } 1279 1280 /* always enable Interrupt on Missed ISOC */ 1281 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 1282 break; 1283 1284 case USB_ENDPOINT_XFER_BULK: 1285 case USB_ENDPOINT_XFER_INT: 1286 trb->ctrl = DWC3_TRBCTL_NORMAL; 1287 break; 1288 default: 1289 /* 1290 * This is only possible with faulty memory because we 1291 * checked it already :) 1292 */ 1293 dev_WARN(dwc->dev, "Unknown endpoint type %d\n", 1294 usb_endpoint_type(dep->endpoint.desc)); 1295 } 1296 1297 /* 1298 * Enable Continue on Short Packet 1299 * when endpoint is not a stream capable 1300 */ 1301 if (usb_endpoint_dir_out(dep->endpoint.desc)) { 1302 if (!dep->stream_capable) 1303 trb->ctrl |= DWC3_TRB_CTRL_CSP; 1304 1305 if (short_not_ok) 1306 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 1307 } 1308 1309 /* All TRBs setup for MST must set CSP=1 when LST=0 */ 1310 if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams)) 1311 trb->ctrl |= DWC3_TRB_CTRL_CSP; 1312 1313 if ((!no_interrupt && !chain) || must_interrupt) 1314 trb->ctrl |= DWC3_TRB_CTRL_IOC; 1315 1316 if (chain) 1317 trb->ctrl |= DWC3_TRB_CTRL_CHN; 1318 else if (dep->stream_capable && is_last && 1319 !DWC3_MST_CAPABLE(&dwc->hwparams)) 1320 trb->ctrl |= DWC3_TRB_CTRL_LST; 1321 1322 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) 1323 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id); 1324 1325 /* 1326 * As per data book 4.2.3.2TRB Control Bit Rules section 1327 * 1328 * The controller autonomously checks the HWO field of a TRB to determine if the 1329 * entire TRB is valid. Therefore, software must ensure that the rest of the TRB 1330 * is valid before setting the HWO field to '1'. In most systems, this means that 1331 * software must update the fourth DWORD of a TRB last. 1332 * 1333 * However there is a possibility of CPU re-ordering here which can cause 1334 * controller to observe the HWO bit set prematurely. 1335 * Add a write memory barrier to prevent CPU re-ordering. 1336 */ 1337 wmb(); 1338 trb->ctrl |= DWC3_TRB_CTRL_HWO; 1339 1340 dwc3_ep_inc_enq(dep); 1341 1342 trace_dwc3_prepare_trb(dep, trb); 1343 } 1344 1345 static bool dwc3_needs_extra_trb(struct dwc3_ep *dep, struct dwc3_request *req) 1346 { 1347 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1348 unsigned int rem = req->request.length % maxp; 1349 1350 if ((req->request.length && req->request.zero && !rem && 1351 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) || 1352 (!req->direction && rem)) 1353 return true; 1354 1355 return false; 1356 } 1357 1358 /** 1359 * dwc3_prepare_last_sg - prepare TRBs for the last SG entry 1360 * @dep: The endpoint that the request belongs to 1361 * @req: The request to prepare 1362 * @entry_length: The last SG entry size 1363 * @node: Indicates whether this is not the first entry (for isoc only) 1364 * 1365 * Return the number of TRBs prepared. 1366 */ 1367 static int dwc3_prepare_last_sg(struct dwc3_ep *dep, 1368 struct dwc3_request *req, unsigned int entry_length, 1369 unsigned int node) 1370 { 1371 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1372 unsigned int rem = req->request.length % maxp; 1373 unsigned int num_trbs = 1; 1374 1375 if (dwc3_needs_extra_trb(dep, req)) 1376 num_trbs++; 1377 1378 if (dwc3_calc_trbs_left(dep) < num_trbs) 1379 return 0; 1380 1381 req->needs_extra_trb = num_trbs > 1; 1382 1383 /* Prepare a normal TRB */ 1384 if (req->direction || req->request.length) 1385 dwc3_prepare_one_trb(dep, req, entry_length, 1386 req->needs_extra_trb, node, false, false); 1387 1388 /* Prepare extra TRBs for ZLP and MPS OUT transfer alignment */ 1389 if ((!req->direction && !req->request.length) || req->needs_extra_trb) 1390 dwc3_prepare_one_trb(dep, req, 1391 req->direction ? 0 : maxp - rem, 1392 false, 1, true, false); 1393 1394 return num_trbs; 1395 } 1396 1397 static int dwc3_prepare_trbs_sg(struct dwc3_ep *dep, 1398 struct dwc3_request *req) 1399 { 1400 struct scatterlist *sg = req->start_sg; 1401 struct scatterlist *s; 1402 int i; 1403 unsigned int length = req->request.length; 1404 unsigned int remaining = req->request.num_mapped_sgs 1405 - req->num_queued_sgs; 1406 unsigned int num_trbs = req->num_trbs; 1407 bool needs_extra_trb = dwc3_needs_extra_trb(dep, req); 1408 1409 /* 1410 * If we resume preparing the request, then get the remaining length of 1411 * the request and resume where we left off. 1412 */ 1413 for_each_sg(req->request.sg, s, req->num_queued_sgs, i) 1414 length -= sg_dma_len(s); 1415 1416 for_each_sg(sg, s, remaining, i) { 1417 unsigned int num_trbs_left = dwc3_calc_trbs_left(dep); 1418 unsigned int trb_length; 1419 bool must_interrupt = false; 1420 bool last_sg = false; 1421 1422 trb_length = min_t(unsigned int, length, sg_dma_len(s)); 1423 1424 length -= trb_length; 1425 1426 /* 1427 * IOMMU driver is coalescing the list of sgs which shares a 1428 * page boundary into one and giving it to USB driver. With 1429 * this the number of sgs mapped is not equal to the number of 1430 * sgs passed. So mark the chain bit to false if it isthe last 1431 * mapped sg. 1432 */ 1433 if ((i == remaining - 1) || !length) 1434 last_sg = true; 1435 1436 if (!num_trbs_left) 1437 break; 1438 1439 if (last_sg) { 1440 if (!dwc3_prepare_last_sg(dep, req, trb_length, i)) 1441 break; 1442 } else { 1443 /* 1444 * Look ahead to check if we have enough TRBs for the 1445 * next SG entry. If not, set interrupt on this TRB to 1446 * resume preparing the next SG entry when more TRBs are 1447 * free. 1448 */ 1449 if (num_trbs_left == 1 || (needs_extra_trb && 1450 num_trbs_left <= 2 && 1451 sg_dma_len(sg_next(s)) >= length)) 1452 must_interrupt = true; 1453 1454 dwc3_prepare_one_trb(dep, req, trb_length, 1, i, false, 1455 must_interrupt); 1456 } 1457 1458 /* 1459 * There can be a situation where all sgs in sglist are not 1460 * queued because of insufficient trb number. To handle this 1461 * case, update start_sg to next sg to be queued, so that 1462 * we have free trbs we can continue queuing from where we 1463 * previously stopped 1464 */ 1465 if (!last_sg) 1466 req->start_sg = sg_next(s); 1467 1468 req->num_queued_sgs++; 1469 req->num_pending_sgs--; 1470 1471 /* 1472 * The number of pending SG entries may not correspond to the 1473 * number of mapped SG entries. If all the data are queued, then 1474 * don't include unused SG entries. 1475 */ 1476 if (length == 0) { 1477 req->num_pending_sgs = 0; 1478 break; 1479 } 1480 1481 if (must_interrupt) 1482 break; 1483 } 1484 1485 return req->num_trbs - num_trbs; 1486 } 1487 1488 static int dwc3_prepare_trbs_linear(struct dwc3_ep *dep, 1489 struct dwc3_request *req) 1490 { 1491 return dwc3_prepare_last_sg(dep, req, req->request.length, 0); 1492 } 1493 1494 /* 1495 * dwc3_prepare_trbs - setup TRBs from requests 1496 * @dep: endpoint for which requests are being prepared 1497 * 1498 * The function goes through the requests list and sets up TRBs for the 1499 * transfers. The function returns once there are no more TRBs available or 1500 * it runs out of requests. 1501 * 1502 * Returns the number of TRBs prepared or negative errno. 1503 */ 1504 static int dwc3_prepare_trbs(struct dwc3_ep *dep) 1505 { 1506 struct dwc3_request *req, *n; 1507 int ret = 0; 1508 1509 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM); 1510 1511 /* 1512 * We can get in a situation where there's a request in the started list 1513 * but there weren't enough TRBs to fully kick it in the first time 1514 * around, so it has been waiting for more TRBs to be freed up. 1515 * 1516 * In that case, we should check if we have a request with pending_sgs 1517 * in the started list and prepare TRBs for that request first, 1518 * otherwise we will prepare TRBs completely out of order and that will 1519 * break things. 1520 */ 1521 list_for_each_entry(req, &dep->started_list, list) { 1522 if (req->num_pending_sgs > 0) { 1523 ret = dwc3_prepare_trbs_sg(dep, req); 1524 if (!ret || req->num_pending_sgs) 1525 return ret; 1526 } 1527 1528 if (!dwc3_calc_trbs_left(dep)) 1529 return ret; 1530 1531 /* 1532 * Don't prepare beyond a transfer. In DWC_usb32, its transfer 1533 * burst capability may try to read and use TRBs beyond the 1534 * active transfer instead of stopping. 1535 */ 1536 if (dep->stream_capable && req->request.is_last && 1537 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) 1538 return ret; 1539 } 1540 1541 list_for_each_entry_safe(req, n, &dep->pending_list, list) { 1542 struct dwc3 *dwc = dep->dwc; 1543 1544 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request, 1545 dep->direction); 1546 if (ret) 1547 return ret; 1548 1549 req->sg = req->request.sg; 1550 req->start_sg = req->sg; 1551 req->num_queued_sgs = 0; 1552 req->num_pending_sgs = req->request.num_mapped_sgs; 1553 1554 if (req->num_pending_sgs > 0) { 1555 ret = dwc3_prepare_trbs_sg(dep, req); 1556 if (req->num_pending_sgs) 1557 return ret; 1558 } else { 1559 ret = dwc3_prepare_trbs_linear(dep, req); 1560 } 1561 1562 if (!ret || !dwc3_calc_trbs_left(dep)) 1563 return ret; 1564 1565 /* 1566 * Don't prepare beyond a transfer. In DWC_usb32, its transfer 1567 * burst capability may try to read and use TRBs beyond the 1568 * active transfer instead of stopping. 1569 */ 1570 if (dep->stream_capable && req->request.is_last && 1571 !DWC3_MST_CAPABLE(&dwc->hwparams)) 1572 return ret; 1573 } 1574 1575 return ret; 1576 } 1577 1578 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep); 1579 1580 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) 1581 { 1582 struct dwc3_gadget_ep_cmd_params params; 1583 struct dwc3_request *req; 1584 int starting; 1585 int ret; 1586 u32 cmd; 1587 1588 /* 1589 * Note that it's normal to have no new TRBs prepared (i.e. ret == 0). 1590 * This happens when we need to stop and restart a transfer such as in 1591 * the case of reinitiating a stream or retrying an isoc transfer. 1592 */ 1593 ret = dwc3_prepare_trbs(dep); 1594 if (ret < 0) 1595 return ret; 1596 1597 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); 1598 1599 /* 1600 * If there's no new TRB prepared and we don't need to restart a 1601 * transfer, there's no need to update the transfer. 1602 */ 1603 if (!ret && !starting) 1604 return ret; 1605 1606 req = next_request(&dep->started_list); 1607 if (!req) { 1608 dep->flags |= DWC3_EP_PENDING_REQUEST; 1609 return 0; 1610 } 1611 1612 memset(¶ms, 0, sizeof(params)); 1613 1614 if (starting) { 1615 params.param0 = upper_32_bits(req->trb_dma); 1616 params.param1 = lower_32_bits(req->trb_dma); 1617 cmd = DWC3_DEPCMD_STARTTRANSFER; 1618 1619 if (dep->stream_capable) 1620 cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id); 1621 1622 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) 1623 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); 1624 } else { 1625 cmd = DWC3_DEPCMD_UPDATETRANSFER | 1626 DWC3_DEPCMD_PARAM(dep->resource_index); 1627 } 1628 1629 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1630 if (ret < 0) { 1631 struct dwc3_request *tmp; 1632 1633 if (ret == -EAGAIN) 1634 return ret; 1635 1636 dwc3_stop_active_transfer(dep, true, true); 1637 1638 list_for_each_entry_safe(req, tmp, &dep->started_list, list) 1639 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_DEQUEUED); 1640 1641 /* If ep isn't started, then there's no end transfer pending */ 1642 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 1643 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 1644 1645 return ret; 1646 } 1647 1648 if (dep->stream_capable && req->request.is_last && 1649 !DWC3_MST_CAPABLE(&dep->dwc->hwparams)) 1650 dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE; 1651 1652 return 0; 1653 } 1654 1655 static int __dwc3_gadget_get_frame(struct dwc3 *dwc) 1656 { 1657 u32 reg; 1658 1659 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1660 return DWC3_DSTS_SOFFN(reg); 1661 } 1662 1663 /** 1664 * __dwc3_stop_active_transfer - stop the current active transfer 1665 * @dep: isoc endpoint 1666 * @force: set forcerm bit in the command 1667 * @interrupt: command complete interrupt after End Transfer command 1668 * 1669 * When setting force, the ForceRM bit will be set. In that case 1670 * the controller won't update the TRB progress on command 1671 * completion. It also won't clear the HWO bit in the TRB. 1672 * The command will also not complete immediately in that case. 1673 */ 1674 static int __dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, bool interrupt) 1675 { 1676 struct dwc3_gadget_ep_cmd_params params; 1677 u32 cmd; 1678 int ret; 1679 1680 cmd = DWC3_DEPCMD_ENDTRANSFER; 1681 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0; 1682 cmd |= interrupt ? DWC3_DEPCMD_CMDIOC : 0; 1683 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); 1684 memset(¶ms, 0, sizeof(params)); 1685 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1686 WARN_ON_ONCE(ret); 1687 dep->resource_index = 0; 1688 1689 if (!interrupt) 1690 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 1691 else if (!ret) 1692 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; 1693 1694 return ret; 1695 } 1696 1697 /** 1698 * dwc3_gadget_start_isoc_quirk - workaround invalid frame number 1699 * @dep: isoc endpoint 1700 * 1701 * This function tests for the correct combination of BIT[15:14] from the 16-bit 1702 * microframe number reported by the XferNotReady event for the future frame 1703 * number to start the isoc transfer. 1704 * 1705 * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed 1706 * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the 1707 * XferNotReady event are invalid. The driver uses this number to schedule the 1708 * isochronous transfer and passes it to the START TRANSFER command. Because 1709 * this number is invalid, the command may fail. If BIT[15:14] matches the 1710 * internal 16-bit microframe, the START TRANSFER command will pass and the 1711 * transfer will start at the scheduled time, if it is off by 1, the command 1712 * will still pass, but the transfer will start 2 seconds in the future. For all 1713 * other conditions, the START TRANSFER command will fail with bus-expiry. 1714 * 1715 * In order to workaround this issue, we can test for the correct combination of 1716 * BIT[15:14] by sending START TRANSFER commands with different values of 1717 * BIT[15:14]: 'b00, 'b01, 'b10, and 'b11. Each combination is 2^14 uframe apart 1718 * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status. 1719 * As the result, within the 4 possible combinations for BIT[15:14], there will 1720 * be 2 successful and 2 failure START COMMAND status. One of the 2 successful 1721 * command status will result in a 2-second delay start. The smaller BIT[15:14] 1722 * value is the correct combination. 1723 * 1724 * Since there are only 4 outcomes and the results are ordered, we can simply 1725 * test 2 START TRANSFER commands with BIT[15:14] combinations 'b00 and 'b01 to 1726 * deduce the smaller successful combination. 1727 * 1728 * Let test0 = test status for combination 'b00 and test1 = test status for 'b01 1729 * of BIT[15:14]. The correct combination is as follow: 1730 * 1731 * if test0 fails and test1 passes, BIT[15:14] is 'b01 1732 * if test0 fails and test1 fails, BIT[15:14] is 'b10 1733 * if test0 passes and test1 fails, BIT[15:14] is 'b11 1734 * if test0 passes and test1 passes, BIT[15:14] is 'b00 1735 * 1736 * Synopsys STAR 9001202023: Wrong microframe number for isochronous IN 1737 * endpoints. 1738 */ 1739 static int dwc3_gadget_start_isoc_quirk(struct dwc3_ep *dep) 1740 { 1741 int cmd_status = 0; 1742 bool test0; 1743 bool test1; 1744 1745 while (dep->combo_num < 2) { 1746 struct dwc3_gadget_ep_cmd_params params; 1747 u32 test_frame_number; 1748 u32 cmd; 1749 1750 /* 1751 * Check if we can start isoc transfer on the next interval or 1752 * 4 uframes in the future with BIT[15:14] as dep->combo_num 1753 */ 1754 test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK; 1755 test_frame_number |= dep->combo_num << 14; 1756 test_frame_number += max_t(u32, 4, dep->interval); 1757 1758 params.param0 = upper_32_bits(dep->dwc->bounce_addr); 1759 params.param1 = lower_32_bits(dep->dwc->bounce_addr); 1760 1761 cmd = DWC3_DEPCMD_STARTTRANSFER; 1762 cmd |= DWC3_DEPCMD_PARAM(test_frame_number); 1763 cmd_status = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1764 1765 /* Redo if some other failure beside bus-expiry is received */ 1766 if (cmd_status && cmd_status != -EAGAIN) { 1767 dep->start_cmd_status = 0; 1768 dep->combo_num = 0; 1769 return 0; 1770 } 1771 1772 /* Store the first test status */ 1773 if (dep->combo_num == 0) 1774 dep->start_cmd_status = cmd_status; 1775 1776 dep->combo_num++; 1777 1778 /* 1779 * End the transfer if the START_TRANSFER command is successful 1780 * to wait for the next XferNotReady to test the command again 1781 */ 1782 if (cmd_status == 0) { 1783 dwc3_stop_active_transfer(dep, true, true); 1784 return 0; 1785 } 1786 } 1787 1788 /* test0 and test1 are both completed at this point */ 1789 test0 = (dep->start_cmd_status == 0); 1790 test1 = (cmd_status == 0); 1791 1792 if (!test0 && test1) 1793 dep->combo_num = 1; 1794 else if (!test0 && !test1) 1795 dep->combo_num = 2; 1796 else if (test0 && !test1) 1797 dep->combo_num = 3; 1798 else if (test0 && test1) 1799 dep->combo_num = 0; 1800 1801 dep->frame_number &= DWC3_FRNUMBER_MASK; 1802 dep->frame_number |= dep->combo_num << 14; 1803 dep->frame_number += max_t(u32, 4, dep->interval); 1804 1805 /* Reinitialize test variables */ 1806 dep->start_cmd_status = 0; 1807 dep->combo_num = 0; 1808 1809 return __dwc3_gadget_kick_transfer(dep); 1810 } 1811 1812 static int __dwc3_gadget_start_isoc(struct dwc3_ep *dep) 1813 { 1814 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 1815 struct dwc3 *dwc = dep->dwc; 1816 int ret; 1817 int i; 1818 1819 if (list_empty(&dep->pending_list) && 1820 list_empty(&dep->started_list)) { 1821 dep->flags |= DWC3_EP_PENDING_REQUEST; 1822 return -EAGAIN; 1823 } 1824 1825 if (!dwc->dis_start_transfer_quirk && 1826 (DWC3_VER_IS_PRIOR(DWC31, 170A) || 1827 DWC3_VER_TYPE_IS_WITHIN(DWC31, 170A, EA01, EA06))) { 1828 if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction) 1829 return dwc3_gadget_start_isoc_quirk(dep); 1830 } 1831 1832 if (desc->bInterval <= 14 && 1833 dwc->gadget->speed >= USB_SPEED_HIGH) { 1834 u32 frame = __dwc3_gadget_get_frame(dwc); 1835 bool rollover = frame < 1836 (dep->frame_number & DWC3_FRNUMBER_MASK); 1837 1838 /* 1839 * frame_number is set from XferNotReady and may be already 1840 * out of date. DSTS only provides the lower 14 bit of the 1841 * current frame number. So add the upper two bits of 1842 * frame_number and handle a possible rollover. 1843 * This will provide the correct frame_number unless more than 1844 * rollover has happened since XferNotReady. 1845 */ 1846 1847 dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) | 1848 frame; 1849 if (rollover) 1850 dep->frame_number += BIT(14); 1851 } 1852 1853 for (i = 0; i < DWC3_ISOC_MAX_RETRIES; i++) { 1854 int future_interval = i + 1; 1855 1856 /* Give the controller at least 500us to schedule transfers */ 1857 if (desc->bInterval < 3) 1858 future_interval += 3 - desc->bInterval; 1859 1860 dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval); 1861 1862 ret = __dwc3_gadget_kick_transfer(dep); 1863 if (ret != -EAGAIN) 1864 break; 1865 } 1866 1867 /* 1868 * After a number of unsuccessful start attempts due to bus-expiry 1869 * status, issue END_TRANSFER command and retry on the next XferNotReady 1870 * event. 1871 */ 1872 if (ret == -EAGAIN) 1873 ret = __dwc3_stop_active_transfer(dep, false, true); 1874 1875 return ret; 1876 } 1877 1878 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) 1879 { 1880 struct dwc3 *dwc = dep->dwc; 1881 1882 if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) { 1883 dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n", 1884 dep->name); 1885 return -ESHUTDOWN; 1886 } 1887 1888 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n", 1889 &req->request, req->dep->name)) 1890 return -EINVAL; 1891 1892 if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED, 1893 "%s: request %pK already in flight\n", 1894 dep->name, &req->request)) 1895 return -EINVAL; 1896 1897 pm_runtime_get(dwc->dev); 1898 1899 req->request.actual = 0; 1900 req->request.status = -EINPROGRESS; 1901 1902 trace_dwc3_ep_queue(req); 1903 1904 list_add_tail(&req->list, &dep->pending_list); 1905 req->status = DWC3_REQUEST_STATUS_QUEUED; 1906 1907 if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE) 1908 return 0; 1909 1910 /* 1911 * Start the transfer only after the END_TRANSFER is completed 1912 * and endpoint STALL is cleared. 1913 */ 1914 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || 1915 (dep->flags & DWC3_EP_WEDGE) || 1916 (dep->flags & DWC3_EP_DELAY_STOP) || 1917 (dep->flags & DWC3_EP_STALL)) { 1918 dep->flags |= DWC3_EP_DELAY_START; 1919 return 0; 1920 } 1921 1922 /* 1923 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must 1924 * wait for a XferNotReady event so we will know what's the current 1925 * (micro-)frame number. 1926 * 1927 * Without this trick, we are very, very likely gonna get Bus Expiry 1928 * errors which will force us issue EndTransfer command. 1929 */ 1930 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 1931 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { 1932 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) 1933 return __dwc3_gadget_start_isoc(dep); 1934 1935 return 0; 1936 } 1937 } 1938 1939 __dwc3_gadget_kick_transfer(dep); 1940 1941 return 0; 1942 } 1943 1944 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, 1945 gfp_t gfp_flags) 1946 { 1947 struct dwc3_request *req = to_dwc3_request(request); 1948 struct dwc3_ep *dep = to_dwc3_ep(ep); 1949 struct dwc3 *dwc = dep->dwc; 1950 1951 unsigned long flags; 1952 1953 int ret; 1954 1955 spin_lock_irqsave(&dwc->lock, flags); 1956 ret = __dwc3_gadget_ep_queue(dep, req); 1957 spin_unlock_irqrestore(&dwc->lock, flags); 1958 1959 return ret; 1960 } 1961 1962 static void dwc3_gadget_ep_skip_trbs(struct dwc3_ep *dep, struct dwc3_request *req) 1963 { 1964 int i; 1965 1966 /* If req->trb is not set, then the request has not started */ 1967 if (!req->trb) 1968 return; 1969 1970 /* 1971 * If request was already started, this means we had to 1972 * stop the transfer. With that we also need to ignore 1973 * all TRBs used by the request, however TRBs can only 1974 * be modified after completion of END_TRANSFER 1975 * command. So what we do here is that we wait for 1976 * END_TRANSFER completion and only after that, we jump 1977 * over TRBs by clearing HWO and incrementing dequeue 1978 * pointer. 1979 */ 1980 for (i = 0; i < req->num_trbs; i++) { 1981 struct dwc3_trb *trb; 1982 1983 trb = &dep->trb_pool[dep->trb_dequeue]; 1984 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 1985 dwc3_ep_inc_deq(dep); 1986 } 1987 1988 req->num_trbs = 0; 1989 } 1990 1991 static void dwc3_gadget_ep_cleanup_cancelled_requests(struct dwc3_ep *dep) 1992 { 1993 struct dwc3_request *req; 1994 struct dwc3 *dwc = dep->dwc; 1995 1996 while (!list_empty(&dep->cancelled_list)) { 1997 req = next_request(&dep->cancelled_list); 1998 dwc3_gadget_ep_skip_trbs(dep, req); 1999 switch (req->status) { 2000 case DWC3_REQUEST_STATUS_DISCONNECTED: 2001 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 2002 break; 2003 case DWC3_REQUEST_STATUS_DEQUEUED: 2004 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2005 break; 2006 case DWC3_REQUEST_STATUS_STALLED: 2007 dwc3_gadget_giveback(dep, req, -EPIPE); 2008 break; 2009 default: 2010 dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status); 2011 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2012 break; 2013 } 2014 /* 2015 * The endpoint is disabled, let the dwc3_remove_requests() 2016 * handle the cleanup. 2017 */ 2018 if (!dep->endpoint.desc) 2019 break; 2020 } 2021 } 2022 2023 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, 2024 struct usb_request *request) 2025 { 2026 struct dwc3_request *req = to_dwc3_request(request); 2027 struct dwc3_request *r = NULL; 2028 2029 struct dwc3_ep *dep = to_dwc3_ep(ep); 2030 struct dwc3 *dwc = dep->dwc; 2031 2032 unsigned long flags; 2033 int ret = 0; 2034 2035 trace_dwc3_ep_dequeue(req); 2036 2037 spin_lock_irqsave(&dwc->lock, flags); 2038 2039 list_for_each_entry(r, &dep->cancelled_list, list) { 2040 if (r == req) 2041 goto out; 2042 } 2043 2044 list_for_each_entry(r, &dep->pending_list, list) { 2045 if (r == req) { 2046 dwc3_gadget_giveback(dep, req, -ECONNRESET); 2047 goto out; 2048 } 2049 } 2050 2051 list_for_each_entry(r, &dep->started_list, list) { 2052 if (r == req) { 2053 struct dwc3_request *t; 2054 2055 /* wait until it is processed */ 2056 dwc3_stop_active_transfer(dep, true, true); 2057 2058 /* 2059 * Remove any started request if the transfer is 2060 * cancelled. 2061 */ 2062 list_for_each_entry_safe(r, t, &dep->started_list, list) 2063 dwc3_gadget_move_cancelled_request(r, 2064 DWC3_REQUEST_STATUS_DEQUEUED); 2065 2066 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; 2067 2068 goto out; 2069 } 2070 } 2071 2072 dev_err(dwc->dev, "request %pK was not queued to %s\n", 2073 request, ep->name); 2074 ret = -EINVAL; 2075 out: 2076 spin_unlock_irqrestore(&dwc->lock, flags); 2077 2078 return ret; 2079 } 2080 2081 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) 2082 { 2083 struct dwc3_gadget_ep_cmd_params params; 2084 struct dwc3 *dwc = dep->dwc; 2085 struct dwc3_request *req; 2086 struct dwc3_request *tmp; 2087 int ret; 2088 2089 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 2090 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); 2091 return -EINVAL; 2092 } 2093 2094 memset(¶ms, 0x00, sizeof(params)); 2095 2096 if (value) { 2097 struct dwc3_trb *trb; 2098 2099 unsigned int transfer_in_flight; 2100 unsigned int started; 2101 2102 if (dep->number > 1) 2103 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); 2104 else 2105 trb = &dwc->ep0_trb[dep->trb_enqueue]; 2106 2107 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO; 2108 started = !list_empty(&dep->started_list); 2109 2110 if (!protocol && ((dep->direction && transfer_in_flight) || 2111 (!dep->direction && started))) { 2112 return -EAGAIN; 2113 } 2114 2115 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL, 2116 ¶ms); 2117 if (ret) 2118 dev_err(dwc->dev, "failed to set STALL on %s\n", 2119 dep->name); 2120 else 2121 dep->flags |= DWC3_EP_STALL; 2122 } else { 2123 /* 2124 * Don't issue CLEAR_STALL command to control endpoints. The 2125 * controller automatically clears the STALL when it receives 2126 * the SETUP token. 2127 */ 2128 if (dep->number <= 1) { 2129 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 2130 return 0; 2131 } 2132 2133 dwc3_stop_active_transfer(dep, true, true); 2134 2135 list_for_each_entry_safe(req, tmp, &dep->started_list, list) 2136 dwc3_gadget_move_cancelled_request(req, DWC3_REQUEST_STATUS_STALLED); 2137 2138 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING || 2139 (dep->flags & DWC3_EP_DELAY_STOP)) { 2140 dep->flags |= DWC3_EP_PENDING_CLEAR_STALL; 2141 if (protocol) 2142 dwc->clear_stall_protocol = dep->number; 2143 2144 return 0; 2145 } 2146 2147 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 2148 2149 ret = dwc3_send_clear_stall_ep_cmd(dep); 2150 if (ret) { 2151 dev_err(dwc->dev, "failed to clear STALL on %s\n", 2152 dep->name); 2153 return ret; 2154 } 2155 2156 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 2157 2158 if ((dep->flags & DWC3_EP_DELAY_START) && 2159 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) 2160 __dwc3_gadget_kick_transfer(dep); 2161 2162 dep->flags &= ~DWC3_EP_DELAY_START; 2163 } 2164 2165 return ret; 2166 } 2167 2168 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value) 2169 { 2170 struct dwc3_ep *dep = to_dwc3_ep(ep); 2171 struct dwc3 *dwc = dep->dwc; 2172 2173 unsigned long flags; 2174 2175 int ret; 2176 2177 spin_lock_irqsave(&dwc->lock, flags); 2178 ret = __dwc3_gadget_ep_set_halt(dep, value, false); 2179 spin_unlock_irqrestore(&dwc->lock, flags); 2180 2181 return ret; 2182 } 2183 2184 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep) 2185 { 2186 struct dwc3_ep *dep = to_dwc3_ep(ep); 2187 struct dwc3 *dwc = dep->dwc; 2188 unsigned long flags; 2189 int ret; 2190 2191 spin_lock_irqsave(&dwc->lock, flags); 2192 dep->flags |= DWC3_EP_WEDGE; 2193 2194 if (dep->number == 0 || dep->number == 1) 2195 ret = __dwc3_gadget_ep0_set_halt(ep, 1); 2196 else 2197 ret = __dwc3_gadget_ep_set_halt(dep, 1, false); 2198 spin_unlock_irqrestore(&dwc->lock, flags); 2199 2200 return ret; 2201 } 2202 2203 /* -------------------------------------------------------------------------- */ 2204 2205 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = { 2206 .bLength = USB_DT_ENDPOINT_SIZE, 2207 .bDescriptorType = USB_DT_ENDPOINT, 2208 .bmAttributes = USB_ENDPOINT_XFER_CONTROL, 2209 }; 2210 2211 static const struct usb_ep_ops dwc3_gadget_ep0_ops = { 2212 .enable = dwc3_gadget_ep0_enable, 2213 .disable = dwc3_gadget_ep0_disable, 2214 .alloc_request = dwc3_gadget_ep_alloc_request, 2215 .free_request = dwc3_gadget_ep_free_request, 2216 .queue = dwc3_gadget_ep0_queue, 2217 .dequeue = dwc3_gadget_ep_dequeue, 2218 .set_halt = dwc3_gadget_ep0_set_halt, 2219 .set_wedge = dwc3_gadget_ep_set_wedge, 2220 }; 2221 2222 static const struct usb_ep_ops dwc3_gadget_ep_ops = { 2223 .enable = dwc3_gadget_ep_enable, 2224 .disable = dwc3_gadget_ep_disable, 2225 .alloc_request = dwc3_gadget_ep_alloc_request, 2226 .free_request = dwc3_gadget_ep_free_request, 2227 .queue = dwc3_gadget_ep_queue, 2228 .dequeue = dwc3_gadget_ep_dequeue, 2229 .set_halt = dwc3_gadget_ep_set_halt, 2230 .set_wedge = dwc3_gadget_ep_set_wedge, 2231 }; 2232 2233 /* -------------------------------------------------------------------------- */ 2234 2235 static int dwc3_gadget_get_frame(struct usb_gadget *g) 2236 { 2237 struct dwc3 *dwc = gadget_to_dwc(g); 2238 2239 return __dwc3_gadget_get_frame(dwc); 2240 } 2241 2242 static int __dwc3_gadget_wakeup(struct dwc3 *dwc) 2243 { 2244 int retries; 2245 2246 int ret; 2247 u32 reg; 2248 2249 u8 link_state; 2250 2251 /* 2252 * According to the Databook Remote wakeup request should 2253 * be issued only when the device is in early suspend state. 2254 * 2255 * We can check that via USB Link State bits in DSTS register. 2256 */ 2257 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2258 2259 link_state = DWC3_DSTS_USBLNKST(reg); 2260 2261 switch (link_state) { 2262 case DWC3_LINK_STATE_RESET: 2263 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */ 2264 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */ 2265 case DWC3_LINK_STATE_U2: /* in HS, means Sleep (L1) */ 2266 case DWC3_LINK_STATE_U1: 2267 case DWC3_LINK_STATE_RESUME: 2268 break; 2269 default: 2270 return -EINVAL; 2271 } 2272 2273 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV); 2274 if (ret < 0) { 2275 dev_err(dwc->dev, "failed to put link in Recovery\n"); 2276 return ret; 2277 } 2278 2279 /* Recent versions do this automatically */ 2280 if (DWC3_VER_IS_PRIOR(DWC3, 194A)) { 2281 /* write zeroes to Link Change Request */ 2282 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2283 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 2284 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2285 } 2286 2287 /* poll until Link State changes to ON */ 2288 retries = 20000; 2289 2290 while (retries--) { 2291 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2292 2293 /* in HS, means ON */ 2294 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0) 2295 break; 2296 } 2297 2298 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) { 2299 dev_err(dwc->dev, "failed to send remote wakeup\n"); 2300 return -EINVAL; 2301 } 2302 2303 return 0; 2304 } 2305 2306 static int dwc3_gadget_wakeup(struct usb_gadget *g) 2307 { 2308 struct dwc3 *dwc = gadget_to_dwc(g); 2309 unsigned long flags; 2310 int ret; 2311 2312 spin_lock_irqsave(&dwc->lock, flags); 2313 ret = __dwc3_gadget_wakeup(dwc); 2314 spin_unlock_irqrestore(&dwc->lock, flags); 2315 2316 return ret; 2317 } 2318 2319 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g, 2320 int is_selfpowered) 2321 { 2322 struct dwc3 *dwc = gadget_to_dwc(g); 2323 unsigned long flags; 2324 2325 spin_lock_irqsave(&dwc->lock, flags); 2326 g->is_selfpowered = !!is_selfpowered; 2327 spin_unlock_irqrestore(&dwc->lock, flags); 2328 2329 return 0; 2330 } 2331 2332 static void dwc3_stop_active_transfers(struct dwc3 *dwc) 2333 { 2334 u32 epnum; 2335 2336 for (epnum = 2; epnum < dwc->num_eps; epnum++) { 2337 struct dwc3_ep *dep; 2338 2339 dep = dwc->eps[epnum]; 2340 if (!dep) 2341 continue; 2342 2343 dwc3_remove_requests(dwc, dep); 2344 } 2345 } 2346 2347 static void __dwc3_gadget_set_ssp_rate(struct dwc3 *dwc) 2348 { 2349 enum usb_ssp_rate ssp_rate = dwc->gadget_ssp_rate; 2350 u32 reg; 2351 2352 if (ssp_rate == USB_SSP_GEN_UNKNOWN) 2353 ssp_rate = dwc->max_ssp_rate; 2354 2355 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2356 reg &= ~DWC3_DCFG_SPEED_MASK; 2357 reg &= ~DWC3_DCFG_NUMLANES(~0); 2358 2359 if (ssp_rate == USB_SSP_GEN_1x2) 2360 reg |= DWC3_DCFG_SUPERSPEED; 2361 else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2) 2362 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2363 2364 if (ssp_rate != USB_SSP_GEN_2x1 && 2365 dwc->max_ssp_rate != USB_SSP_GEN_2x1) 2366 reg |= DWC3_DCFG_NUMLANES(1); 2367 2368 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2369 } 2370 2371 static void __dwc3_gadget_set_speed(struct dwc3 *dwc) 2372 { 2373 enum usb_device_speed speed; 2374 u32 reg; 2375 2376 speed = dwc->gadget_max_speed; 2377 if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed) 2378 speed = dwc->maximum_speed; 2379 2380 if (speed == USB_SPEED_SUPER_PLUS && 2381 DWC3_IP_IS(DWC32)) { 2382 __dwc3_gadget_set_ssp_rate(dwc); 2383 return; 2384 } 2385 2386 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2387 reg &= ~(DWC3_DCFG_SPEED_MASK); 2388 2389 /* 2390 * WORKAROUND: DWC3 revision < 2.20a have an issue 2391 * which would cause metastability state on Run/Stop 2392 * bit if we try to force the IP to USB2-only mode. 2393 * 2394 * Because of that, we cannot configure the IP to any 2395 * speed other than the SuperSpeed 2396 * 2397 * Refers to: 2398 * 2399 * STAR#9000525659: Clock Domain Crossing on DCTL in 2400 * USB 2.0 Mode 2401 */ 2402 if (DWC3_VER_IS_PRIOR(DWC3, 220A) && 2403 !dwc->dis_metastability_quirk) { 2404 reg |= DWC3_DCFG_SUPERSPEED; 2405 } else { 2406 switch (speed) { 2407 case USB_SPEED_FULL: 2408 reg |= DWC3_DCFG_FULLSPEED; 2409 break; 2410 case USB_SPEED_HIGH: 2411 reg |= DWC3_DCFG_HIGHSPEED; 2412 break; 2413 case USB_SPEED_SUPER: 2414 reg |= DWC3_DCFG_SUPERSPEED; 2415 break; 2416 case USB_SPEED_SUPER_PLUS: 2417 if (DWC3_IP_IS(DWC3)) 2418 reg |= DWC3_DCFG_SUPERSPEED; 2419 else 2420 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2421 break; 2422 default: 2423 dev_err(dwc->dev, "invalid speed (%d)\n", speed); 2424 2425 if (DWC3_IP_IS(DWC3)) 2426 reg |= DWC3_DCFG_SUPERSPEED; 2427 else 2428 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2429 } 2430 } 2431 2432 if (DWC3_IP_IS(DWC32) && 2433 speed > USB_SPEED_UNKNOWN && 2434 speed < USB_SPEED_SUPER_PLUS) 2435 reg &= ~DWC3_DCFG_NUMLANES(~0); 2436 2437 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2438 } 2439 2440 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend) 2441 { 2442 u32 reg; 2443 u32 timeout = 500; 2444 2445 if (pm_runtime_suspended(dwc->dev)) 2446 return 0; 2447 2448 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2449 if (is_on) { 2450 if (DWC3_VER_IS_WITHIN(DWC3, ANY, 187A)) { 2451 reg &= ~DWC3_DCTL_TRGTULST_MASK; 2452 reg |= DWC3_DCTL_TRGTULST_RX_DET; 2453 } 2454 2455 if (!DWC3_VER_IS_PRIOR(DWC3, 194A)) 2456 reg &= ~DWC3_DCTL_KEEP_CONNECT; 2457 reg |= DWC3_DCTL_RUN_STOP; 2458 2459 if (dwc->has_hibernation) 2460 reg |= DWC3_DCTL_KEEP_CONNECT; 2461 2462 __dwc3_gadget_set_speed(dwc); 2463 dwc->pullups_connected = true; 2464 } else { 2465 reg &= ~DWC3_DCTL_RUN_STOP; 2466 2467 if (dwc->has_hibernation && !suspend) 2468 reg &= ~DWC3_DCTL_KEEP_CONNECT; 2469 2470 dwc->pullups_connected = false; 2471 } 2472 2473 dwc3_gadget_dctl_write_safe(dwc, reg); 2474 2475 do { 2476 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2477 reg &= DWC3_DSTS_DEVCTRLHLT; 2478 } while (--timeout && !(!is_on ^ !reg)); 2479 2480 if (!timeout) 2481 return -ETIMEDOUT; 2482 2483 return 0; 2484 } 2485 2486 static void dwc3_gadget_disable_irq(struct dwc3 *dwc); 2487 static void __dwc3_gadget_stop(struct dwc3 *dwc); 2488 static int __dwc3_gadget_start(struct dwc3 *dwc); 2489 2490 static int dwc3_gadget_soft_disconnect(struct dwc3 *dwc) 2491 { 2492 unsigned long flags; 2493 2494 spin_lock_irqsave(&dwc->lock, flags); 2495 dwc->connected = false; 2496 2497 /* 2498 * Per databook, when we want to stop the gadget, if a control transfer 2499 * is still in process, complete it and get the core into setup phase. 2500 */ 2501 if (dwc->ep0state != EP0_SETUP_PHASE) { 2502 int ret; 2503 2504 reinit_completion(&dwc->ep0_in_setup); 2505 2506 spin_unlock_irqrestore(&dwc->lock, flags); 2507 ret = wait_for_completion_timeout(&dwc->ep0_in_setup, 2508 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT)); 2509 spin_lock_irqsave(&dwc->lock, flags); 2510 if (ret == 0) 2511 dev_warn(dwc->dev, "timed out waiting for SETUP phase\n"); 2512 } 2513 2514 /* 2515 * In the Synopsys DesignWare Cores USB3 Databook Rev. 3.30a 2516 * Section 4.1.8 Table 4-7, it states that for a device-initiated 2517 * disconnect, the SW needs to ensure that it sends "a DEPENDXFER 2518 * command for any active transfers" before clearing the RunStop 2519 * bit. 2520 */ 2521 dwc3_stop_active_transfers(dwc); 2522 __dwc3_gadget_stop(dwc); 2523 spin_unlock_irqrestore(&dwc->lock, flags); 2524 2525 /* 2526 * Note: if the GEVNTCOUNT indicates events in the event buffer, the 2527 * driver needs to acknowledge them before the controller can halt. 2528 * Simply let the interrupt handler acknowledges and handle the 2529 * remaining event generated by the controller while polling for 2530 * DSTS.DEVCTLHLT. 2531 */ 2532 return dwc3_gadget_run_stop(dwc, false, false); 2533 } 2534 2535 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) 2536 { 2537 struct dwc3 *dwc = gadget_to_dwc(g); 2538 int ret; 2539 2540 is_on = !!is_on; 2541 2542 if (dwc->pullups_connected == is_on) 2543 return 0; 2544 2545 dwc->softconnect = is_on; 2546 2547 /* 2548 * Avoid issuing a runtime resume if the device is already in the 2549 * suspended state during gadget disconnect. DWC3 gadget was already 2550 * halted/stopped during runtime suspend. 2551 */ 2552 if (!is_on) { 2553 pm_runtime_barrier(dwc->dev); 2554 if (pm_runtime_suspended(dwc->dev)) 2555 return 0; 2556 } 2557 2558 /* 2559 * Check the return value for successful resume, or error. For a 2560 * successful resume, the DWC3 runtime PM resume routine will handle 2561 * the run stop sequence, so avoid duplicate operations here. 2562 */ 2563 ret = pm_runtime_get_sync(dwc->dev); 2564 if (!ret || ret < 0) { 2565 pm_runtime_put(dwc->dev); 2566 return 0; 2567 } 2568 2569 if (!is_on) { 2570 ret = dwc3_gadget_soft_disconnect(dwc); 2571 } else { 2572 /* 2573 * In the Synopsys DWC_usb31 1.90a programming guide section 2574 * 4.1.9, it specifies that for a reconnect after a 2575 * device-initiated disconnect requires a core soft reset 2576 * (DCTL.CSftRst) before enabling the run/stop bit. 2577 */ 2578 dwc3_core_soft_reset(dwc); 2579 2580 dwc3_event_buffers_setup(dwc); 2581 __dwc3_gadget_start(dwc); 2582 ret = dwc3_gadget_run_stop(dwc, true, false); 2583 } 2584 2585 pm_runtime_put(dwc->dev); 2586 2587 return ret; 2588 } 2589 2590 static void dwc3_gadget_enable_irq(struct dwc3 *dwc) 2591 { 2592 u32 reg; 2593 2594 /* Enable all but Start and End of Frame IRQs */ 2595 reg = (DWC3_DEVTEN_EVNTOVERFLOWEN | 2596 DWC3_DEVTEN_CMDCMPLTEN | 2597 DWC3_DEVTEN_ERRTICERREN | 2598 DWC3_DEVTEN_WKUPEVTEN | 2599 DWC3_DEVTEN_CONNECTDONEEN | 2600 DWC3_DEVTEN_USBRSTEN | 2601 DWC3_DEVTEN_DISCONNEVTEN); 2602 2603 if (DWC3_VER_IS_PRIOR(DWC3, 250A)) 2604 reg |= DWC3_DEVTEN_ULSTCNGEN; 2605 2606 /* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */ 2607 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) 2608 reg |= DWC3_DEVTEN_U3L2L1SUSPEN; 2609 2610 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); 2611 } 2612 2613 static void dwc3_gadget_disable_irq(struct dwc3 *dwc) 2614 { 2615 /* mask all interrupts */ 2616 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); 2617 } 2618 2619 static irqreturn_t dwc3_interrupt(int irq, void *_dwc); 2620 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc); 2621 2622 /** 2623 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG 2624 * @dwc: pointer to our context structure 2625 * 2626 * The following looks like complex but it's actually very simple. In order to 2627 * calculate the number of packets we can burst at once on OUT transfers, we're 2628 * gonna use RxFIFO size. 2629 * 2630 * To calculate RxFIFO size we need two numbers: 2631 * MDWIDTH = size, in bits, of the internal memory bus 2632 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits) 2633 * 2634 * Given these two numbers, the formula is simple: 2635 * 2636 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16; 2637 * 2638 * 24 bytes is for 3x SETUP packets 2639 * 16 bytes is a clock domain crossing tolerance 2640 * 2641 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024; 2642 */ 2643 static void dwc3_gadget_setup_nump(struct dwc3 *dwc) 2644 { 2645 u32 ram2_depth; 2646 u32 mdwidth; 2647 u32 nump; 2648 u32 reg; 2649 2650 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7); 2651 mdwidth = dwc3_mdwidth(dwc); 2652 2653 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024; 2654 nump = min_t(u32, nump, 16); 2655 2656 /* update NumP */ 2657 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2658 reg &= ~DWC3_DCFG_NUMP_MASK; 2659 reg |= nump << DWC3_DCFG_NUMP_SHIFT; 2660 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2661 } 2662 2663 static int __dwc3_gadget_start(struct dwc3 *dwc) 2664 { 2665 struct dwc3_ep *dep; 2666 int ret = 0; 2667 u32 reg; 2668 2669 /* 2670 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if 2671 * the core supports IMOD, disable it. 2672 */ 2673 if (dwc->imod_interval) { 2674 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 2675 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 2676 } else if (dwc3_has_imod(dwc)) { 2677 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0); 2678 } 2679 2680 /* 2681 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP 2682 * field instead of letting dwc3 itself calculate that automatically. 2683 * 2684 * This way, we maximize the chances that we'll be able to get several 2685 * bursts of data without going through any sort of endpoint throttling. 2686 */ 2687 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); 2688 if (DWC3_IP_IS(DWC3)) 2689 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL; 2690 else 2691 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL; 2692 2693 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); 2694 2695 dwc3_gadget_setup_nump(dwc); 2696 2697 /* 2698 * Currently the controller handles single stream only. So, Ignore 2699 * Packet Pending bit for stream selection and don't search for another 2700 * stream if the host sends Data Packet with PP=0 (for OUT direction) or 2701 * ACK with NumP=0 and PP=0 (for IN direction). This slightly improves 2702 * the stream performance. 2703 */ 2704 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2705 reg |= DWC3_DCFG_IGNSTRMPP; 2706 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2707 2708 /* Enable MST by default if the device is capable of MST */ 2709 if (DWC3_MST_CAPABLE(&dwc->hwparams)) { 2710 reg = dwc3_readl(dwc->regs, DWC3_DCFG1); 2711 reg &= ~DWC3_DCFG1_DIS_MST_ENH; 2712 dwc3_writel(dwc->regs, DWC3_DCFG1, reg); 2713 } 2714 2715 /* Start with SuperSpeed Default */ 2716 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 2717 2718 dep = dwc->eps[0]; 2719 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 2720 if (ret) { 2721 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2722 goto err0; 2723 } 2724 2725 dep = dwc->eps[1]; 2726 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 2727 if (ret) { 2728 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2729 goto err1; 2730 } 2731 2732 /* begin to receive SETUP packets */ 2733 dwc->ep0state = EP0_SETUP_PHASE; 2734 dwc->ep0_bounced = false; 2735 dwc->link_state = DWC3_LINK_STATE_SS_DIS; 2736 dwc->delayed_status = false; 2737 dwc3_ep0_out_start(dwc); 2738 2739 dwc3_gadget_enable_irq(dwc); 2740 2741 return 0; 2742 2743 err1: 2744 __dwc3_gadget_ep_disable(dwc->eps[0]); 2745 2746 err0: 2747 return ret; 2748 } 2749 2750 static int dwc3_gadget_start(struct usb_gadget *g, 2751 struct usb_gadget_driver *driver) 2752 { 2753 struct dwc3 *dwc = gadget_to_dwc(g); 2754 unsigned long flags; 2755 int ret; 2756 int irq; 2757 2758 irq = dwc->irq_gadget; 2759 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt, 2760 IRQF_SHARED, "dwc3", dwc->ev_buf); 2761 if (ret) { 2762 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", 2763 irq, ret); 2764 return ret; 2765 } 2766 2767 spin_lock_irqsave(&dwc->lock, flags); 2768 dwc->gadget_driver = driver; 2769 spin_unlock_irqrestore(&dwc->lock, flags); 2770 2771 return 0; 2772 } 2773 2774 static void __dwc3_gadget_stop(struct dwc3 *dwc) 2775 { 2776 dwc3_gadget_disable_irq(dwc); 2777 __dwc3_gadget_ep_disable(dwc->eps[0]); 2778 __dwc3_gadget_ep_disable(dwc->eps[1]); 2779 } 2780 2781 static int dwc3_gadget_stop(struct usb_gadget *g) 2782 { 2783 struct dwc3 *dwc = gadget_to_dwc(g); 2784 unsigned long flags; 2785 2786 spin_lock_irqsave(&dwc->lock, flags); 2787 dwc->gadget_driver = NULL; 2788 dwc->max_cfg_eps = 0; 2789 spin_unlock_irqrestore(&dwc->lock, flags); 2790 2791 free_irq(dwc->irq_gadget, dwc->ev_buf); 2792 2793 return 0; 2794 } 2795 2796 static void dwc3_gadget_config_params(struct usb_gadget *g, 2797 struct usb_dcd_config_params *params) 2798 { 2799 struct dwc3 *dwc = gadget_to_dwc(g); 2800 2801 params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED; 2802 params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED; 2803 2804 /* Recommended BESL */ 2805 if (!dwc->dis_enblslpm_quirk) { 2806 /* 2807 * If the recommended BESL baseline is 0 or if the BESL deep is 2808 * less than 2, Microsoft's Windows 10 host usb stack will issue 2809 * a usb reset immediately after it receives the extended BOS 2810 * descriptor and the enumeration will fail. To maintain 2811 * compatibility with the Windows' usb stack, let's set the 2812 * recommended BESL baseline to 1 and clamp the BESL deep to be 2813 * within 2 to 15. 2814 */ 2815 params->besl_baseline = 1; 2816 if (dwc->is_utmi_l1_suspend) 2817 params->besl_deep = 2818 clamp_t(u8, dwc->hird_threshold, 2, 15); 2819 } 2820 2821 /* U1 Device exit Latency */ 2822 if (dwc->dis_u1_entry_quirk) 2823 params->bU1devExitLat = 0; 2824 else 2825 params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT; 2826 2827 /* U2 Device exit Latency */ 2828 if (dwc->dis_u2_entry_quirk) 2829 params->bU2DevExitLat = 0; 2830 else 2831 params->bU2DevExitLat = 2832 cpu_to_le16(DWC3_DEFAULT_U2_DEV_EXIT_LAT); 2833 } 2834 2835 static void dwc3_gadget_set_speed(struct usb_gadget *g, 2836 enum usb_device_speed speed) 2837 { 2838 struct dwc3 *dwc = gadget_to_dwc(g); 2839 unsigned long flags; 2840 2841 spin_lock_irqsave(&dwc->lock, flags); 2842 dwc->gadget_max_speed = speed; 2843 spin_unlock_irqrestore(&dwc->lock, flags); 2844 } 2845 2846 static void dwc3_gadget_set_ssp_rate(struct usb_gadget *g, 2847 enum usb_ssp_rate rate) 2848 { 2849 struct dwc3 *dwc = gadget_to_dwc(g); 2850 unsigned long flags; 2851 2852 spin_lock_irqsave(&dwc->lock, flags); 2853 dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS; 2854 dwc->gadget_ssp_rate = rate; 2855 spin_unlock_irqrestore(&dwc->lock, flags); 2856 } 2857 2858 static int dwc3_gadget_vbus_draw(struct usb_gadget *g, unsigned int mA) 2859 { 2860 struct dwc3 *dwc = gadget_to_dwc(g); 2861 union power_supply_propval val = {0}; 2862 int ret; 2863 2864 if (dwc->usb2_phy) 2865 return usb_phy_set_power(dwc->usb2_phy, mA); 2866 2867 if (!dwc->usb_psy) 2868 return -EOPNOTSUPP; 2869 2870 val.intval = 1000 * mA; 2871 ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val); 2872 2873 return ret; 2874 } 2875 2876 /** 2877 * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration 2878 * @g: pointer to the USB gadget 2879 * 2880 * Used to record the maximum number of endpoints being used in a USB composite 2881 * device. (across all configurations) This is to be used in the calculation 2882 * of the TXFIFO sizes when resizing internal memory for individual endpoints. 2883 * It will help ensured that the resizing logic reserves enough space for at 2884 * least one max packet. 2885 */ 2886 static int dwc3_gadget_check_config(struct usb_gadget *g) 2887 { 2888 struct dwc3 *dwc = gadget_to_dwc(g); 2889 struct usb_ep *ep; 2890 int fifo_size = 0; 2891 int ram1_depth; 2892 int ep_num = 0; 2893 2894 if (!dwc->do_fifo_resize) 2895 return 0; 2896 2897 list_for_each_entry(ep, &g->ep_list, ep_list) { 2898 /* Only interested in the IN endpoints */ 2899 if (ep->claimed && (ep->address & USB_DIR_IN)) 2900 ep_num++; 2901 } 2902 2903 if (ep_num <= dwc->max_cfg_eps) 2904 return 0; 2905 2906 /* Update the max number of eps in the composition */ 2907 dwc->max_cfg_eps = ep_num; 2908 2909 fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps); 2910 /* Based on the equation, increment by one for every ep */ 2911 fifo_size += dwc->max_cfg_eps; 2912 2913 /* Check if we can fit a single fifo per endpoint */ 2914 ram1_depth = DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7); 2915 if (fifo_size > ram1_depth) 2916 return -ENOMEM; 2917 2918 return 0; 2919 } 2920 2921 static void dwc3_gadget_async_callbacks(struct usb_gadget *g, bool enable) 2922 { 2923 struct dwc3 *dwc = gadget_to_dwc(g); 2924 unsigned long flags; 2925 2926 spin_lock_irqsave(&dwc->lock, flags); 2927 dwc->async_callbacks = enable; 2928 spin_unlock_irqrestore(&dwc->lock, flags); 2929 } 2930 2931 static const struct usb_gadget_ops dwc3_gadget_ops = { 2932 .get_frame = dwc3_gadget_get_frame, 2933 .wakeup = dwc3_gadget_wakeup, 2934 .set_selfpowered = dwc3_gadget_set_selfpowered, 2935 .pullup = dwc3_gadget_pullup, 2936 .udc_start = dwc3_gadget_start, 2937 .udc_stop = dwc3_gadget_stop, 2938 .udc_set_speed = dwc3_gadget_set_speed, 2939 .udc_set_ssp_rate = dwc3_gadget_set_ssp_rate, 2940 .get_config_params = dwc3_gadget_config_params, 2941 .vbus_draw = dwc3_gadget_vbus_draw, 2942 .check_config = dwc3_gadget_check_config, 2943 .udc_async_callbacks = dwc3_gadget_async_callbacks, 2944 }; 2945 2946 /* -------------------------------------------------------------------------- */ 2947 2948 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep) 2949 { 2950 struct dwc3 *dwc = dep->dwc; 2951 2952 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); 2953 dep->endpoint.maxburst = 1; 2954 dep->endpoint.ops = &dwc3_gadget_ep0_ops; 2955 if (!dep->direction) 2956 dwc->gadget->ep0 = &dep->endpoint; 2957 2958 dep->endpoint.caps.type_control = true; 2959 2960 return 0; 2961 } 2962 2963 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) 2964 { 2965 struct dwc3 *dwc = dep->dwc; 2966 u32 mdwidth; 2967 int size; 2968 int maxpacket; 2969 2970 mdwidth = dwc3_mdwidth(dwc); 2971 2972 /* MDWIDTH is represented in bits, we need it in bytes */ 2973 mdwidth /= 8; 2974 2975 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); 2976 if (DWC3_IP_IS(DWC3)) 2977 size = DWC3_GTXFIFOSIZ_TXFDEP(size); 2978 else 2979 size = DWC31_GTXFIFOSIZ_TXFDEP(size); 2980 2981 /* 2982 * maxpacket size is determined as part of the following, after assuming 2983 * a mult value of one maxpacket: 2984 * DWC3 revision 280A and prior: 2985 * fifo_size = mult * (max_packet / mdwidth) + 1; 2986 * maxpacket = mdwidth * (fifo_size - 1); 2987 * 2988 * DWC3 revision 290A and onwards: 2989 * fifo_size = mult * ((max_packet + mdwidth)/mdwidth + 1) + 1 2990 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth; 2991 */ 2992 if (DWC3_VER_IS_PRIOR(DWC3, 290A)) 2993 maxpacket = mdwidth * (size - 1); 2994 else 2995 maxpacket = mdwidth * ((size - 1) - 1) - mdwidth; 2996 2997 /* Functionally, space for one max packet is sufficient */ 2998 size = min_t(int, maxpacket, 1024); 2999 usb_ep_set_maxpacket_limit(&dep->endpoint, size); 3000 3001 dep->endpoint.max_streams = 16; 3002 dep->endpoint.ops = &dwc3_gadget_ep_ops; 3003 list_add_tail(&dep->endpoint.ep_list, 3004 &dwc->gadget->ep_list); 3005 dep->endpoint.caps.type_iso = true; 3006 dep->endpoint.caps.type_bulk = true; 3007 dep->endpoint.caps.type_int = true; 3008 3009 return dwc3_alloc_trb_pool(dep); 3010 } 3011 3012 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep) 3013 { 3014 struct dwc3 *dwc = dep->dwc; 3015 u32 mdwidth; 3016 int size; 3017 3018 mdwidth = dwc3_mdwidth(dwc); 3019 3020 /* MDWIDTH is represented in bits, convert to bytes */ 3021 mdwidth /= 8; 3022 3023 /* All OUT endpoints share a single RxFIFO space */ 3024 size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0)); 3025 if (DWC3_IP_IS(DWC3)) 3026 size = DWC3_GRXFIFOSIZ_RXFDEP(size); 3027 else 3028 size = DWC31_GRXFIFOSIZ_RXFDEP(size); 3029 3030 /* FIFO depth is in MDWDITH bytes */ 3031 size *= mdwidth; 3032 3033 /* 3034 * To meet performance requirement, a minimum recommended RxFIFO size 3035 * is defined as follow: 3036 * RxFIFO size >= (3 x MaxPacketSize) + 3037 * (3 x 8 bytes setup packets size) + (16 bytes clock crossing margin) 3038 * 3039 * Then calculate the max packet limit as below. 3040 */ 3041 size -= (3 * 8) + 16; 3042 if (size < 0) 3043 size = 0; 3044 else 3045 size /= 3; 3046 3047 usb_ep_set_maxpacket_limit(&dep->endpoint, size); 3048 dep->endpoint.max_streams = 16; 3049 dep->endpoint.ops = &dwc3_gadget_ep_ops; 3050 list_add_tail(&dep->endpoint.ep_list, 3051 &dwc->gadget->ep_list); 3052 dep->endpoint.caps.type_iso = true; 3053 dep->endpoint.caps.type_bulk = true; 3054 dep->endpoint.caps.type_int = true; 3055 3056 return dwc3_alloc_trb_pool(dep); 3057 } 3058 3059 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum) 3060 { 3061 struct dwc3_ep *dep; 3062 bool direction = epnum & 1; 3063 int ret; 3064 u8 num = epnum >> 1; 3065 3066 dep = kzalloc(sizeof(*dep), GFP_KERNEL); 3067 if (!dep) 3068 return -ENOMEM; 3069 3070 dep->dwc = dwc; 3071 dep->number = epnum; 3072 dep->direction = direction; 3073 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); 3074 dwc->eps[epnum] = dep; 3075 dep->combo_num = 0; 3076 dep->start_cmd_status = 0; 3077 3078 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, 3079 direction ? "in" : "out"); 3080 3081 dep->endpoint.name = dep->name; 3082 3083 if (!(dep->number > 1)) { 3084 dep->endpoint.desc = &dwc3_gadget_ep0_desc; 3085 dep->endpoint.comp_desc = NULL; 3086 } 3087 3088 if (num == 0) 3089 ret = dwc3_gadget_init_control_endpoint(dep); 3090 else if (direction) 3091 ret = dwc3_gadget_init_in_endpoint(dep); 3092 else 3093 ret = dwc3_gadget_init_out_endpoint(dep); 3094 3095 if (ret) 3096 return ret; 3097 3098 dep->endpoint.caps.dir_in = direction; 3099 dep->endpoint.caps.dir_out = !direction; 3100 3101 INIT_LIST_HEAD(&dep->pending_list); 3102 INIT_LIST_HEAD(&dep->started_list); 3103 INIT_LIST_HEAD(&dep->cancelled_list); 3104 3105 dwc3_debugfs_create_endpoint_dir(dep); 3106 3107 return 0; 3108 } 3109 3110 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total) 3111 { 3112 u8 epnum; 3113 3114 INIT_LIST_HEAD(&dwc->gadget->ep_list); 3115 3116 for (epnum = 0; epnum < total; epnum++) { 3117 int ret; 3118 3119 ret = dwc3_gadget_init_endpoint(dwc, epnum); 3120 if (ret) 3121 return ret; 3122 } 3123 3124 return 0; 3125 } 3126 3127 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc) 3128 { 3129 struct dwc3_ep *dep; 3130 u8 epnum; 3131 3132 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 3133 dep = dwc->eps[epnum]; 3134 if (!dep) 3135 continue; 3136 /* 3137 * Physical endpoints 0 and 1 are special; they form the 3138 * bi-directional USB endpoint 0. 3139 * 3140 * For those two physical endpoints, we don't allocate a TRB 3141 * pool nor do we add them the endpoints list. Due to that, we 3142 * shouldn't do these two operations otherwise we would end up 3143 * with all sorts of bugs when removing dwc3.ko. 3144 */ 3145 if (epnum != 0 && epnum != 1) { 3146 dwc3_free_trb_pool(dep); 3147 list_del(&dep->endpoint.ep_list); 3148 } 3149 3150 debugfs_remove_recursive(debugfs_lookup(dep->name, 3151 debugfs_lookup(dev_name(dep->dwc->dev), 3152 usb_debug_root))); 3153 kfree(dep); 3154 } 3155 } 3156 3157 /* -------------------------------------------------------------------------- */ 3158 3159 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep, 3160 struct dwc3_request *req, struct dwc3_trb *trb, 3161 const struct dwc3_event_depevt *event, int status, int chain) 3162 { 3163 unsigned int count; 3164 3165 dwc3_ep_inc_deq(dep); 3166 3167 trace_dwc3_complete_trb(dep, trb); 3168 req->num_trbs--; 3169 3170 /* 3171 * If we're in the middle of series of chained TRBs and we 3172 * receive a short transfer along the way, DWC3 will skip 3173 * through all TRBs including the last TRB in the chain (the 3174 * where CHN bit is zero. DWC3 will also avoid clearing HWO 3175 * bit and SW has to do it manually. 3176 * 3177 * We're going to do that here to avoid problems of HW trying 3178 * to use bogus TRBs for transfers. 3179 */ 3180 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO)) 3181 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 3182 3183 /* 3184 * For isochronous transfers, the first TRB in a service interval must 3185 * have the Isoc-First type. Track and report its interval frame number. 3186 */ 3187 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 3188 (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) { 3189 unsigned int frame_number; 3190 3191 frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl); 3192 frame_number &= ~(dep->interval - 1); 3193 req->request.frame_number = frame_number; 3194 } 3195 3196 /* 3197 * We use bounce buffer for requests that needs extra TRB or OUT ZLP. If 3198 * this TRB points to the bounce buffer address, it's a MPS alignment 3199 * TRB. Don't add it to req->remaining calculation. 3200 */ 3201 if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) && 3202 trb->bph == upper_32_bits(dep->dwc->bounce_addr)) { 3203 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 3204 return 1; 3205 } 3206 3207 count = trb->size & DWC3_TRB_SIZE_MASK; 3208 req->remaining += count; 3209 3210 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) 3211 return 1; 3212 3213 if (event->status & DEPEVT_STATUS_SHORT && !chain) 3214 return 1; 3215 3216 if ((trb->ctrl & DWC3_TRB_CTRL_IOC) || 3217 (trb->ctrl & DWC3_TRB_CTRL_LST)) 3218 return 1; 3219 3220 return 0; 3221 } 3222 3223 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep, 3224 struct dwc3_request *req, const struct dwc3_event_depevt *event, 3225 int status) 3226 { 3227 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; 3228 struct scatterlist *sg = req->sg; 3229 struct scatterlist *s; 3230 unsigned int num_queued = req->num_queued_sgs; 3231 unsigned int i; 3232 int ret = 0; 3233 3234 for_each_sg(sg, s, num_queued, i) { 3235 trb = &dep->trb_pool[dep->trb_dequeue]; 3236 3237 req->sg = sg_next(s); 3238 req->num_queued_sgs--; 3239 3240 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req, 3241 trb, event, status, true); 3242 if (ret) 3243 break; 3244 } 3245 3246 return ret; 3247 } 3248 3249 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep, 3250 struct dwc3_request *req, const struct dwc3_event_depevt *event, 3251 int status) 3252 { 3253 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; 3254 3255 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb, 3256 event, status, false); 3257 } 3258 3259 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req) 3260 { 3261 return req->num_pending_sgs == 0 && req->num_queued_sgs == 0; 3262 } 3263 3264 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep, 3265 const struct dwc3_event_depevt *event, 3266 struct dwc3_request *req, int status) 3267 { 3268 int request_status; 3269 int ret; 3270 3271 if (req->request.num_mapped_sgs) 3272 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event, 3273 status); 3274 else 3275 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, 3276 status); 3277 3278 req->request.actual = req->request.length - req->remaining; 3279 3280 if (!dwc3_gadget_ep_request_completed(req)) 3281 goto out; 3282 3283 if (req->needs_extra_trb) { 3284 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, 3285 status); 3286 req->needs_extra_trb = false; 3287 } 3288 3289 /* 3290 * The event status only reflects the status of the TRB with IOC set. 3291 * For the requests that don't set interrupt on completion, the driver 3292 * needs to check and return the status of the completed TRBs associated 3293 * with the request. Use the status of the last TRB of the request. 3294 */ 3295 if (req->request.no_interrupt) { 3296 struct dwc3_trb *trb; 3297 3298 trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue); 3299 switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) { 3300 case DWC3_TRBSTS_MISSED_ISOC: 3301 /* Isoc endpoint only */ 3302 request_status = -EXDEV; 3303 break; 3304 case DWC3_TRB_STS_XFER_IN_PROG: 3305 /* Applicable when End Transfer with ForceRM=0 */ 3306 case DWC3_TRBSTS_SETUP_PENDING: 3307 /* Control endpoint only */ 3308 case DWC3_TRBSTS_OK: 3309 default: 3310 request_status = 0; 3311 break; 3312 } 3313 } else { 3314 request_status = status; 3315 } 3316 3317 dwc3_gadget_giveback(dep, req, request_status); 3318 3319 out: 3320 return ret; 3321 } 3322 3323 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, 3324 const struct dwc3_event_depevt *event, int status) 3325 { 3326 struct dwc3_request *req; 3327 3328 while (!list_empty(&dep->started_list)) { 3329 int ret; 3330 3331 req = next_request(&dep->started_list); 3332 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, 3333 req, status); 3334 if (ret) 3335 break; 3336 /* 3337 * The endpoint is disabled, let the dwc3_remove_requests() 3338 * handle the cleanup. 3339 */ 3340 if (!dep->endpoint.desc) 3341 break; 3342 } 3343 } 3344 3345 static bool dwc3_gadget_ep_should_continue(struct dwc3_ep *dep) 3346 { 3347 struct dwc3_request *req; 3348 struct dwc3 *dwc = dep->dwc; 3349 3350 if (!dep->endpoint.desc || !dwc->pullups_connected || 3351 !dwc->connected) 3352 return false; 3353 3354 if (!list_empty(&dep->pending_list)) 3355 return true; 3356 3357 /* 3358 * We only need to check the first entry of the started list. We can 3359 * assume the completed requests are removed from the started list. 3360 */ 3361 req = next_request(&dep->started_list); 3362 if (!req) 3363 return false; 3364 3365 return !dwc3_gadget_ep_request_completed(req); 3366 } 3367 3368 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep, 3369 const struct dwc3_event_depevt *event) 3370 { 3371 dep->frame_number = event->parameters; 3372 } 3373 3374 static bool dwc3_gadget_endpoint_trbs_complete(struct dwc3_ep *dep, 3375 const struct dwc3_event_depevt *event, int status) 3376 { 3377 struct dwc3 *dwc = dep->dwc; 3378 bool no_started_trb = true; 3379 3380 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status); 3381 3382 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) 3383 goto out; 3384 3385 if (!dep->endpoint.desc) 3386 return no_started_trb; 3387 3388 if (usb_endpoint_xfer_isoc(dep->endpoint.desc) && 3389 list_empty(&dep->started_list) && 3390 (list_empty(&dep->pending_list) || status == -EXDEV)) 3391 dwc3_stop_active_transfer(dep, true, true); 3392 else if (dwc3_gadget_ep_should_continue(dep)) 3393 if (__dwc3_gadget_kick_transfer(dep) == 0) 3394 no_started_trb = false; 3395 3396 out: 3397 /* 3398 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. 3399 * See dwc3_gadget_linksts_change_interrupt() for 1st half. 3400 */ 3401 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) { 3402 u32 reg; 3403 int i; 3404 3405 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 3406 dep = dwc->eps[i]; 3407 3408 if (!(dep->flags & DWC3_EP_ENABLED)) 3409 continue; 3410 3411 if (!list_empty(&dep->started_list)) 3412 return no_started_trb; 3413 } 3414 3415 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3416 reg |= dwc->u1u2; 3417 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 3418 3419 dwc->u1u2 = 0; 3420 } 3421 3422 return no_started_trb; 3423 } 3424 3425 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep, 3426 const struct dwc3_event_depevt *event) 3427 { 3428 int status = 0; 3429 3430 if (!dep->endpoint.desc) 3431 return; 3432 3433 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) 3434 dwc3_gadget_endpoint_frame_from_event(dep, event); 3435 3436 if (event->status & DEPEVT_STATUS_BUSERR) 3437 status = -ECONNRESET; 3438 3439 if (event->status & DEPEVT_STATUS_MISSED_ISOC) 3440 status = -EXDEV; 3441 3442 dwc3_gadget_endpoint_trbs_complete(dep, event, status); 3443 } 3444 3445 static void dwc3_gadget_endpoint_transfer_complete(struct dwc3_ep *dep, 3446 const struct dwc3_event_depevt *event) 3447 { 3448 int status = 0; 3449 3450 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 3451 3452 if (event->status & DEPEVT_STATUS_BUSERR) 3453 status = -ECONNRESET; 3454 3455 if (dwc3_gadget_endpoint_trbs_complete(dep, event, status)) 3456 dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE; 3457 } 3458 3459 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep, 3460 const struct dwc3_event_depevt *event) 3461 { 3462 dwc3_gadget_endpoint_frame_from_event(dep, event); 3463 3464 /* 3465 * The XferNotReady event is generated only once before the endpoint 3466 * starts. It will be generated again when END_TRANSFER command is 3467 * issued. For some controller versions, the XferNotReady event may be 3468 * generated while the END_TRANSFER command is still in process. Ignore 3469 * it and wait for the next XferNotReady event after the command is 3470 * completed. 3471 */ 3472 if (dep->flags & DWC3_EP_END_TRANSFER_PENDING) 3473 return; 3474 3475 (void) __dwc3_gadget_start_isoc(dep); 3476 } 3477 3478 static void dwc3_gadget_endpoint_command_complete(struct dwc3_ep *dep, 3479 const struct dwc3_event_depevt *event) 3480 { 3481 u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters); 3482 3483 if (cmd != DWC3_DEPCMD_ENDTRANSFER) 3484 return; 3485 3486 /* 3487 * The END_TRANSFER command will cause the controller to generate a 3488 * NoStream Event, and it's not due to the host DP NoStream rejection. 3489 * Ignore the next NoStream event. 3490 */ 3491 if (dep->stream_capable) 3492 dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM; 3493 3494 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; 3495 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 3496 dwc3_gadget_ep_cleanup_cancelled_requests(dep); 3497 3498 if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) { 3499 struct dwc3 *dwc = dep->dwc; 3500 3501 dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL; 3502 if (dwc3_send_clear_stall_ep_cmd(dep)) { 3503 struct usb_ep *ep0 = &dwc->eps[0]->endpoint; 3504 3505 dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name); 3506 if (dwc->delayed_status) 3507 __dwc3_gadget_ep0_set_halt(ep0, 1); 3508 return; 3509 } 3510 3511 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 3512 if (dwc->clear_stall_protocol == dep->number) 3513 dwc3_ep0_send_delayed_status(dwc); 3514 } 3515 3516 if ((dep->flags & DWC3_EP_DELAY_START) && 3517 !usb_endpoint_xfer_isoc(dep->endpoint.desc)) 3518 __dwc3_gadget_kick_transfer(dep); 3519 3520 dep->flags &= ~DWC3_EP_DELAY_START; 3521 } 3522 3523 static void dwc3_gadget_endpoint_stream_event(struct dwc3_ep *dep, 3524 const struct dwc3_event_depevt *event) 3525 { 3526 struct dwc3 *dwc = dep->dwc; 3527 3528 if (event->status == DEPEVT_STREAMEVT_FOUND) { 3529 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; 3530 goto out; 3531 } 3532 3533 /* Note: NoStream rejection event param value is 0 and not 0xFFFF */ 3534 switch (event->parameters) { 3535 case DEPEVT_STREAM_PRIME: 3536 /* 3537 * If the host can properly transition the endpoint state from 3538 * idle to prime after a NoStream rejection, there's no need to 3539 * force restarting the endpoint to reinitiate the stream. To 3540 * simplify the check, assume the host follows the USB spec if 3541 * it primed the endpoint more than once. 3542 */ 3543 if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) { 3544 if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED) 3545 dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM; 3546 else 3547 dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED; 3548 } 3549 3550 break; 3551 case DEPEVT_STREAM_NOSTREAM: 3552 if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) || 3553 !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) || 3554 (!DWC3_MST_CAPABLE(&dwc->hwparams) && 3555 !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE))) 3556 break; 3557 3558 /* 3559 * If the host rejects a stream due to no active stream, by the 3560 * USB and xHCI spec, the endpoint will be put back to idle 3561 * state. When the host is ready (buffer added/updated), it will 3562 * prime the endpoint to inform the usb device controller. This 3563 * triggers the device controller to issue ERDY to restart the 3564 * stream. However, some hosts don't follow this and keep the 3565 * endpoint in the idle state. No prime will come despite host 3566 * streams are updated, and the device controller will not be 3567 * triggered to generate ERDY to move the next stream data. To 3568 * workaround this and maintain compatibility with various 3569 * hosts, force to reinitate the stream until the host is ready 3570 * instead of waiting for the host to prime the endpoint. 3571 */ 3572 if (DWC3_VER_IS_WITHIN(DWC32, 100A, ANY)) { 3573 unsigned int cmd = DWC3_DGCMD_SET_ENDPOINT_PRIME; 3574 3575 dwc3_send_gadget_generic_command(dwc, cmd, dep->number); 3576 } else { 3577 dep->flags |= DWC3_EP_DELAY_START; 3578 dwc3_stop_active_transfer(dep, true, true); 3579 return; 3580 } 3581 break; 3582 } 3583 3584 out: 3585 dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM; 3586 } 3587 3588 static void dwc3_endpoint_interrupt(struct dwc3 *dwc, 3589 const struct dwc3_event_depevt *event) 3590 { 3591 struct dwc3_ep *dep; 3592 u8 epnum = event->endpoint_number; 3593 3594 dep = dwc->eps[epnum]; 3595 3596 if (!(dep->flags & DWC3_EP_ENABLED)) { 3597 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) 3598 return; 3599 3600 /* Handle only EPCMDCMPLT when EP disabled */ 3601 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) 3602 return; 3603 } 3604 3605 if (epnum == 0 || epnum == 1) { 3606 dwc3_ep0_interrupt(dwc, event); 3607 return; 3608 } 3609 3610 switch (event->endpoint_event) { 3611 case DWC3_DEPEVT_XFERINPROGRESS: 3612 dwc3_gadget_endpoint_transfer_in_progress(dep, event); 3613 break; 3614 case DWC3_DEPEVT_XFERNOTREADY: 3615 dwc3_gadget_endpoint_transfer_not_ready(dep, event); 3616 break; 3617 case DWC3_DEPEVT_EPCMDCMPLT: 3618 dwc3_gadget_endpoint_command_complete(dep, event); 3619 break; 3620 case DWC3_DEPEVT_XFERCOMPLETE: 3621 dwc3_gadget_endpoint_transfer_complete(dep, event); 3622 break; 3623 case DWC3_DEPEVT_STREAMEVT: 3624 dwc3_gadget_endpoint_stream_event(dep, event); 3625 break; 3626 case DWC3_DEPEVT_RXTXFIFOEVT: 3627 break; 3628 } 3629 } 3630 3631 static void dwc3_disconnect_gadget(struct dwc3 *dwc) 3632 { 3633 if (dwc->async_callbacks && dwc->gadget_driver->disconnect) { 3634 spin_unlock(&dwc->lock); 3635 dwc->gadget_driver->disconnect(dwc->gadget); 3636 spin_lock(&dwc->lock); 3637 } 3638 } 3639 3640 static void dwc3_suspend_gadget(struct dwc3 *dwc) 3641 { 3642 if (dwc->async_callbacks && dwc->gadget_driver->suspend) { 3643 spin_unlock(&dwc->lock); 3644 dwc->gadget_driver->suspend(dwc->gadget); 3645 spin_lock(&dwc->lock); 3646 } 3647 } 3648 3649 static void dwc3_resume_gadget(struct dwc3 *dwc) 3650 { 3651 if (dwc->async_callbacks && dwc->gadget_driver->resume) { 3652 spin_unlock(&dwc->lock); 3653 dwc->gadget_driver->resume(dwc->gadget); 3654 spin_lock(&dwc->lock); 3655 } 3656 } 3657 3658 static void dwc3_reset_gadget(struct dwc3 *dwc) 3659 { 3660 if (!dwc->gadget_driver) 3661 return; 3662 3663 if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) { 3664 spin_unlock(&dwc->lock); 3665 usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver); 3666 spin_lock(&dwc->lock); 3667 } 3668 } 3669 3670 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force, 3671 bool interrupt) 3672 { 3673 struct dwc3 *dwc = dep->dwc; 3674 3675 /* 3676 * Only issue End Transfer command to the control endpoint of a started 3677 * Data Phase. Typically we should only do so in error cases such as 3678 * invalid/unexpected direction as described in the control transfer 3679 * flow of the programming guide. 3680 */ 3681 if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE) 3682 return; 3683 3684 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) || 3685 (dep->flags & DWC3_EP_DELAY_STOP) || 3686 (dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 3687 return; 3688 3689 /* 3690 * If a Setup packet is received but yet to DMA out, the controller will 3691 * not process the End Transfer command of any endpoint. Polling of its 3692 * DEPCMD.CmdAct may block setting up TRB for Setup packet, causing a 3693 * timeout. Delay issuing the End Transfer command until the Setup TRB is 3694 * prepared. 3695 */ 3696 if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) { 3697 dep->flags |= DWC3_EP_DELAY_STOP; 3698 return; 3699 } 3700 3701 /* 3702 * NOTICE: We are violating what the Databook says about the 3703 * EndTransfer command. Ideally we would _always_ wait for the 3704 * EndTransfer Command Completion IRQ, but that's causing too 3705 * much trouble synchronizing between us and gadget driver. 3706 * 3707 * We have discussed this with the IP Provider and it was 3708 * suggested to giveback all requests here. 3709 * 3710 * Note also that a similar handling was tested by Synopsys 3711 * (thanks a lot Paul) and nothing bad has come out of it. 3712 * In short, what we're doing is issuing EndTransfer with 3713 * CMDIOC bit set and delay kicking transfer until the 3714 * EndTransfer command had completed. 3715 * 3716 * As of IP version 3.10a of the DWC_usb3 IP, the controller 3717 * supports a mode to work around the above limitation. The 3718 * software can poll the CMDACT bit in the DEPCMD register 3719 * after issuing a EndTransfer command. This mode is enabled 3720 * by writing GUCTL2[14]. This polling is already done in the 3721 * dwc3_send_gadget_ep_cmd() function so if the mode is 3722 * enabled, the EndTransfer command will have completed upon 3723 * returning from this function. 3724 * 3725 * This mode is NOT available on the DWC_usb31 IP. 3726 */ 3727 3728 __dwc3_stop_active_transfer(dep, force, interrupt); 3729 } 3730 3731 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc) 3732 { 3733 u32 epnum; 3734 3735 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 3736 struct dwc3_ep *dep; 3737 int ret; 3738 3739 dep = dwc->eps[epnum]; 3740 if (!dep) 3741 continue; 3742 3743 if (!(dep->flags & DWC3_EP_STALL)) 3744 continue; 3745 3746 dep->flags &= ~DWC3_EP_STALL; 3747 3748 ret = dwc3_send_clear_stall_ep_cmd(dep); 3749 WARN_ON_ONCE(ret); 3750 } 3751 } 3752 3753 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc) 3754 { 3755 int reg; 3756 3757 dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RX_DET); 3758 3759 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3760 reg &= ~DWC3_DCTL_INITU1ENA; 3761 reg &= ~DWC3_DCTL_INITU2ENA; 3762 dwc3_gadget_dctl_write_safe(dwc, reg); 3763 3764 dwc3_disconnect_gadget(dwc); 3765 3766 dwc->gadget->speed = USB_SPEED_UNKNOWN; 3767 dwc->setup_packet_pending = false; 3768 usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED); 3769 3770 dwc->connected = false; 3771 } 3772 3773 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) 3774 { 3775 u32 reg; 3776 3777 /* 3778 * Ideally, dwc3_reset_gadget() would trigger the function 3779 * drivers to stop any active transfers through ep disable. 3780 * However, for functions which defer ep disable, such as mass 3781 * storage, we will need to rely on the call to stop active 3782 * transfers here, and avoid allowing of request queuing. 3783 */ 3784 dwc->connected = false; 3785 3786 /* 3787 * WORKAROUND: DWC3 revisions <1.88a have an issue which 3788 * would cause a missing Disconnect Event if there's a 3789 * pending Setup Packet in the FIFO. 3790 * 3791 * There's no suggested workaround on the official Bug 3792 * report, which states that "unless the driver/application 3793 * is doing any special handling of a disconnect event, 3794 * there is no functional issue". 3795 * 3796 * Unfortunately, it turns out that we _do_ some special 3797 * handling of a disconnect event, namely complete all 3798 * pending transfers, notify gadget driver of the 3799 * disconnection, and so on. 3800 * 3801 * Our suggested workaround is to follow the Disconnect 3802 * Event steps here, instead, based on a setup_packet_pending 3803 * flag. Such flag gets set whenever we have a SETUP_PENDING 3804 * status for EP0 TRBs and gets cleared on XferComplete for the 3805 * same endpoint. 3806 * 3807 * Refers to: 3808 * 3809 * STAR#9000466709: RTL: Device : Disconnect event not 3810 * generated if setup packet pending in FIFO 3811 */ 3812 if (DWC3_VER_IS_PRIOR(DWC3, 188A)) { 3813 if (dwc->setup_packet_pending) 3814 dwc3_gadget_disconnect_interrupt(dwc); 3815 } 3816 3817 dwc3_reset_gadget(dwc); 3818 3819 /* 3820 * From SNPS databook section 8.1.2, the EP0 should be in setup 3821 * phase. So ensure that EP0 is in setup phase by issuing a stall 3822 * and restart if EP0 is not in setup phase. 3823 */ 3824 if (dwc->ep0state != EP0_SETUP_PHASE) { 3825 unsigned int dir; 3826 3827 dir = !!dwc->ep0_expect_in; 3828 if (dwc->ep0state == EP0_DATA_PHASE) 3829 dwc3_ep0_end_control_data(dwc, dwc->eps[dir]); 3830 else 3831 dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]); 3832 3833 dwc->eps[0]->trb_enqueue = 0; 3834 dwc->eps[1]->trb_enqueue = 0; 3835 3836 dwc3_ep0_stall_and_restart(dwc); 3837 } 3838 3839 /* 3840 * In the Synopsis DesignWare Cores USB3 Databook Rev. 3.30a 3841 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW 3842 * needs to ensure that it sends "a DEPENDXFER command for any active 3843 * transfers." 3844 */ 3845 dwc3_stop_active_transfers(dwc); 3846 dwc->connected = true; 3847 3848 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3849 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 3850 dwc3_gadget_dctl_write_safe(dwc, reg); 3851 dwc->test_mode = false; 3852 dwc3_clear_stall_all_ep(dwc); 3853 3854 /* Reset device address to zero */ 3855 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 3856 reg &= ~(DWC3_DCFG_DEVADDR_MASK); 3857 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 3858 } 3859 3860 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) 3861 { 3862 struct dwc3_ep *dep; 3863 int ret; 3864 u32 reg; 3865 u8 lanes = 1; 3866 u8 speed; 3867 3868 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 3869 speed = reg & DWC3_DSTS_CONNECTSPD; 3870 dwc->speed = speed; 3871 3872 if (DWC3_IP_IS(DWC32)) 3873 lanes = DWC3_DSTS_CONNLANES(reg) + 1; 3874 3875 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; 3876 3877 /* 3878 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed 3879 * each time on Connect Done. 3880 * 3881 * Currently we always use the reset value. If any platform 3882 * wants to set this to a different value, we need to add a 3883 * setting and update GCTL.RAMCLKSEL here. 3884 */ 3885 3886 switch (speed) { 3887 case DWC3_DSTS_SUPERSPEED_PLUS: 3888 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 3889 dwc->gadget->ep0->maxpacket = 512; 3890 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; 3891 3892 if (lanes > 1) 3893 dwc->gadget->ssp_rate = USB_SSP_GEN_2x2; 3894 else 3895 dwc->gadget->ssp_rate = USB_SSP_GEN_2x1; 3896 break; 3897 case DWC3_DSTS_SUPERSPEED: 3898 /* 3899 * WORKAROUND: DWC3 revisions <1.90a have an issue which 3900 * would cause a missing USB3 Reset event. 3901 * 3902 * In such situations, we should force a USB3 Reset 3903 * event by calling our dwc3_gadget_reset_interrupt() 3904 * routine. 3905 * 3906 * Refers to: 3907 * 3908 * STAR#9000483510: RTL: SS : USB3 reset event may 3909 * not be generated always when the link enters poll 3910 */ 3911 if (DWC3_VER_IS_PRIOR(DWC3, 190A)) 3912 dwc3_gadget_reset_interrupt(dwc); 3913 3914 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 3915 dwc->gadget->ep0->maxpacket = 512; 3916 dwc->gadget->speed = USB_SPEED_SUPER; 3917 3918 if (lanes > 1) { 3919 dwc->gadget->speed = USB_SPEED_SUPER_PLUS; 3920 dwc->gadget->ssp_rate = USB_SSP_GEN_1x2; 3921 } 3922 break; 3923 case DWC3_DSTS_HIGHSPEED: 3924 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 3925 dwc->gadget->ep0->maxpacket = 64; 3926 dwc->gadget->speed = USB_SPEED_HIGH; 3927 break; 3928 case DWC3_DSTS_FULLSPEED: 3929 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 3930 dwc->gadget->ep0->maxpacket = 64; 3931 dwc->gadget->speed = USB_SPEED_FULL; 3932 break; 3933 } 3934 3935 dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket; 3936 3937 /* Enable USB2 LPM Capability */ 3938 3939 if (!DWC3_VER_IS_WITHIN(DWC3, ANY, 194A) && 3940 !dwc->usb2_gadget_lpm_disable && 3941 (speed != DWC3_DSTS_SUPERSPEED) && 3942 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) { 3943 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 3944 reg |= DWC3_DCFG_LPM_CAP; 3945 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 3946 3947 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3948 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN); 3949 3950 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold | 3951 (dwc->is_utmi_l1_suspend << 4)); 3952 3953 /* 3954 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and 3955 * DCFG.LPMCap is set, core responses with an ACK and the 3956 * BESL value in the LPM token is less than or equal to LPM 3957 * NYET threshold. 3958 */ 3959 WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum, 3960 "LPM Erratum not available on dwc3 revisions < 2.40a\n"); 3961 3962 if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) 3963 reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold); 3964 3965 dwc3_gadget_dctl_write_safe(dwc, reg); 3966 } else { 3967 if (dwc->usb2_gadget_lpm_disable) { 3968 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 3969 reg &= ~DWC3_DCFG_LPM_CAP; 3970 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 3971 } 3972 3973 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 3974 reg &= ~DWC3_DCTL_HIRD_THRES_MASK; 3975 dwc3_gadget_dctl_write_safe(dwc, reg); 3976 } 3977 3978 dep = dwc->eps[0]; 3979 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 3980 if (ret) { 3981 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 3982 return; 3983 } 3984 3985 dep = dwc->eps[1]; 3986 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 3987 if (ret) { 3988 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 3989 return; 3990 } 3991 3992 /* 3993 * Configure PHY via GUSB3PIPECTLn if required. 3994 * 3995 * Update GTXFIFOSIZn 3996 * 3997 * In both cases reset values should be sufficient. 3998 */ 3999 } 4000 4001 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc) 4002 { 4003 /* 4004 * TODO take core out of low power mode when that's 4005 * implemented. 4006 */ 4007 4008 if (dwc->async_callbacks && dwc->gadget_driver->resume) { 4009 spin_unlock(&dwc->lock); 4010 dwc->gadget_driver->resume(dwc->gadget); 4011 spin_lock(&dwc->lock); 4012 } 4013 } 4014 4015 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc, 4016 unsigned int evtinfo) 4017 { 4018 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 4019 unsigned int pwropt; 4020 4021 /* 4022 * WORKAROUND: DWC3 < 2.50a have an issue when configured without 4023 * Hibernation mode enabled which would show up when device detects 4024 * host-initiated U3 exit. 4025 * 4026 * In that case, device will generate a Link State Change Interrupt 4027 * from U3 to RESUME which is only necessary if Hibernation is 4028 * configured in. 4029 * 4030 * There are no functional changes due to such spurious event and we 4031 * just need to ignore it. 4032 * 4033 * Refers to: 4034 * 4035 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation 4036 * operational mode 4037 */ 4038 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); 4039 if (DWC3_VER_IS_PRIOR(DWC3, 250A) && 4040 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) { 4041 if ((dwc->link_state == DWC3_LINK_STATE_U3) && 4042 (next == DWC3_LINK_STATE_RESUME)) { 4043 return; 4044 } 4045 } 4046 4047 /* 4048 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending 4049 * on the link partner, the USB session might do multiple entry/exit 4050 * of low power states before a transfer takes place. 4051 * 4052 * Due to this problem, we might experience lower throughput. The 4053 * suggested workaround is to disable DCTL[12:9] bits if we're 4054 * transitioning from U1/U2 to U0 and enable those bits again 4055 * after a transfer completes and there are no pending transfers 4056 * on any of the enabled endpoints. 4057 * 4058 * This is the first half of that workaround. 4059 * 4060 * Refers to: 4061 * 4062 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us 4063 * core send LGO_Ux entering U0 4064 */ 4065 if (DWC3_VER_IS_PRIOR(DWC3, 183A)) { 4066 if (next == DWC3_LINK_STATE_U0) { 4067 u32 u1u2; 4068 u32 reg; 4069 4070 switch (dwc->link_state) { 4071 case DWC3_LINK_STATE_U1: 4072 case DWC3_LINK_STATE_U2: 4073 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 4074 u1u2 = reg & (DWC3_DCTL_INITU2ENA 4075 | DWC3_DCTL_ACCEPTU2ENA 4076 | DWC3_DCTL_INITU1ENA 4077 | DWC3_DCTL_ACCEPTU1ENA); 4078 4079 if (!dwc->u1u2) 4080 dwc->u1u2 = reg & u1u2; 4081 4082 reg &= ~u1u2; 4083 4084 dwc3_gadget_dctl_write_safe(dwc, reg); 4085 break; 4086 default: 4087 /* do nothing */ 4088 break; 4089 } 4090 } 4091 } 4092 4093 switch (next) { 4094 case DWC3_LINK_STATE_U1: 4095 if (dwc->speed == USB_SPEED_SUPER) 4096 dwc3_suspend_gadget(dwc); 4097 break; 4098 case DWC3_LINK_STATE_U2: 4099 case DWC3_LINK_STATE_U3: 4100 dwc3_suspend_gadget(dwc); 4101 break; 4102 case DWC3_LINK_STATE_RESUME: 4103 dwc3_resume_gadget(dwc); 4104 break; 4105 default: 4106 /* do nothing */ 4107 break; 4108 } 4109 4110 dwc->link_state = next; 4111 } 4112 4113 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc, 4114 unsigned int evtinfo) 4115 { 4116 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 4117 4118 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3) 4119 dwc3_suspend_gadget(dwc); 4120 4121 dwc->link_state = next; 4122 } 4123 4124 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc, 4125 unsigned int evtinfo) 4126 { 4127 unsigned int is_ss = evtinfo & BIT(4); 4128 4129 /* 4130 * WORKAROUND: DWC3 revison 2.20a with hibernation support 4131 * have a known issue which can cause USB CV TD.9.23 to fail 4132 * randomly. 4133 * 4134 * Because of this issue, core could generate bogus hibernation 4135 * events which SW needs to ignore. 4136 * 4137 * Refers to: 4138 * 4139 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0 4140 * Device Fallback from SuperSpeed 4141 */ 4142 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER)) 4143 return; 4144 4145 /* enter hibernation here */ 4146 } 4147 4148 static void dwc3_gadget_interrupt(struct dwc3 *dwc, 4149 const struct dwc3_event_devt *event) 4150 { 4151 switch (event->type) { 4152 case DWC3_DEVICE_EVENT_DISCONNECT: 4153 dwc3_gadget_disconnect_interrupt(dwc); 4154 break; 4155 case DWC3_DEVICE_EVENT_RESET: 4156 dwc3_gadget_reset_interrupt(dwc); 4157 break; 4158 case DWC3_DEVICE_EVENT_CONNECT_DONE: 4159 dwc3_gadget_conndone_interrupt(dwc); 4160 break; 4161 case DWC3_DEVICE_EVENT_WAKEUP: 4162 dwc3_gadget_wakeup_interrupt(dwc); 4163 break; 4164 case DWC3_DEVICE_EVENT_HIBER_REQ: 4165 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation, 4166 "unexpected hibernation event\n")) 4167 break; 4168 4169 dwc3_gadget_hibernation_interrupt(dwc, event->event_info); 4170 break; 4171 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE: 4172 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); 4173 break; 4174 case DWC3_DEVICE_EVENT_SUSPEND: 4175 /* It changed to be suspend event for version 2.30a and above */ 4176 if (!DWC3_VER_IS_PRIOR(DWC3, 230A)) { 4177 /* 4178 * Ignore suspend event until the gadget enters into 4179 * USB_STATE_CONFIGURED state. 4180 */ 4181 if (dwc->gadget->state >= USB_STATE_CONFIGURED) 4182 dwc3_gadget_suspend_interrupt(dwc, 4183 event->event_info); 4184 } 4185 break; 4186 case DWC3_DEVICE_EVENT_SOF: 4187 case DWC3_DEVICE_EVENT_ERRATIC_ERROR: 4188 case DWC3_DEVICE_EVENT_CMD_CMPL: 4189 case DWC3_DEVICE_EVENT_OVERFLOW: 4190 break; 4191 default: 4192 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); 4193 } 4194 } 4195 4196 static void dwc3_process_event_entry(struct dwc3 *dwc, 4197 const union dwc3_event *event) 4198 { 4199 trace_dwc3_event(event->raw, dwc); 4200 4201 if (!event->type.is_devspec) 4202 dwc3_endpoint_interrupt(dwc, &event->depevt); 4203 else if (event->type.type == DWC3_EVENT_TYPE_DEV) 4204 dwc3_gadget_interrupt(dwc, &event->devt); 4205 else 4206 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); 4207 } 4208 4209 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) 4210 { 4211 struct dwc3 *dwc = evt->dwc; 4212 irqreturn_t ret = IRQ_NONE; 4213 int left; 4214 4215 left = evt->count; 4216 4217 if (!(evt->flags & DWC3_EVENT_PENDING)) 4218 return IRQ_NONE; 4219 4220 while (left > 0) { 4221 union dwc3_event event; 4222 4223 event.raw = *(u32 *) (evt->cache + evt->lpos); 4224 4225 dwc3_process_event_entry(dwc, &event); 4226 4227 /* 4228 * FIXME we wrap around correctly to the next entry as 4229 * almost all entries are 4 bytes in size. There is one 4230 * entry which has 12 bytes which is a regular entry 4231 * followed by 8 bytes data. ATM I don't know how 4232 * things are organized if we get next to the a 4233 * boundary so I worry about that once we try to handle 4234 * that. 4235 */ 4236 evt->lpos = (evt->lpos + 4) % evt->length; 4237 left -= 4; 4238 } 4239 4240 evt->count = 0; 4241 ret = IRQ_HANDLED; 4242 4243 /* Unmask interrupt */ 4244 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), 4245 DWC3_GEVNTSIZ_SIZE(evt->length)); 4246 4247 if (dwc->imod_interval) { 4248 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 4249 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 4250 } 4251 4252 /* Keep the clearing of DWC3_EVENT_PENDING at the end */ 4253 evt->flags &= ~DWC3_EVENT_PENDING; 4254 4255 return ret; 4256 } 4257 4258 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt) 4259 { 4260 struct dwc3_event_buffer *evt = _evt; 4261 struct dwc3 *dwc = evt->dwc; 4262 unsigned long flags; 4263 irqreturn_t ret = IRQ_NONE; 4264 4265 local_bh_disable(); 4266 spin_lock_irqsave(&dwc->lock, flags); 4267 ret = dwc3_process_event_buf(evt); 4268 spin_unlock_irqrestore(&dwc->lock, flags); 4269 local_bh_enable(); 4270 4271 return ret; 4272 } 4273 4274 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) 4275 { 4276 struct dwc3 *dwc = evt->dwc; 4277 u32 amount; 4278 u32 count; 4279 4280 if (pm_runtime_suspended(dwc->dev)) { 4281 pm_runtime_get(dwc->dev); 4282 disable_irq_nosync(dwc->irq_gadget); 4283 dwc->pending_events = true; 4284 return IRQ_HANDLED; 4285 } 4286 4287 /* 4288 * With PCIe legacy interrupt, test shows that top-half irq handler can 4289 * be called again after HW interrupt deassertion. Check if bottom-half 4290 * irq event handler completes before caching new event to prevent 4291 * losing events. 4292 */ 4293 if (evt->flags & DWC3_EVENT_PENDING) 4294 return IRQ_HANDLED; 4295 4296 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); 4297 count &= DWC3_GEVNTCOUNT_MASK; 4298 if (!count) 4299 return IRQ_NONE; 4300 4301 evt->count = count; 4302 evt->flags |= DWC3_EVENT_PENDING; 4303 4304 /* Mask interrupt */ 4305 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), 4306 DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length)); 4307 4308 amount = min(count, evt->length - evt->lpos); 4309 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); 4310 4311 if (amount < count) 4312 memcpy(evt->cache, evt->buf, count - amount); 4313 4314 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); 4315 4316 return IRQ_WAKE_THREAD; 4317 } 4318 4319 static irqreturn_t dwc3_interrupt(int irq, void *_evt) 4320 { 4321 struct dwc3_event_buffer *evt = _evt; 4322 4323 return dwc3_check_event_buf(evt); 4324 } 4325 4326 static int dwc3_gadget_get_irq(struct dwc3 *dwc) 4327 { 4328 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev); 4329 int irq; 4330 4331 irq = platform_get_irq_byname_optional(dwc3_pdev, "peripheral"); 4332 if (irq > 0) 4333 goto out; 4334 4335 if (irq == -EPROBE_DEFER) 4336 goto out; 4337 4338 irq = platform_get_irq_byname_optional(dwc3_pdev, "dwc_usb3"); 4339 if (irq > 0) 4340 goto out; 4341 4342 if (irq == -EPROBE_DEFER) 4343 goto out; 4344 4345 irq = platform_get_irq(dwc3_pdev, 0); 4346 if (irq > 0) 4347 goto out; 4348 4349 if (!irq) 4350 irq = -EINVAL; 4351 4352 out: 4353 return irq; 4354 } 4355 4356 static void dwc_gadget_release(struct device *dev) 4357 { 4358 struct usb_gadget *gadget = container_of(dev, struct usb_gadget, dev); 4359 4360 kfree(gadget); 4361 } 4362 4363 /** 4364 * dwc3_gadget_init - initializes gadget related registers 4365 * @dwc: pointer to our controller context structure 4366 * 4367 * Returns 0 on success otherwise negative errno. 4368 */ 4369 int dwc3_gadget_init(struct dwc3 *dwc) 4370 { 4371 int ret; 4372 int irq; 4373 struct device *dev; 4374 4375 irq = dwc3_gadget_get_irq(dwc); 4376 if (irq < 0) { 4377 ret = irq; 4378 goto err0; 4379 } 4380 4381 dwc->irq_gadget = irq; 4382 4383 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev, 4384 sizeof(*dwc->ep0_trb) * 2, 4385 &dwc->ep0_trb_addr, GFP_KERNEL); 4386 if (!dwc->ep0_trb) { 4387 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); 4388 ret = -ENOMEM; 4389 goto err0; 4390 } 4391 4392 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL); 4393 if (!dwc->setup_buf) { 4394 ret = -ENOMEM; 4395 goto err1; 4396 } 4397 4398 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, 4399 &dwc->bounce_addr, GFP_KERNEL); 4400 if (!dwc->bounce) { 4401 ret = -ENOMEM; 4402 goto err2; 4403 } 4404 4405 init_completion(&dwc->ep0_in_setup); 4406 dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL); 4407 if (!dwc->gadget) { 4408 ret = -ENOMEM; 4409 goto err3; 4410 } 4411 4412 4413 usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release); 4414 dev = &dwc->gadget->dev; 4415 dev->platform_data = dwc; 4416 dwc->gadget->ops = &dwc3_gadget_ops; 4417 dwc->gadget->speed = USB_SPEED_UNKNOWN; 4418 dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN; 4419 dwc->gadget->sg_supported = true; 4420 dwc->gadget->name = "dwc3-gadget"; 4421 dwc->gadget->lpm_capable = !dwc->usb2_gadget_lpm_disable; 4422 4423 /* 4424 * FIXME We might be setting max_speed to <SUPER, however versions 4425 * <2.20a of dwc3 have an issue with metastability (documented 4426 * elsewhere in this driver) which tells us we can't set max speed to 4427 * anything lower than SUPER. 4428 * 4429 * Because gadget.max_speed is only used by composite.c and function 4430 * drivers (i.e. it won't go into dwc3's registers) we are allowing this 4431 * to happen so we avoid sending SuperSpeed Capability descriptor 4432 * together with our BOS descriptor as that could confuse host into 4433 * thinking we can handle super speed. 4434 * 4435 * Note that, in fact, we won't even support GetBOS requests when speed 4436 * is less than super speed because we don't have means, yet, to tell 4437 * composite.c that we are USB 2.0 + LPM ECN. 4438 */ 4439 if (DWC3_VER_IS_PRIOR(DWC3, 220A) && 4440 !dwc->dis_metastability_quirk) 4441 dev_info(dwc->dev, "changing max_speed on rev %08x\n", 4442 dwc->revision); 4443 4444 dwc->gadget->max_speed = dwc->maximum_speed; 4445 dwc->gadget->max_ssp_rate = dwc->max_ssp_rate; 4446 4447 /* 4448 * REVISIT: Here we should clear all pending IRQs to be 4449 * sure we're starting from a well known location. 4450 */ 4451 4452 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps); 4453 if (ret) 4454 goto err4; 4455 4456 ret = usb_add_gadget(dwc->gadget); 4457 if (ret) { 4458 dev_err(dwc->dev, "failed to add gadget\n"); 4459 goto err5; 4460 } 4461 4462 if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS) 4463 dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate); 4464 else 4465 dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed); 4466 4467 return 0; 4468 4469 err5: 4470 dwc3_gadget_free_endpoints(dwc); 4471 err4: 4472 usb_put_gadget(dwc->gadget); 4473 dwc->gadget = NULL; 4474 err3: 4475 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 4476 dwc->bounce_addr); 4477 4478 err2: 4479 kfree(dwc->setup_buf); 4480 4481 err1: 4482 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 4483 dwc->ep0_trb, dwc->ep0_trb_addr); 4484 4485 err0: 4486 return ret; 4487 } 4488 4489 /* -------------------------------------------------------------------------- */ 4490 4491 void dwc3_gadget_exit(struct dwc3 *dwc) 4492 { 4493 if (!dwc->gadget) 4494 return; 4495 4496 usb_del_gadget(dwc->gadget); 4497 dwc3_gadget_free_endpoints(dwc); 4498 usb_put_gadget(dwc->gadget); 4499 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 4500 dwc->bounce_addr); 4501 kfree(dwc->setup_buf); 4502 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 4503 dwc->ep0_trb, dwc->ep0_trb_addr); 4504 } 4505 4506 int dwc3_gadget_suspend(struct dwc3 *dwc) 4507 { 4508 if (!dwc->gadget_driver) 4509 return 0; 4510 4511 dwc3_gadget_run_stop(dwc, false, false); 4512 dwc3_disconnect_gadget(dwc); 4513 __dwc3_gadget_stop(dwc); 4514 4515 return 0; 4516 } 4517 4518 int dwc3_gadget_resume(struct dwc3 *dwc) 4519 { 4520 int ret; 4521 4522 if (!dwc->gadget_driver || !dwc->softconnect) 4523 return 0; 4524 4525 ret = __dwc3_gadget_start(dwc); 4526 if (ret < 0) 4527 goto err0; 4528 4529 ret = dwc3_gadget_run_stop(dwc, true, false); 4530 if (ret < 0) 4531 goto err1; 4532 4533 return 0; 4534 4535 err1: 4536 __dwc3_gadget_stop(dwc); 4537 4538 err0: 4539 return ret; 4540 } 4541 4542 void dwc3_gadget_process_pending_events(struct dwc3 *dwc) 4543 { 4544 if (dwc->pending_events) { 4545 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf); 4546 dwc->pending_events = false; 4547 enable_irq(dwc->irq_gadget); 4548 } 4549 } 4550