1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link 4 * 5 * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com 6 * 7 * Authors: Felipe Balbi <balbi@ti.com>, 8 * Sebastian Andrzej Siewior <bigeasy@linutronix.de> 9 */ 10 11 #include <linux/kernel.h> 12 #include <linux/delay.h> 13 #include <linux/slab.h> 14 #include <linux/spinlock.h> 15 #include <linux/platform_device.h> 16 #include <linux/pm_runtime.h> 17 #include <linux/interrupt.h> 18 #include <linux/io.h> 19 #include <linux/list.h> 20 #include <linux/dma-mapping.h> 21 22 #include <linux/usb/ch9.h> 23 #include <linux/usb/gadget.h> 24 25 #include "debug.h" 26 #include "core.h" 27 #include "gadget.h" 28 #include "io.h" 29 30 #define DWC3_ALIGN_FRAME(d) (((d)->frame_number + (d)->interval) \ 31 & ~((d)->interval - 1)) 32 33 /** 34 * dwc3_gadget_set_test_mode - enables usb2 test modes 35 * @dwc: pointer to our context structure 36 * @mode: the mode to set (J, K SE0 NAK, Force Enable) 37 * 38 * Caller should take care of locking. This function will return 0 on 39 * success or -EINVAL if wrong Test Selector is passed. 40 */ 41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode) 42 { 43 u32 reg; 44 45 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 46 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 47 48 switch (mode) { 49 case TEST_J: 50 case TEST_K: 51 case TEST_SE0_NAK: 52 case TEST_PACKET: 53 case TEST_FORCE_EN: 54 reg |= mode << 1; 55 break; 56 default: 57 return -EINVAL; 58 } 59 60 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 61 62 return 0; 63 } 64 65 /** 66 * dwc3_gadget_get_link_state - gets current state of usb link 67 * @dwc: pointer to our context structure 68 * 69 * Caller should take care of locking. This function will 70 * return the link state on success (>= 0) or -ETIMEDOUT. 71 */ 72 int dwc3_gadget_get_link_state(struct dwc3 *dwc) 73 { 74 u32 reg; 75 76 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 77 78 return DWC3_DSTS_USBLNKST(reg); 79 } 80 81 /** 82 * dwc3_gadget_set_link_state - sets usb link to a particular state 83 * @dwc: pointer to our context structure 84 * @state: the state to put link into 85 * 86 * Caller should take care of locking. This function will 87 * return 0 on success or -ETIMEDOUT. 88 */ 89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state) 90 { 91 int retries = 10000; 92 u32 reg; 93 94 /* 95 * Wait until device controller is ready. Only applies to 1.94a and 96 * later RTL. 97 */ 98 if (dwc->revision >= DWC3_REVISION_194A) { 99 while (--retries) { 100 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 101 if (reg & DWC3_DSTS_DCNRD) 102 udelay(5); 103 else 104 break; 105 } 106 107 if (retries <= 0) 108 return -ETIMEDOUT; 109 } 110 111 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 112 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 113 114 /* set requested state */ 115 reg |= DWC3_DCTL_ULSTCHNGREQ(state); 116 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 117 118 /* 119 * The following code is racy when called from dwc3_gadget_wakeup, 120 * and is not needed, at least on newer versions 121 */ 122 if (dwc->revision >= DWC3_REVISION_194A) 123 return 0; 124 125 /* wait for a change in DSTS */ 126 retries = 10000; 127 while (--retries) { 128 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 129 130 if (DWC3_DSTS_USBLNKST(reg) == state) 131 return 0; 132 133 udelay(5); 134 } 135 136 return -ETIMEDOUT; 137 } 138 139 /** 140 * dwc3_ep_inc_trb - increment a trb index. 141 * @index: Pointer to the TRB index to increment. 142 * 143 * The index should never point to the link TRB. After incrementing, 144 * if it is point to the link TRB, wrap around to the beginning. The 145 * link TRB is always at the last TRB entry. 146 */ 147 static void dwc3_ep_inc_trb(u8 *index) 148 { 149 (*index)++; 150 if (*index == (DWC3_TRB_NUM - 1)) 151 *index = 0; 152 } 153 154 /** 155 * dwc3_ep_inc_enq - increment endpoint's enqueue pointer 156 * @dep: The endpoint whose enqueue pointer we're incrementing 157 */ 158 static void dwc3_ep_inc_enq(struct dwc3_ep *dep) 159 { 160 dwc3_ep_inc_trb(&dep->trb_enqueue); 161 } 162 163 /** 164 * dwc3_ep_inc_deq - increment endpoint's dequeue pointer 165 * @dep: The endpoint whose enqueue pointer we're incrementing 166 */ 167 static void dwc3_ep_inc_deq(struct dwc3_ep *dep) 168 { 169 dwc3_ep_inc_trb(&dep->trb_dequeue); 170 } 171 172 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep, 173 struct dwc3_request *req, int status) 174 { 175 struct dwc3 *dwc = dep->dwc; 176 177 req->started = false; 178 list_del(&req->list); 179 req->remaining = 0; 180 181 if (req->request.status == -EINPROGRESS) 182 req->request.status = status; 183 184 if (req->trb) 185 usb_gadget_unmap_request_by_dev(dwc->sysdev, 186 &req->request, req->direction); 187 188 req->trb = NULL; 189 trace_dwc3_gadget_giveback(req); 190 191 if (dep->number > 1) 192 pm_runtime_put(dwc->dev); 193 } 194 195 /** 196 * dwc3_gadget_giveback - call struct usb_request's ->complete callback 197 * @dep: The endpoint to whom the request belongs to 198 * @req: The request we're giving back 199 * @status: completion code for the request 200 * 201 * Must be called with controller's lock held and interrupts disabled. This 202 * function will unmap @req and call its ->complete() callback to notify upper 203 * layers that it has completed. 204 */ 205 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req, 206 int status) 207 { 208 struct dwc3 *dwc = dep->dwc; 209 210 dwc3_gadget_del_and_unmap_request(dep, req, status); 211 212 spin_unlock(&dwc->lock); 213 usb_gadget_giveback_request(&dep->endpoint, &req->request); 214 spin_lock(&dwc->lock); 215 } 216 217 /** 218 * dwc3_send_gadget_generic_command - issue a generic command for the controller 219 * @dwc: pointer to the controller context 220 * @cmd: the command to be issued 221 * @param: command parameter 222 * 223 * Caller should take care of locking. Issue @cmd with a given @param to @dwc 224 * and wait for its completion. 225 */ 226 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param) 227 { 228 u32 timeout = 500; 229 int status = 0; 230 int ret = 0; 231 u32 reg; 232 233 dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param); 234 dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT); 235 236 do { 237 reg = dwc3_readl(dwc->regs, DWC3_DGCMD); 238 if (!(reg & DWC3_DGCMD_CMDACT)) { 239 status = DWC3_DGCMD_STATUS(reg); 240 if (status) 241 ret = -EINVAL; 242 break; 243 } 244 } while (--timeout); 245 246 if (!timeout) { 247 ret = -ETIMEDOUT; 248 status = -ETIMEDOUT; 249 } 250 251 trace_dwc3_gadget_generic_cmd(cmd, param, status); 252 253 return ret; 254 } 255 256 static int __dwc3_gadget_wakeup(struct dwc3 *dwc); 257 258 /** 259 * dwc3_send_gadget_ep_cmd - issue an endpoint command 260 * @dep: the endpoint to which the command is going to be issued 261 * @cmd: the command to be issued 262 * @params: parameters to the command 263 * 264 * Caller should handle locking. This function will issue @cmd with given 265 * @params to @dep and wait for its completion. 266 */ 267 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd, 268 struct dwc3_gadget_ep_cmd_params *params) 269 { 270 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 271 struct dwc3 *dwc = dep->dwc; 272 u32 timeout = 1000; 273 u32 saved_config = 0; 274 u32 reg; 275 276 int cmd_status = 0; 277 int ret = -EINVAL; 278 279 /* 280 * When operating in USB 2.0 speeds (HS/FS), if GUSB2PHYCFG.ENBLSLPM or 281 * GUSB2PHYCFG.SUSPHY is set, it must be cleared before issuing an 282 * endpoint command. 283 * 284 * Save and clear both GUSB2PHYCFG.ENBLSLPM and GUSB2PHYCFG.SUSPHY 285 * settings. Restore them after the command is completed. 286 * 287 * DWC_usb3 3.30a and DWC_usb31 1.90a programming guide section 3.2.2 288 */ 289 if (dwc->gadget.speed <= USB_SPEED_HIGH) { 290 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 291 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) { 292 saved_config |= DWC3_GUSB2PHYCFG_SUSPHY; 293 reg &= ~DWC3_GUSB2PHYCFG_SUSPHY; 294 } 295 296 if (reg & DWC3_GUSB2PHYCFG_ENBLSLPM) { 297 saved_config |= DWC3_GUSB2PHYCFG_ENBLSLPM; 298 reg &= ~DWC3_GUSB2PHYCFG_ENBLSLPM; 299 } 300 301 if (saved_config) 302 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 303 } 304 305 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) { 306 int needs_wakeup; 307 308 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 || 309 dwc->link_state == DWC3_LINK_STATE_U2 || 310 dwc->link_state == DWC3_LINK_STATE_U3); 311 312 if (unlikely(needs_wakeup)) { 313 ret = __dwc3_gadget_wakeup(dwc); 314 dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n", 315 ret); 316 } 317 } 318 319 dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0); 320 dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1); 321 dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2); 322 323 /* 324 * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're 325 * not relying on XferNotReady, we can make use of a special "No 326 * Response Update Transfer" command where we should clear both CmdAct 327 * and CmdIOC bits. 328 * 329 * With this, we don't need to wait for command completion and can 330 * straight away issue further commands to the endpoint. 331 * 332 * NOTICE: We're making an assumption that control endpoints will never 333 * make use of Update Transfer command. This is a safe assumption 334 * because we can never have more than one request at a time with 335 * Control Endpoints. If anybody changes that assumption, this chunk 336 * needs to be updated accordingly. 337 */ 338 if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER && 339 !usb_endpoint_xfer_isoc(desc)) 340 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT); 341 else 342 cmd |= DWC3_DEPCMD_CMDACT; 343 344 dwc3_writel(dep->regs, DWC3_DEPCMD, cmd); 345 do { 346 reg = dwc3_readl(dep->regs, DWC3_DEPCMD); 347 if (!(reg & DWC3_DEPCMD_CMDACT)) { 348 cmd_status = DWC3_DEPCMD_STATUS(reg); 349 350 switch (cmd_status) { 351 case 0: 352 ret = 0; 353 break; 354 case DEPEVT_TRANSFER_NO_RESOURCE: 355 ret = -EINVAL; 356 break; 357 case DEPEVT_TRANSFER_BUS_EXPIRY: 358 /* 359 * SW issues START TRANSFER command to 360 * isochronous ep with future frame interval. If 361 * future interval time has already passed when 362 * core receives the command, it will respond 363 * with an error status of 'Bus Expiry'. 364 * 365 * Instead of always returning -EINVAL, let's 366 * give a hint to the gadget driver that this is 367 * the case by returning -EAGAIN. 368 */ 369 ret = -EAGAIN; 370 break; 371 default: 372 dev_WARN(dwc->dev, "UNKNOWN cmd status\n"); 373 } 374 375 break; 376 } 377 } while (--timeout); 378 379 if (timeout == 0) { 380 ret = -ETIMEDOUT; 381 cmd_status = -ETIMEDOUT; 382 } 383 384 trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status); 385 386 if (ret == 0) { 387 switch (DWC3_DEPCMD_CMD(cmd)) { 388 case DWC3_DEPCMD_STARTTRANSFER: 389 dep->flags |= DWC3_EP_TRANSFER_STARTED; 390 dwc3_gadget_ep_get_transfer_index(dep); 391 break; 392 case DWC3_DEPCMD_ENDTRANSFER: 393 dep->flags &= ~DWC3_EP_TRANSFER_STARTED; 394 break; 395 default: 396 /* nothing */ 397 break; 398 } 399 } 400 401 if (saved_config) { 402 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0)); 403 reg |= saved_config; 404 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg); 405 } 406 407 return ret; 408 } 409 410 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep) 411 { 412 struct dwc3 *dwc = dep->dwc; 413 struct dwc3_gadget_ep_cmd_params params; 414 u32 cmd = DWC3_DEPCMD_CLEARSTALL; 415 416 /* 417 * As of core revision 2.60a the recommended programming model 418 * is to set the ClearPendIN bit when issuing a Clear Stall EP 419 * command for IN endpoints. This is to prevent an issue where 420 * some (non-compliant) hosts may not send ACK TPs for pending 421 * IN transfers due to a mishandled error condition. Synopsys 422 * STAR 9000614252. 423 */ 424 if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) && 425 (dwc->gadget.speed >= USB_SPEED_SUPER)) 426 cmd |= DWC3_DEPCMD_CLEARPENDIN; 427 428 memset(¶ms, 0, sizeof(params)); 429 430 return dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 431 } 432 433 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, 434 struct dwc3_trb *trb) 435 { 436 u32 offset = (char *) trb - (char *) dep->trb_pool; 437 438 return dep->trb_pool_dma + offset; 439 } 440 441 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep) 442 { 443 struct dwc3 *dwc = dep->dwc; 444 445 if (dep->trb_pool) 446 return 0; 447 448 dep->trb_pool = dma_alloc_coherent(dwc->sysdev, 449 sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 450 &dep->trb_pool_dma, GFP_KERNEL); 451 if (!dep->trb_pool) { 452 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n", 453 dep->name); 454 return -ENOMEM; 455 } 456 457 return 0; 458 } 459 460 static void dwc3_free_trb_pool(struct dwc3_ep *dep) 461 { 462 struct dwc3 *dwc = dep->dwc; 463 464 dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM, 465 dep->trb_pool, dep->trb_pool_dma); 466 467 dep->trb_pool = NULL; 468 dep->trb_pool_dma = 0; 469 } 470 471 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep) 472 { 473 struct dwc3_gadget_ep_cmd_params params; 474 475 memset(¶ms, 0x00, sizeof(params)); 476 477 params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1); 478 479 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE, 480 ¶ms); 481 } 482 483 /** 484 * dwc3_gadget_start_config - configure ep resources 485 * @dep: endpoint that is being enabled 486 * 487 * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's 488 * completion, it will set Transfer Resource for all available endpoints. 489 * 490 * The assignment of transfer resources cannot perfectly follow the data book 491 * due to the fact that the controller driver does not have all knowledge of the 492 * configuration in advance. It is given this information piecemeal by the 493 * composite gadget framework after every SET_CONFIGURATION and 494 * SET_INTERFACE. Trying to follow the databook programming model in this 495 * scenario can cause errors. For two reasons: 496 * 497 * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every 498 * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is 499 * incorrect in the scenario of multiple interfaces. 500 * 501 * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new 502 * endpoint on alt setting (8.1.6). 503 * 504 * The following simplified method is used instead: 505 * 506 * All hardware endpoints can be assigned a transfer resource and this setting 507 * will stay persistent until either a core reset or hibernation. So whenever we 508 * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do 509 * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are 510 * guaranteed that there are as many transfer resources as endpoints. 511 * 512 * This function is called for each endpoint when it is being enabled but is 513 * triggered only when called for EP0-out, which always happens first, and which 514 * should only happen in one of the above conditions. 515 */ 516 static int dwc3_gadget_start_config(struct dwc3_ep *dep) 517 { 518 struct dwc3_gadget_ep_cmd_params params; 519 struct dwc3 *dwc; 520 u32 cmd; 521 int i; 522 int ret; 523 524 if (dep->number) 525 return 0; 526 527 memset(¶ms, 0x00, sizeof(params)); 528 cmd = DWC3_DEPCMD_DEPSTARTCFG; 529 dwc = dep->dwc; 530 531 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 532 if (ret) 533 return ret; 534 535 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 536 struct dwc3_ep *dep = dwc->eps[i]; 537 538 if (!dep) 539 continue; 540 541 ret = dwc3_gadget_set_xfer_resource(dep); 542 if (ret) 543 return ret; 544 } 545 546 return 0; 547 } 548 549 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action) 550 { 551 const struct usb_ss_ep_comp_descriptor *comp_desc; 552 const struct usb_endpoint_descriptor *desc; 553 struct dwc3_gadget_ep_cmd_params params; 554 struct dwc3 *dwc = dep->dwc; 555 556 comp_desc = dep->endpoint.comp_desc; 557 desc = dep->endpoint.desc; 558 559 memset(¶ms, 0x00, sizeof(params)); 560 561 params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc)) 562 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc)); 563 564 /* Burst size is only needed in SuperSpeed mode */ 565 if (dwc->gadget.speed >= USB_SPEED_SUPER) { 566 u32 burst = dep->endpoint.maxburst; 567 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1); 568 } 569 570 params.param0 |= action; 571 if (action == DWC3_DEPCFG_ACTION_RESTORE) 572 params.param2 |= dep->saved_state; 573 574 if (usb_endpoint_xfer_control(desc)) 575 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN; 576 577 if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc)) 578 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN; 579 580 if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) { 581 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE 582 | DWC3_DEPCFG_STREAM_EVENT_EN; 583 dep->stream_capable = true; 584 } 585 586 if (!usb_endpoint_xfer_control(desc)) 587 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN; 588 589 /* 590 * We are doing 1:1 mapping for endpoints, meaning 591 * Physical Endpoints 2 maps to Logical Endpoint 2 and 592 * so on. We consider the direction bit as part of the physical 593 * endpoint number. So USB endpoint 0x81 is 0x03. 594 */ 595 params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number); 596 597 /* 598 * We must use the lower 16 TX FIFOs even though 599 * HW might have more 600 */ 601 if (dep->direction) 602 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1); 603 604 if (desc->bInterval) { 605 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1); 606 dep->interval = 1 << (desc->bInterval - 1); 607 } 608 609 return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, ¶ms); 610 } 611 612 /** 613 * __dwc3_gadget_ep_enable - initializes a hw endpoint 614 * @dep: endpoint to be initialized 615 * @action: one of INIT, MODIFY or RESTORE 616 * 617 * Caller should take care of locking. Execute all necessary commands to 618 * initialize a HW endpoint so it can be used by a gadget driver. 619 */ 620 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action) 621 { 622 const struct usb_endpoint_descriptor *desc = dep->endpoint.desc; 623 struct dwc3 *dwc = dep->dwc; 624 625 u32 reg; 626 int ret; 627 628 if (!(dep->flags & DWC3_EP_ENABLED)) { 629 ret = dwc3_gadget_start_config(dep); 630 if (ret) 631 return ret; 632 } 633 634 ret = dwc3_gadget_set_ep_config(dep, action); 635 if (ret) 636 return ret; 637 638 if (!(dep->flags & DWC3_EP_ENABLED)) { 639 struct dwc3_trb *trb_st_hw; 640 struct dwc3_trb *trb_link; 641 642 dep->type = usb_endpoint_type(desc); 643 dep->flags |= DWC3_EP_ENABLED; 644 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; 645 646 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 647 reg |= DWC3_DALEPENA_EP(dep->number); 648 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 649 650 init_waitqueue_head(&dep->wait_end_transfer); 651 652 if (usb_endpoint_xfer_control(desc)) 653 goto out; 654 655 /* Initialize the TRB ring */ 656 dep->trb_dequeue = 0; 657 dep->trb_enqueue = 0; 658 memset(dep->trb_pool, 0, 659 sizeof(struct dwc3_trb) * DWC3_TRB_NUM); 660 661 /* Link TRB. The HWO bit is never reset */ 662 trb_st_hw = &dep->trb_pool[0]; 663 664 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1]; 665 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 666 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw)); 667 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB; 668 trb_link->ctrl |= DWC3_TRB_CTRL_HWO; 669 } 670 671 /* 672 * Issue StartTransfer here with no-op TRB so we can always rely on No 673 * Response Update Transfer command. 674 */ 675 if (usb_endpoint_xfer_bulk(desc) || 676 usb_endpoint_xfer_int(desc)) { 677 struct dwc3_gadget_ep_cmd_params params; 678 struct dwc3_trb *trb; 679 dma_addr_t trb_dma; 680 u32 cmd; 681 682 memset(¶ms, 0, sizeof(params)); 683 trb = &dep->trb_pool[0]; 684 trb_dma = dwc3_trb_dma_offset(dep, trb); 685 686 params.param0 = upper_32_bits(trb_dma); 687 params.param1 = lower_32_bits(trb_dma); 688 689 cmd = DWC3_DEPCMD_STARTTRANSFER; 690 691 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 692 if (ret < 0) 693 return ret; 694 } 695 696 out: 697 trace_dwc3_gadget_ep_enable(dep); 698 699 return 0; 700 } 701 702 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force); 703 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep) 704 { 705 struct dwc3_request *req; 706 707 dwc3_stop_active_transfer(dep, true); 708 709 /* - giveback all requests to gadget driver */ 710 while (!list_empty(&dep->started_list)) { 711 req = next_request(&dep->started_list); 712 713 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 714 } 715 716 while (!list_empty(&dep->pending_list)) { 717 req = next_request(&dep->pending_list); 718 719 dwc3_gadget_giveback(dep, req, -ESHUTDOWN); 720 } 721 } 722 723 /** 724 * __dwc3_gadget_ep_disable - disables a hw endpoint 725 * @dep: the endpoint to disable 726 * 727 * This function undoes what __dwc3_gadget_ep_enable did and also removes 728 * requests which are currently being processed by the hardware and those which 729 * are not yet scheduled. 730 * 731 * Caller should take care of locking. 732 */ 733 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep) 734 { 735 struct dwc3 *dwc = dep->dwc; 736 u32 reg; 737 738 trace_dwc3_gadget_ep_disable(dep); 739 740 dwc3_remove_requests(dwc, dep); 741 742 /* make sure HW endpoint isn't stalled */ 743 if (dep->flags & DWC3_EP_STALL) 744 __dwc3_gadget_ep_set_halt(dep, 0, false); 745 746 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA); 747 reg &= ~DWC3_DALEPENA_EP(dep->number); 748 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg); 749 750 dep->stream_capable = false; 751 dep->type = 0; 752 dep->flags &= DWC3_EP_END_TRANSFER_PENDING; 753 754 /* Clear out the ep descriptors for non-ep0 */ 755 if (dep->number > 1) { 756 dep->endpoint.comp_desc = NULL; 757 dep->endpoint.desc = NULL; 758 } 759 760 return 0; 761 } 762 763 /* -------------------------------------------------------------------------- */ 764 765 static int dwc3_gadget_ep0_enable(struct usb_ep *ep, 766 const struct usb_endpoint_descriptor *desc) 767 { 768 return -EINVAL; 769 } 770 771 static int dwc3_gadget_ep0_disable(struct usb_ep *ep) 772 { 773 return -EINVAL; 774 } 775 776 /* -------------------------------------------------------------------------- */ 777 778 static int dwc3_gadget_ep_enable(struct usb_ep *ep, 779 const struct usb_endpoint_descriptor *desc) 780 { 781 struct dwc3_ep *dep; 782 struct dwc3 *dwc; 783 unsigned long flags; 784 int ret; 785 786 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) { 787 pr_debug("dwc3: invalid parameters\n"); 788 return -EINVAL; 789 } 790 791 if (!desc->wMaxPacketSize) { 792 pr_debug("dwc3: missing wMaxPacketSize\n"); 793 return -EINVAL; 794 } 795 796 dep = to_dwc3_ep(ep); 797 dwc = dep->dwc; 798 799 if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED, 800 "%s is already enabled\n", 801 dep->name)) 802 return 0; 803 804 spin_lock_irqsave(&dwc->lock, flags); 805 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 806 spin_unlock_irqrestore(&dwc->lock, flags); 807 808 return ret; 809 } 810 811 static int dwc3_gadget_ep_disable(struct usb_ep *ep) 812 { 813 struct dwc3_ep *dep; 814 struct dwc3 *dwc; 815 unsigned long flags; 816 int ret; 817 818 if (!ep) { 819 pr_debug("dwc3: invalid parameters\n"); 820 return -EINVAL; 821 } 822 823 dep = to_dwc3_ep(ep); 824 dwc = dep->dwc; 825 826 if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED), 827 "%s is already disabled\n", 828 dep->name)) 829 return 0; 830 831 spin_lock_irqsave(&dwc->lock, flags); 832 ret = __dwc3_gadget_ep_disable(dep); 833 spin_unlock_irqrestore(&dwc->lock, flags); 834 835 return ret; 836 } 837 838 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep, 839 gfp_t gfp_flags) 840 { 841 struct dwc3_request *req; 842 struct dwc3_ep *dep = to_dwc3_ep(ep); 843 844 req = kzalloc(sizeof(*req), gfp_flags); 845 if (!req) 846 return NULL; 847 848 req->direction = dep->direction; 849 req->epnum = dep->number; 850 req->dep = dep; 851 852 trace_dwc3_alloc_request(req); 853 854 return &req->request; 855 } 856 857 static void dwc3_gadget_ep_free_request(struct usb_ep *ep, 858 struct usb_request *request) 859 { 860 struct dwc3_request *req = to_dwc3_request(request); 861 862 trace_dwc3_free_request(req); 863 kfree(req); 864 } 865 866 /** 867 * dwc3_ep_prev_trb - returns the previous TRB in the ring 868 * @dep: The endpoint with the TRB ring 869 * @index: The index of the current TRB in the ring 870 * 871 * Returns the TRB prior to the one pointed to by the index. If the 872 * index is 0, we will wrap backwards, skip the link TRB, and return 873 * the one just before that. 874 */ 875 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index) 876 { 877 u8 tmp = index; 878 879 if (!tmp) 880 tmp = DWC3_TRB_NUM - 1; 881 882 return &dep->trb_pool[tmp - 1]; 883 } 884 885 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep) 886 { 887 struct dwc3_trb *tmp; 888 u8 trbs_left; 889 890 /* 891 * If enqueue & dequeue are equal than it is either full or empty. 892 * 893 * One way to know for sure is if the TRB right before us has HWO bit 894 * set or not. If it has, then we're definitely full and can't fit any 895 * more transfers in our ring. 896 */ 897 if (dep->trb_enqueue == dep->trb_dequeue) { 898 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue); 899 if (tmp->ctrl & DWC3_TRB_CTRL_HWO) 900 return 0; 901 902 return DWC3_TRB_NUM - 1; 903 } 904 905 trbs_left = dep->trb_dequeue - dep->trb_enqueue; 906 trbs_left &= (DWC3_TRB_NUM - 1); 907 908 if (dep->trb_dequeue < dep->trb_enqueue) 909 trbs_left--; 910 911 return trbs_left; 912 } 913 914 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb, 915 dma_addr_t dma, unsigned length, unsigned chain, unsigned node, 916 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt) 917 { 918 struct dwc3 *dwc = dep->dwc; 919 struct usb_gadget *gadget = &dwc->gadget; 920 enum usb_device_speed speed = gadget->speed; 921 922 dwc3_ep_inc_enq(dep); 923 924 trb->size = DWC3_TRB_SIZE_LENGTH(length); 925 trb->bpl = lower_32_bits(dma); 926 trb->bph = upper_32_bits(dma); 927 928 switch (usb_endpoint_type(dep->endpoint.desc)) { 929 case USB_ENDPOINT_XFER_CONTROL: 930 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP; 931 break; 932 933 case USB_ENDPOINT_XFER_ISOC: 934 if (!node) { 935 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST; 936 937 /* 938 * USB Specification 2.0 Section 5.9.2 states that: "If 939 * there is only a single transaction in the microframe, 940 * only a DATA0 data packet PID is used. If there are 941 * two transactions per microframe, DATA1 is used for 942 * the first transaction data packet and DATA0 is used 943 * for the second transaction data packet. If there are 944 * three transactions per microframe, DATA2 is used for 945 * the first transaction data packet, DATA1 is used for 946 * the second, and DATA0 is used for the third." 947 * 948 * IOW, we should satisfy the following cases: 949 * 950 * 1) length <= maxpacket 951 * - DATA0 952 * 953 * 2) maxpacket < length <= (2 * maxpacket) 954 * - DATA1, DATA0 955 * 956 * 3) (2 * maxpacket) < length <= (3 * maxpacket) 957 * - DATA2, DATA1, DATA0 958 */ 959 if (speed == USB_SPEED_HIGH) { 960 struct usb_ep *ep = &dep->endpoint; 961 unsigned int mult = 2; 962 unsigned int maxp = usb_endpoint_maxp(ep->desc); 963 964 if (length <= (2 * maxp)) 965 mult--; 966 967 if (length <= maxp) 968 mult--; 969 970 trb->size |= DWC3_TRB_SIZE_PCM1(mult); 971 } 972 } else { 973 trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS; 974 } 975 976 /* always enable Interrupt on Missed ISOC */ 977 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 978 break; 979 980 case USB_ENDPOINT_XFER_BULK: 981 case USB_ENDPOINT_XFER_INT: 982 trb->ctrl = DWC3_TRBCTL_NORMAL; 983 break; 984 default: 985 /* 986 * This is only possible with faulty memory because we 987 * checked it already :) 988 */ 989 dev_WARN(dwc->dev, "Unknown endpoint type %d\n", 990 usb_endpoint_type(dep->endpoint.desc)); 991 } 992 993 /* always enable Continue on Short Packet */ 994 if (usb_endpoint_dir_out(dep->endpoint.desc)) { 995 trb->ctrl |= DWC3_TRB_CTRL_CSP; 996 997 if (short_not_ok) 998 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI; 999 } 1000 1001 if ((!no_interrupt && !chain) || 1002 (dwc3_calc_trbs_left(dep) == 0)) 1003 trb->ctrl |= DWC3_TRB_CTRL_IOC; 1004 1005 if (chain) 1006 trb->ctrl |= DWC3_TRB_CTRL_CHN; 1007 1008 if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable) 1009 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id); 1010 1011 trb->ctrl |= DWC3_TRB_CTRL_HWO; 1012 1013 trace_dwc3_prepare_trb(dep, trb); 1014 } 1015 1016 /** 1017 * dwc3_prepare_one_trb - setup one TRB from one request 1018 * @dep: endpoint for which this request is prepared 1019 * @req: dwc3_request pointer 1020 * @chain: should this TRB be chained to the next? 1021 * @node: only for isochronous endpoints. First TRB needs different type. 1022 */ 1023 static void dwc3_prepare_one_trb(struct dwc3_ep *dep, 1024 struct dwc3_request *req, unsigned chain, unsigned node) 1025 { 1026 struct dwc3_trb *trb; 1027 unsigned int length; 1028 dma_addr_t dma; 1029 unsigned stream_id = req->request.stream_id; 1030 unsigned short_not_ok = req->request.short_not_ok; 1031 unsigned no_interrupt = req->request.no_interrupt; 1032 1033 if (req->request.num_sgs > 0) { 1034 length = sg_dma_len(req->start_sg); 1035 dma = sg_dma_address(req->start_sg); 1036 } else { 1037 length = req->request.length; 1038 dma = req->request.dma; 1039 } 1040 1041 trb = &dep->trb_pool[dep->trb_enqueue]; 1042 1043 if (!req->trb) { 1044 dwc3_gadget_move_started_request(req); 1045 req->trb = trb; 1046 req->trb_dma = dwc3_trb_dma_offset(dep, trb); 1047 } 1048 1049 __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node, 1050 stream_id, short_not_ok, no_interrupt); 1051 } 1052 1053 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep, 1054 struct dwc3_request *req) 1055 { 1056 struct scatterlist *sg = req->start_sg; 1057 struct scatterlist *s; 1058 int i; 1059 1060 unsigned int remaining = req->request.num_mapped_sgs 1061 - req->num_queued_sgs; 1062 1063 for_each_sg(sg, s, remaining, i) { 1064 unsigned int length = req->request.length; 1065 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1066 unsigned int rem = length % maxp; 1067 unsigned chain = true; 1068 1069 if (sg_is_last(s)) 1070 chain = false; 1071 1072 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) { 1073 struct dwc3 *dwc = dep->dwc; 1074 struct dwc3_trb *trb; 1075 1076 req->unaligned = true; 1077 1078 /* prepare normal TRB */ 1079 dwc3_prepare_one_trb(dep, req, true, i); 1080 1081 /* Now prepare one extra TRB to align transfer size */ 1082 trb = &dep->trb_pool[dep->trb_enqueue]; 1083 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 1084 maxp - rem, false, 1, 1085 req->request.stream_id, 1086 req->request.short_not_ok, 1087 req->request.no_interrupt); 1088 } else { 1089 dwc3_prepare_one_trb(dep, req, chain, i); 1090 } 1091 1092 /* 1093 * There can be a situation where all sgs in sglist are not 1094 * queued because of insufficient trb number. To handle this 1095 * case, update start_sg to next sg to be queued, so that 1096 * we have free trbs we can continue queuing from where we 1097 * previously stopped 1098 */ 1099 if (chain) 1100 req->start_sg = sg_next(s); 1101 1102 req->num_queued_sgs++; 1103 1104 if (!dwc3_calc_trbs_left(dep)) 1105 break; 1106 } 1107 } 1108 1109 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep, 1110 struct dwc3_request *req) 1111 { 1112 unsigned int length = req->request.length; 1113 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc); 1114 unsigned int rem = length % maxp; 1115 1116 if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) { 1117 struct dwc3 *dwc = dep->dwc; 1118 struct dwc3_trb *trb; 1119 1120 req->unaligned = true; 1121 1122 /* prepare normal TRB */ 1123 dwc3_prepare_one_trb(dep, req, true, 0); 1124 1125 /* Now prepare one extra TRB to align transfer size */ 1126 trb = &dep->trb_pool[dep->trb_enqueue]; 1127 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem, 1128 false, 1, req->request.stream_id, 1129 req->request.short_not_ok, 1130 req->request.no_interrupt); 1131 } else if (req->request.zero && req->request.length && 1132 (IS_ALIGNED(req->request.length, maxp))) { 1133 struct dwc3 *dwc = dep->dwc; 1134 struct dwc3_trb *trb; 1135 1136 req->zero = true; 1137 1138 /* prepare normal TRB */ 1139 dwc3_prepare_one_trb(dep, req, true, 0); 1140 1141 /* Now prepare one extra TRB to handle ZLP */ 1142 trb = &dep->trb_pool[dep->trb_enqueue]; 1143 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0, 1144 false, 1, req->request.stream_id, 1145 req->request.short_not_ok, 1146 req->request.no_interrupt); 1147 } else { 1148 dwc3_prepare_one_trb(dep, req, false, 0); 1149 } 1150 } 1151 1152 /* 1153 * dwc3_prepare_trbs - setup TRBs from requests 1154 * @dep: endpoint for which requests are being prepared 1155 * 1156 * The function goes through the requests list and sets up TRBs for the 1157 * transfers. The function returns once there are no more TRBs available or 1158 * it runs out of requests. 1159 */ 1160 static void dwc3_prepare_trbs(struct dwc3_ep *dep) 1161 { 1162 struct dwc3_request *req, *n; 1163 1164 BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM); 1165 1166 /* 1167 * We can get in a situation where there's a request in the started list 1168 * but there weren't enough TRBs to fully kick it in the first time 1169 * around, so it has been waiting for more TRBs to be freed up. 1170 * 1171 * In that case, we should check if we have a request with pending_sgs 1172 * in the started list and prepare TRBs for that request first, 1173 * otherwise we will prepare TRBs completely out of order and that will 1174 * break things. 1175 */ 1176 list_for_each_entry(req, &dep->started_list, list) { 1177 if (req->num_pending_sgs > 0) 1178 dwc3_prepare_one_trb_sg(dep, req); 1179 1180 if (!dwc3_calc_trbs_left(dep)) 1181 return; 1182 } 1183 1184 list_for_each_entry_safe(req, n, &dep->pending_list, list) { 1185 struct dwc3 *dwc = dep->dwc; 1186 int ret; 1187 1188 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request, 1189 dep->direction); 1190 if (ret) 1191 return; 1192 1193 req->sg = req->request.sg; 1194 req->start_sg = req->sg; 1195 req->num_queued_sgs = 0; 1196 req->num_pending_sgs = req->request.num_mapped_sgs; 1197 1198 if (req->num_pending_sgs > 0) 1199 dwc3_prepare_one_trb_sg(dep, req); 1200 else 1201 dwc3_prepare_one_trb_linear(dep, req); 1202 1203 if (!dwc3_calc_trbs_left(dep)) 1204 return; 1205 } 1206 } 1207 1208 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep) 1209 { 1210 struct dwc3_gadget_ep_cmd_params params; 1211 struct dwc3_request *req; 1212 int starting; 1213 int ret; 1214 u32 cmd; 1215 1216 if (!dwc3_calc_trbs_left(dep)) 1217 return 0; 1218 1219 starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED); 1220 1221 dwc3_prepare_trbs(dep); 1222 req = next_request(&dep->started_list); 1223 if (!req) { 1224 dep->flags |= DWC3_EP_PENDING_REQUEST; 1225 return 0; 1226 } 1227 1228 memset(¶ms, 0, sizeof(params)); 1229 1230 if (starting) { 1231 params.param0 = upper_32_bits(req->trb_dma); 1232 params.param1 = lower_32_bits(req->trb_dma); 1233 cmd = DWC3_DEPCMD_STARTTRANSFER; 1234 1235 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) 1236 cmd |= DWC3_DEPCMD_PARAM(dep->frame_number); 1237 } else { 1238 cmd = DWC3_DEPCMD_UPDATETRANSFER | 1239 DWC3_DEPCMD_PARAM(dep->resource_index); 1240 } 1241 1242 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 1243 if (ret < 0) { 1244 /* 1245 * FIXME we need to iterate over the list of requests 1246 * here and stop, unmap, free and del each of the linked 1247 * requests instead of what we do now. 1248 */ 1249 if (req->trb) 1250 memset(req->trb, 0, sizeof(struct dwc3_trb)); 1251 dwc3_gadget_del_and_unmap_request(dep, req, ret); 1252 return ret; 1253 } 1254 1255 return 0; 1256 } 1257 1258 static int __dwc3_gadget_get_frame(struct dwc3 *dwc) 1259 { 1260 u32 reg; 1261 1262 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1263 return DWC3_DSTS_SOFFN(reg); 1264 } 1265 1266 static void __dwc3_gadget_start_isoc(struct dwc3_ep *dep) 1267 { 1268 if (list_empty(&dep->pending_list)) { 1269 dev_info(dep->dwc->dev, "%s: ran out of requests\n", 1270 dep->name); 1271 dep->flags |= DWC3_EP_PENDING_REQUEST; 1272 return; 1273 } 1274 1275 dep->frame_number = DWC3_ALIGN_FRAME(dep); 1276 __dwc3_gadget_kick_transfer(dep); 1277 } 1278 1279 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req) 1280 { 1281 struct dwc3 *dwc = dep->dwc; 1282 1283 if (!dep->endpoint.desc) { 1284 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n", 1285 dep->name); 1286 return -ESHUTDOWN; 1287 } 1288 1289 if (WARN(req->dep != dep, "request %pK belongs to '%s'\n", 1290 &req->request, req->dep->name)) 1291 return -EINVAL; 1292 1293 pm_runtime_get(dwc->dev); 1294 1295 req->request.actual = 0; 1296 req->request.status = -EINPROGRESS; 1297 1298 trace_dwc3_ep_queue(req); 1299 1300 list_add_tail(&req->list, &dep->pending_list); 1301 1302 /* 1303 * NOTICE: Isochronous endpoints should NEVER be prestarted. We must 1304 * wait for a XferNotReady event so we will know what's the current 1305 * (micro-)frame number. 1306 * 1307 * Without this trick, we are very, very likely gonna get Bus Expiry 1308 * errors which will force us issue EndTransfer command. 1309 */ 1310 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 1311 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) && 1312 !(dep->flags & DWC3_EP_TRANSFER_STARTED)) 1313 return 0; 1314 1315 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) { 1316 if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) { 1317 __dwc3_gadget_start_isoc(dep); 1318 return 0; 1319 } 1320 } 1321 } 1322 1323 return __dwc3_gadget_kick_transfer(dep); 1324 } 1325 1326 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request, 1327 gfp_t gfp_flags) 1328 { 1329 struct dwc3_request *req = to_dwc3_request(request); 1330 struct dwc3_ep *dep = to_dwc3_ep(ep); 1331 struct dwc3 *dwc = dep->dwc; 1332 1333 unsigned long flags; 1334 1335 int ret; 1336 1337 spin_lock_irqsave(&dwc->lock, flags); 1338 ret = __dwc3_gadget_ep_queue(dep, req); 1339 spin_unlock_irqrestore(&dwc->lock, flags); 1340 1341 return ret; 1342 } 1343 1344 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep, 1345 struct usb_request *request) 1346 { 1347 struct dwc3_request *req = to_dwc3_request(request); 1348 struct dwc3_request *r = NULL; 1349 1350 struct dwc3_ep *dep = to_dwc3_ep(ep); 1351 struct dwc3 *dwc = dep->dwc; 1352 1353 unsigned long flags; 1354 int ret = 0; 1355 1356 trace_dwc3_ep_dequeue(req); 1357 1358 spin_lock_irqsave(&dwc->lock, flags); 1359 1360 list_for_each_entry(r, &dep->pending_list, list) { 1361 if (r == req) 1362 break; 1363 } 1364 1365 if (r != req) { 1366 list_for_each_entry(r, &dep->started_list, list) { 1367 if (r == req) 1368 break; 1369 } 1370 if (r == req) { 1371 /* wait until it is processed */ 1372 dwc3_stop_active_transfer(dep, true); 1373 1374 /* 1375 * If request was already started, this means we had to 1376 * stop the transfer. With that we also need to ignore 1377 * all TRBs used by the request, however TRBs can only 1378 * be modified after completion of END_TRANSFER 1379 * command. So what we do here is that we wait for 1380 * END_TRANSFER completion and only after that, we jump 1381 * over TRBs by clearing HWO and incrementing dequeue 1382 * pointer. 1383 * 1384 * Note that we have 2 possible types of transfers here: 1385 * 1386 * i) Linear buffer request 1387 * ii) SG-list based request 1388 * 1389 * SG-list based requests will have r->num_pending_sgs 1390 * set to a valid number (> 0). Linear requests, 1391 * normally use a single TRB. 1392 * 1393 * For each of these two cases, if r->unaligned flag is 1394 * set, one extra TRB has been used to align transfer 1395 * size to wMaxPacketSize. 1396 * 1397 * All of these cases need to be taken into 1398 * consideration so we don't mess up our TRB ring 1399 * pointers. 1400 */ 1401 wait_event_lock_irq(dep->wait_end_transfer, 1402 !(dep->flags & DWC3_EP_END_TRANSFER_PENDING), 1403 dwc->lock); 1404 1405 if (!r->trb) 1406 goto out0; 1407 1408 if (r->num_pending_sgs) { 1409 struct dwc3_trb *trb; 1410 int i = 0; 1411 1412 for (i = 0; i < r->num_pending_sgs; i++) { 1413 trb = r->trb + i; 1414 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 1415 dwc3_ep_inc_deq(dep); 1416 } 1417 1418 if (r->unaligned || r->zero) { 1419 trb = r->trb + r->num_pending_sgs + 1; 1420 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 1421 dwc3_ep_inc_deq(dep); 1422 } 1423 } else { 1424 struct dwc3_trb *trb = r->trb; 1425 1426 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 1427 dwc3_ep_inc_deq(dep); 1428 1429 if (r->unaligned || r->zero) { 1430 trb = r->trb + 1; 1431 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 1432 dwc3_ep_inc_deq(dep); 1433 } 1434 } 1435 goto out1; 1436 } 1437 dev_err(dwc->dev, "request %pK was not queued to %s\n", 1438 request, ep->name); 1439 ret = -EINVAL; 1440 goto out0; 1441 } 1442 1443 out1: 1444 /* giveback the request */ 1445 1446 dwc3_gadget_giveback(dep, req, -ECONNRESET); 1447 1448 out0: 1449 spin_unlock_irqrestore(&dwc->lock, flags); 1450 1451 return ret; 1452 } 1453 1454 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol) 1455 { 1456 struct dwc3_gadget_ep_cmd_params params; 1457 struct dwc3 *dwc = dep->dwc; 1458 int ret; 1459 1460 if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) { 1461 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name); 1462 return -EINVAL; 1463 } 1464 1465 memset(¶ms, 0x00, sizeof(params)); 1466 1467 if (value) { 1468 struct dwc3_trb *trb; 1469 1470 unsigned transfer_in_flight; 1471 unsigned started; 1472 1473 if (dep->number > 1) 1474 trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue); 1475 else 1476 trb = &dwc->ep0_trb[dep->trb_enqueue]; 1477 1478 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO; 1479 started = !list_empty(&dep->started_list); 1480 1481 if (!protocol && ((dep->direction && transfer_in_flight) || 1482 (!dep->direction && started))) { 1483 return -EAGAIN; 1484 } 1485 1486 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL, 1487 ¶ms); 1488 if (ret) 1489 dev_err(dwc->dev, "failed to set STALL on %s\n", 1490 dep->name); 1491 else 1492 dep->flags |= DWC3_EP_STALL; 1493 } else { 1494 1495 ret = dwc3_send_clear_stall_ep_cmd(dep); 1496 if (ret) 1497 dev_err(dwc->dev, "failed to clear STALL on %s\n", 1498 dep->name); 1499 else 1500 dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE); 1501 } 1502 1503 return ret; 1504 } 1505 1506 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value) 1507 { 1508 struct dwc3_ep *dep = to_dwc3_ep(ep); 1509 struct dwc3 *dwc = dep->dwc; 1510 1511 unsigned long flags; 1512 1513 int ret; 1514 1515 spin_lock_irqsave(&dwc->lock, flags); 1516 ret = __dwc3_gadget_ep_set_halt(dep, value, false); 1517 spin_unlock_irqrestore(&dwc->lock, flags); 1518 1519 return ret; 1520 } 1521 1522 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep) 1523 { 1524 struct dwc3_ep *dep = to_dwc3_ep(ep); 1525 struct dwc3 *dwc = dep->dwc; 1526 unsigned long flags; 1527 int ret; 1528 1529 spin_lock_irqsave(&dwc->lock, flags); 1530 dep->flags |= DWC3_EP_WEDGE; 1531 1532 if (dep->number == 0 || dep->number == 1) 1533 ret = __dwc3_gadget_ep0_set_halt(ep, 1); 1534 else 1535 ret = __dwc3_gadget_ep_set_halt(dep, 1, false); 1536 spin_unlock_irqrestore(&dwc->lock, flags); 1537 1538 return ret; 1539 } 1540 1541 /* -------------------------------------------------------------------------- */ 1542 1543 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = { 1544 .bLength = USB_DT_ENDPOINT_SIZE, 1545 .bDescriptorType = USB_DT_ENDPOINT, 1546 .bmAttributes = USB_ENDPOINT_XFER_CONTROL, 1547 }; 1548 1549 static const struct usb_ep_ops dwc3_gadget_ep0_ops = { 1550 .enable = dwc3_gadget_ep0_enable, 1551 .disable = dwc3_gadget_ep0_disable, 1552 .alloc_request = dwc3_gadget_ep_alloc_request, 1553 .free_request = dwc3_gadget_ep_free_request, 1554 .queue = dwc3_gadget_ep0_queue, 1555 .dequeue = dwc3_gadget_ep_dequeue, 1556 .set_halt = dwc3_gadget_ep0_set_halt, 1557 .set_wedge = dwc3_gadget_ep_set_wedge, 1558 }; 1559 1560 static const struct usb_ep_ops dwc3_gadget_ep_ops = { 1561 .enable = dwc3_gadget_ep_enable, 1562 .disable = dwc3_gadget_ep_disable, 1563 .alloc_request = dwc3_gadget_ep_alloc_request, 1564 .free_request = dwc3_gadget_ep_free_request, 1565 .queue = dwc3_gadget_ep_queue, 1566 .dequeue = dwc3_gadget_ep_dequeue, 1567 .set_halt = dwc3_gadget_ep_set_halt, 1568 .set_wedge = dwc3_gadget_ep_set_wedge, 1569 }; 1570 1571 /* -------------------------------------------------------------------------- */ 1572 1573 static int dwc3_gadget_get_frame(struct usb_gadget *g) 1574 { 1575 struct dwc3 *dwc = gadget_to_dwc(g); 1576 1577 return __dwc3_gadget_get_frame(dwc); 1578 } 1579 1580 static int __dwc3_gadget_wakeup(struct dwc3 *dwc) 1581 { 1582 int retries; 1583 1584 int ret; 1585 u32 reg; 1586 1587 u8 link_state; 1588 u8 speed; 1589 1590 /* 1591 * According to the Databook Remote wakeup request should 1592 * be issued only when the device is in early suspend state. 1593 * 1594 * We can check that via USB Link State bits in DSTS register. 1595 */ 1596 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1597 1598 speed = reg & DWC3_DSTS_CONNECTSPD; 1599 if ((speed == DWC3_DSTS_SUPERSPEED) || 1600 (speed == DWC3_DSTS_SUPERSPEED_PLUS)) 1601 return 0; 1602 1603 link_state = DWC3_DSTS_USBLNKST(reg); 1604 1605 switch (link_state) { 1606 case DWC3_LINK_STATE_RX_DET: /* in HS, means Early Suspend */ 1607 case DWC3_LINK_STATE_U3: /* in HS, means SUSPEND */ 1608 break; 1609 default: 1610 return -EINVAL; 1611 } 1612 1613 ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV); 1614 if (ret < 0) { 1615 dev_err(dwc->dev, "failed to put link in Recovery\n"); 1616 return ret; 1617 } 1618 1619 /* Recent versions do this automatically */ 1620 if (dwc->revision < DWC3_REVISION_194A) { 1621 /* write zeroes to Link Change Request */ 1622 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 1623 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK; 1624 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 1625 } 1626 1627 /* poll until Link State changes to ON */ 1628 retries = 20000; 1629 1630 while (retries--) { 1631 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1632 1633 /* in HS, means ON */ 1634 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0) 1635 break; 1636 } 1637 1638 if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) { 1639 dev_err(dwc->dev, "failed to send remote wakeup\n"); 1640 return -EINVAL; 1641 } 1642 1643 return 0; 1644 } 1645 1646 static int dwc3_gadget_wakeup(struct usb_gadget *g) 1647 { 1648 struct dwc3 *dwc = gadget_to_dwc(g); 1649 unsigned long flags; 1650 int ret; 1651 1652 spin_lock_irqsave(&dwc->lock, flags); 1653 ret = __dwc3_gadget_wakeup(dwc); 1654 spin_unlock_irqrestore(&dwc->lock, flags); 1655 1656 return ret; 1657 } 1658 1659 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g, 1660 int is_selfpowered) 1661 { 1662 struct dwc3 *dwc = gadget_to_dwc(g); 1663 unsigned long flags; 1664 1665 spin_lock_irqsave(&dwc->lock, flags); 1666 g->is_selfpowered = !!is_selfpowered; 1667 spin_unlock_irqrestore(&dwc->lock, flags); 1668 1669 return 0; 1670 } 1671 1672 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend) 1673 { 1674 u32 reg; 1675 u32 timeout = 500; 1676 1677 if (pm_runtime_suspended(dwc->dev)) 1678 return 0; 1679 1680 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 1681 if (is_on) { 1682 if (dwc->revision <= DWC3_REVISION_187A) { 1683 reg &= ~DWC3_DCTL_TRGTULST_MASK; 1684 reg |= DWC3_DCTL_TRGTULST_RX_DET; 1685 } 1686 1687 if (dwc->revision >= DWC3_REVISION_194A) 1688 reg &= ~DWC3_DCTL_KEEP_CONNECT; 1689 reg |= DWC3_DCTL_RUN_STOP; 1690 1691 if (dwc->has_hibernation) 1692 reg |= DWC3_DCTL_KEEP_CONNECT; 1693 1694 dwc->pullups_connected = true; 1695 } else { 1696 reg &= ~DWC3_DCTL_RUN_STOP; 1697 1698 if (dwc->has_hibernation && !suspend) 1699 reg &= ~DWC3_DCTL_KEEP_CONNECT; 1700 1701 dwc->pullups_connected = false; 1702 } 1703 1704 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 1705 1706 do { 1707 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 1708 reg &= DWC3_DSTS_DEVCTRLHLT; 1709 } while (--timeout && !(!is_on ^ !reg)); 1710 1711 if (!timeout) 1712 return -ETIMEDOUT; 1713 1714 return 0; 1715 } 1716 1717 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on) 1718 { 1719 struct dwc3 *dwc = gadget_to_dwc(g); 1720 unsigned long flags; 1721 int ret; 1722 1723 is_on = !!is_on; 1724 1725 /* 1726 * Per databook, when we want to stop the gadget, if a control transfer 1727 * is still in process, complete it and get the core into setup phase. 1728 */ 1729 if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) { 1730 reinit_completion(&dwc->ep0_in_setup); 1731 1732 ret = wait_for_completion_timeout(&dwc->ep0_in_setup, 1733 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT)); 1734 if (ret == 0) { 1735 dev_err(dwc->dev, "timed out waiting for SETUP phase\n"); 1736 return -ETIMEDOUT; 1737 } 1738 } 1739 1740 spin_lock_irqsave(&dwc->lock, flags); 1741 ret = dwc3_gadget_run_stop(dwc, is_on, false); 1742 spin_unlock_irqrestore(&dwc->lock, flags); 1743 1744 return ret; 1745 } 1746 1747 static void dwc3_gadget_enable_irq(struct dwc3 *dwc) 1748 { 1749 u32 reg; 1750 1751 /* Enable all but Start and End of Frame IRQs */ 1752 reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN | 1753 DWC3_DEVTEN_EVNTOVERFLOWEN | 1754 DWC3_DEVTEN_CMDCMPLTEN | 1755 DWC3_DEVTEN_ERRTICERREN | 1756 DWC3_DEVTEN_WKUPEVTEN | 1757 DWC3_DEVTEN_CONNECTDONEEN | 1758 DWC3_DEVTEN_USBRSTEN | 1759 DWC3_DEVTEN_DISCONNEVTEN); 1760 1761 if (dwc->revision < DWC3_REVISION_250A) 1762 reg |= DWC3_DEVTEN_ULSTCNGEN; 1763 1764 dwc3_writel(dwc->regs, DWC3_DEVTEN, reg); 1765 } 1766 1767 static void dwc3_gadget_disable_irq(struct dwc3 *dwc) 1768 { 1769 /* mask all interrupts */ 1770 dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00); 1771 } 1772 1773 static irqreturn_t dwc3_interrupt(int irq, void *_dwc); 1774 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc); 1775 1776 /** 1777 * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG 1778 * @dwc: pointer to our context structure 1779 * 1780 * The following looks like complex but it's actually very simple. In order to 1781 * calculate the number of packets we can burst at once on OUT transfers, we're 1782 * gonna use RxFIFO size. 1783 * 1784 * To calculate RxFIFO size we need two numbers: 1785 * MDWIDTH = size, in bits, of the internal memory bus 1786 * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits) 1787 * 1788 * Given these two numbers, the formula is simple: 1789 * 1790 * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16; 1791 * 1792 * 24 bytes is for 3x SETUP packets 1793 * 16 bytes is a clock domain crossing tolerance 1794 * 1795 * Given RxFIFO Size, NUMP = RxFIFOSize / 1024; 1796 */ 1797 static void dwc3_gadget_setup_nump(struct dwc3 *dwc) 1798 { 1799 u32 ram2_depth; 1800 u32 mdwidth; 1801 u32 nump; 1802 u32 reg; 1803 1804 ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7); 1805 mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0); 1806 1807 nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024; 1808 nump = min_t(u32, nump, 16); 1809 1810 /* update NumP */ 1811 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 1812 reg &= ~DWC3_DCFG_NUMP_MASK; 1813 reg |= nump << DWC3_DCFG_NUMP_SHIFT; 1814 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 1815 } 1816 1817 static int __dwc3_gadget_start(struct dwc3 *dwc) 1818 { 1819 struct dwc3_ep *dep; 1820 int ret = 0; 1821 u32 reg; 1822 1823 /* 1824 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if 1825 * the core supports IMOD, disable it. 1826 */ 1827 if (dwc->imod_interval) { 1828 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 1829 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 1830 } else if (dwc3_has_imod(dwc)) { 1831 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0); 1832 } 1833 1834 /* 1835 * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP 1836 * field instead of letting dwc3 itself calculate that automatically. 1837 * 1838 * This way, we maximize the chances that we'll be able to get several 1839 * bursts of data without going through any sort of endpoint throttling. 1840 */ 1841 reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG); 1842 if (dwc3_is_usb31(dwc)) 1843 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL; 1844 else 1845 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL; 1846 1847 dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg); 1848 1849 dwc3_gadget_setup_nump(dwc); 1850 1851 /* Start with SuperSpeed Default */ 1852 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 1853 1854 dep = dwc->eps[0]; 1855 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 1856 if (ret) { 1857 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 1858 goto err0; 1859 } 1860 1861 dep = dwc->eps[1]; 1862 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT); 1863 if (ret) { 1864 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 1865 goto err1; 1866 } 1867 1868 /* begin to receive SETUP packets */ 1869 dwc->ep0state = EP0_SETUP_PHASE; 1870 dwc3_ep0_out_start(dwc); 1871 1872 dwc3_gadget_enable_irq(dwc); 1873 1874 return 0; 1875 1876 err1: 1877 __dwc3_gadget_ep_disable(dwc->eps[0]); 1878 1879 err0: 1880 return ret; 1881 } 1882 1883 static int dwc3_gadget_start(struct usb_gadget *g, 1884 struct usb_gadget_driver *driver) 1885 { 1886 struct dwc3 *dwc = gadget_to_dwc(g); 1887 unsigned long flags; 1888 int ret = 0; 1889 int irq; 1890 1891 irq = dwc->irq_gadget; 1892 ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt, 1893 IRQF_SHARED, "dwc3", dwc->ev_buf); 1894 if (ret) { 1895 dev_err(dwc->dev, "failed to request irq #%d --> %d\n", 1896 irq, ret); 1897 goto err0; 1898 } 1899 1900 spin_lock_irqsave(&dwc->lock, flags); 1901 if (dwc->gadget_driver) { 1902 dev_err(dwc->dev, "%s is already bound to %s\n", 1903 dwc->gadget.name, 1904 dwc->gadget_driver->driver.name); 1905 ret = -EBUSY; 1906 goto err1; 1907 } 1908 1909 dwc->gadget_driver = driver; 1910 1911 if (pm_runtime_active(dwc->dev)) 1912 __dwc3_gadget_start(dwc); 1913 1914 spin_unlock_irqrestore(&dwc->lock, flags); 1915 1916 return 0; 1917 1918 err1: 1919 spin_unlock_irqrestore(&dwc->lock, flags); 1920 free_irq(irq, dwc); 1921 1922 err0: 1923 return ret; 1924 } 1925 1926 static void __dwc3_gadget_stop(struct dwc3 *dwc) 1927 { 1928 dwc3_gadget_disable_irq(dwc); 1929 __dwc3_gadget_ep_disable(dwc->eps[0]); 1930 __dwc3_gadget_ep_disable(dwc->eps[1]); 1931 } 1932 1933 static int dwc3_gadget_stop(struct usb_gadget *g) 1934 { 1935 struct dwc3 *dwc = gadget_to_dwc(g); 1936 unsigned long flags; 1937 int epnum; 1938 u32 tmo_eps = 0; 1939 1940 spin_lock_irqsave(&dwc->lock, flags); 1941 1942 if (pm_runtime_suspended(dwc->dev)) 1943 goto out; 1944 1945 __dwc3_gadget_stop(dwc); 1946 1947 for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 1948 struct dwc3_ep *dep = dwc->eps[epnum]; 1949 int ret; 1950 1951 if (!dep) 1952 continue; 1953 1954 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 1955 continue; 1956 1957 ret = wait_event_interruptible_lock_irq_timeout(dep->wait_end_transfer, 1958 !(dep->flags & DWC3_EP_END_TRANSFER_PENDING), 1959 dwc->lock, msecs_to_jiffies(5)); 1960 1961 if (ret <= 0) { 1962 /* Timed out or interrupted! There's nothing much 1963 * we can do so we just log here and print which 1964 * endpoints timed out at the end. 1965 */ 1966 tmo_eps |= 1 << epnum; 1967 dep->flags &= DWC3_EP_END_TRANSFER_PENDING; 1968 } 1969 } 1970 1971 if (tmo_eps) { 1972 dev_err(dwc->dev, 1973 "end transfer timed out on endpoints 0x%x [bitmap]\n", 1974 tmo_eps); 1975 } 1976 1977 out: 1978 dwc->gadget_driver = NULL; 1979 spin_unlock_irqrestore(&dwc->lock, flags); 1980 1981 free_irq(dwc->irq_gadget, dwc->ev_buf); 1982 1983 return 0; 1984 } 1985 1986 static void dwc3_gadget_set_speed(struct usb_gadget *g, 1987 enum usb_device_speed speed) 1988 { 1989 struct dwc3 *dwc = gadget_to_dwc(g); 1990 unsigned long flags; 1991 u32 reg; 1992 1993 spin_lock_irqsave(&dwc->lock, flags); 1994 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 1995 reg &= ~(DWC3_DCFG_SPEED_MASK); 1996 1997 /* 1998 * WORKAROUND: DWC3 revision < 2.20a have an issue 1999 * which would cause metastability state on Run/Stop 2000 * bit if we try to force the IP to USB2-only mode. 2001 * 2002 * Because of that, we cannot configure the IP to any 2003 * speed other than the SuperSpeed 2004 * 2005 * Refers to: 2006 * 2007 * STAR#9000525659: Clock Domain Crossing on DCTL in 2008 * USB 2.0 Mode 2009 */ 2010 if (dwc->revision < DWC3_REVISION_220A && 2011 !dwc->dis_metastability_quirk) { 2012 reg |= DWC3_DCFG_SUPERSPEED; 2013 } else { 2014 switch (speed) { 2015 case USB_SPEED_LOW: 2016 reg |= DWC3_DCFG_LOWSPEED; 2017 break; 2018 case USB_SPEED_FULL: 2019 reg |= DWC3_DCFG_FULLSPEED; 2020 break; 2021 case USB_SPEED_HIGH: 2022 reg |= DWC3_DCFG_HIGHSPEED; 2023 break; 2024 case USB_SPEED_SUPER: 2025 reg |= DWC3_DCFG_SUPERSPEED; 2026 break; 2027 case USB_SPEED_SUPER_PLUS: 2028 if (dwc3_is_usb31(dwc)) 2029 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2030 else 2031 reg |= DWC3_DCFG_SUPERSPEED; 2032 break; 2033 default: 2034 dev_err(dwc->dev, "invalid speed (%d)\n", speed); 2035 2036 if (dwc->revision & DWC3_REVISION_IS_DWC31) 2037 reg |= DWC3_DCFG_SUPERSPEED_PLUS; 2038 else 2039 reg |= DWC3_DCFG_SUPERSPEED; 2040 } 2041 } 2042 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2043 2044 spin_unlock_irqrestore(&dwc->lock, flags); 2045 } 2046 2047 static const struct usb_gadget_ops dwc3_gadget_ops = { 2048 .get_frame = dwc3_gadget_get_frame, 2049 .wakeup = dwc3_gadget_wakeup, 2050 .set_selfpowered = dwc3_gadget_set_selfpowered, 2051 .pullup = dwc3_gadget_pullup, 2052 .udc_start = dwc3_gadget_start, 2053 .udc_stop = dwc3_gadget_stop, 2054 .udc_set_speed = dwc3_gadget_set_speed, 2055 }; 2056 2057 /* -------------------------------------------------------------------------- */ 2058 2059 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep) 2060 { 2061 struct dwc3 *dwc = dep->dwc; 2062 2063 usb_ep_set_maxpacket_limit(&dep->endpoint, 512); 2064 dep->endpoint.maxburst = 1; 2065 dep->endpoint.ops = &dwc3_gadget_ep0_ops; 2066 if (!dep->direction) 2067 dwc->gadget.ep0 = &dep->endpoint; 2068 2069 dep->endpoint.caps.type_control = true; 2070 2071 return 0; 2072 } 2073 2074 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep) 2075 { 2076 struct dwc3 *dwc = dep->dwc; 2077 int mdwidth; 2078 int kbytes; 2079 int size; 2080 2081 mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0); 2082 /* MDWIDTH is represented in bits, we need it in bytes */ 2083 mdwidth /= 8; 2084 2085 size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1)); 2086 if (dwc3_is_usb31(dwc)) 2087 size = DWC31_GTXFIFOSIZ_TXFDEF(size); 2088 else 2089 size = DWC3_GTXFIFOSIZ_TXFDEF(size); 2090 2091 /* FIFO Depth is in MDWDITH bytes. Multiply */ 2092 size *= mdwidth; 2093 2094 kbytes = size / 1024; 2095 if (kbytes == 0) 2096 kbytes = 1; 2097 2098 /* 2099 * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for 2100 * internal overhead. We don't really know how these are used, 2101 * but documentation say it exists. 2102 */ 2103 size -= mdwidth * (kbytes + 1); 2104 size /= kbytes; 2105 2106 usb_ep_set_maxpacket_limit(&dep->endpoint, size); 2107 2108 dep->endpoint.max_streams = 15; 2109 dep->endpoint.ops = &dwc3_gadget_ep_ops; 2110 list_add_tail(&dep->endpoint.ep_list, 2111 &dwc->gadget.ep_list); 2112 dep->endpoint.caps.type_iso = true; 2113 dep->endpoint.caps.type_bulk = true; 2114 dep->endpoint.caps.type_int = true; 2115 2116 return dwc3_alloc_trb_pool(dep); 2117 } 2118 2119 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep) 2120 { 2121 struct dwc3 *dwc = dep->dwc; 2122 2123 usb_ep_set_maxpacket_limit(&dep->endpoint, 1024); 2124 dep->endpoint.max_streams = 15; 2125 dep->endpoint.ops = &dwc3_gadget_ep_ops; 2126 list_add_tail(&dep->endpoint.ep_list, 2127 &dwc->gadget.ep_list); 2128 dep->endpoint.caps.type_iso = true; 2129 dep->endpoint.caps.type_bulk = true; 2130 dep->endpoint.caps.type_int = true; 2131 2132 return dwc3_alloc_trb_pool(dep); 2133 } 2134 2135 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum) 2136 { 2137 struct dwc3_ep *dep; 2138 bool direction = epnum & 1; 2139 int ret; 2140 u8 num = epnum >> 1; 2141 2142 dep = kzalloc(sizeof(*dep), GFP_KERNEL); 2143 if (!dep) 2144 return -ENOMEM; 2145 2146 dep->dwc = dwc; 2147 dep->number = epnum; 2148 dep->direction = direction; 2149 dep->regs = dwc->regs + DWC3_DEP_BASE(epnum); 2150 dwc->eps[epnum] = dep; 2151 2152 snprintf(dep->name, sizeof(dep->name), "ep%u%s", num, 2153 direction ? "in" : "out"); 2154 2155 dep->endpoint.name = dep->name; 2156 2157 if (!(dep->number > 1)) { 2158 dep->endpoint.desc = &dwc3_gadget_ep0_desc; 2159 dep->endpoint.comp_desc = NULL; 2160 } 2161 2162 spin_lock_init(&dep->lock); 2163 2164 if (num == 0) 2165 ret = dwc3_gadget_init_control_endpoint(dep); 2166 else if (direction) 2167 ret = dwc3_gadget_init_in_endpoint(dep); 2168 else 2169 ret = dwc3_gadget_init_out_endpoint(dep); 2170 2171 if (ret) 2172 return ret; 2173 2174 dep->endpoint.caps.dir_in = direction; 2175 dep->endpoint.caps.dir_out = !direction; 2176 2177 INIT_LIST_HEAD(&dep->pending_list); 2178 INIT_LIST_HEAD(&dep->started_list); 2179 2180 return 0; 2181 } 2182 2183 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total) 2184 { 2185 u8 epnum; 2186 2187 INIT_LIST_HEAD(&dwc->gadget.ep_list); 2188 2189 for (epnum = 0; epnum < total; epnum++) { 2190 int ret; 2191 2192 ret = dwc3_gadget_init_endpoint(dwc, epnum); 2193 if (ret) 2194 return ret; 2195 } 2196 2197 return 0; 2198 } 2199 2200 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc) 2201 { 2202 struct dwc3_ep *dep; 2203 u8 epnum; 2204 2205 for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 2206 dep = dwc->eps[epnum]; 2207 if (!dep) 2208 continue; 2209 /* 2210 * Physical endpoints 0 and 1 are special; they form the 2211 * bi-directional USB endpoint 0. 2212 * 2213 * For those two physical endpoints, we don't allocate a TRB 2214 * pool nor do we add them the endpoints list. Due to that, we 2215 * shouldn't do these two operations otherwise we would end up 2216 * with all sorts of bugs when removing dwc3.ko. 2217 */ 2218 if (epnum != 0 && epnum != 1) { 2219 dwc3_free_trb_pool(dep); 2220 list_del(&dep->endpoint.ep_list); 2221 } 2222 2223 kfree(dep); 2224 } 2225 } 2226 2227 /* -------------------------------------------------------------------------- */ 2228 2229 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep, 2230 struct dwc3_request *req, struct dwc3_trb *trb, 2231 const struct dwc3_event_depevt *event, int status, int chain) 2232 { 2233 unsigned int count; 2234 2235 dwc3_ep_inc_deq(dep); 2236 2237 trace_dwc3_complete_trb(dep, trb); 2238 2239 /* 2240 * If we're in the middle of series of chained TRBs and we 2241 * receive a short transfer along the way, DWC3 will skip 2242 * through all TRBs including the last TRB in the chain (the 2243 * where CHN bit is zero. DWC3 will also avoid clearing HWO 2244 * bit and SW has to do it manually. 2245 * 2246 * We're going to do that here to avoid problems of HW trying 2247 * to use bogus TRBs for transfers. 2248 */ 2249 if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO)) 2250 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 2251 2252 /* 2253 * If we're dealing with unaligned size OUT transfer, we will be left 2254 * with one TRB pending in the ring. We need to manually clear HWO bit 2255 * from that TRB. 2256 */ 2257 if ((req->zero || req->unaligned) && !(trb->ctrl & DWC3_TRB_CTRL_CHN)) { 2258 trb->ctrl &= ~DWC3_TRB_CTRL_HWO; 2259 return 1; 2260 } 2261 2262 count = trb->size & DWC3_TRB_SIZE_MASK; 2263 req->remaining += count; 2264 2265 if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN) 2266 return 1; 2267 2268 if (event->status & DEPEVT_STATUS_SHORT && !chain) 2269 return 1; 2270 2271 if (event->status & DEPEVT_STATUS_IOC) 2272 return 1; 2273 2274 return 0; 2275 } 2276 2277 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep, 2278 struct dwc3_request *req, const struct dwc3_event_depevt *event, 2279 int status) 2280 { 2281 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; 2282 struct scatterlist *sg = req->sg; 2283 struct scatterlist *s; 2284 unsigned int pending = req->num_pending_sgs; 2285 unsigned int i; 2286 int ret = 0; 2287 2288 for_each_sg(sg, s, pending, i) { 2289 trb = &dep->trb_pool[dep->trb_dequeue]; 2290 2291 if (trb->ctrl & DWC3_TRB_CTRL_HWO) 2292 break; 2293 2294 req->sg = sg_next(s); 2295 req->num_pending_sgs--; 2296 2297 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req, 2298 trb, event, status, true); 2299 if (ret) 2300 break; 2301 } 2302 2303 return ret; 2304 } 2305 2306 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep, 2307 struct dwc3_request *req, const struct dwc3_event_depevt *event, 2308 int status) 2309 { 2310 struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue]; 2311 2312 return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb, 2313 event, status, false); 2314 } 2315 2316 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req) 2317 { 2318 return req->request.actual == req->request.length; 2319 } 2320 2321 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep, 2322 const struct dwc3_event_depevt *event, 2323 struct dwc3_request *req, int status) 2324 { 2325 int ret; 2326 2327 if (req->num_pending_sgs) 2328 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event, 2329 status); 2330 else 2331 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, 2332 status); 2333 2334 if (req->unaligned || req->zero) { 2335 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event, 2336 status); 2337 req->unaligned = false; 2338 req->zero = false; 2339 } 2340 2341 req->request.actual = req->request.length - req->remaining; 2342 2343 if (!dwc3_gadget_ep_request_completed(req) && 2344 req->num_pending_sgs) { 2345 __dwc3_gadget_kick_transfer(dep); 2346 goto out; 2347 } 2348 2349 dwc3_gadget_giveback(dep, req, status); 2350 2351 out: 2352 return ret; 2353 } 2354 2355 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep, 2356 const struct dwc3_event_depevt *event, int status) 2357 { 2358 struct dwc3_request *req; 2359 struct dwc3_request *tmp; 2360 2361 list_for_each_entry_safe(req, tmp, &dep->started_list, list) { 2362 int ret; 2363 2364 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event, 2365 req, status); 2366 if (ret) 2367 break; 2368 } 2369 } 2370 2371 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep, 2372 const struct dwc3_event_depevt *event) 2373 { 2374 dep->frame_number = event->parameters; 2375 } 2376 2377 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep, 2378 const struct dwc3_event_depevt *event) 2379 { 2380 struct dwc3 *dwc = dep->dwc; 2381 unsigned status = 0; 2382 bool stop = false; 2383 2384 dwc3_gadget_endpoint_frame_from_event(dep, event); 2385 2386 if (event->status & DEPEVT_STATUS_BUSERR) 2387 status = -ECONNRESET; 2388 2389 if (event->status & DEPEVT_STATUS_MISSED_ISOC) { 2390 status = -EXDEV; 2391 2392 if (list_empty(&dep->started_list)) 2393 stop = true; 2394 } 2395 2396 dwc3_gadget_ep_cleanup_completed_requests(dep, event, status); 2397 2398 if (stop) { 2399 dwc3_stop_active_transfer(dep, true); 2400 dep->flags = DWC3_EP_ENABLED; 2401 } 2402 2403 /* 2404 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround. 2405 * See dwc3_gadget_linksts_change_interrupt() for 1st half. 2406 */ 2407 if (dwc->revision < DWC3_REVISION_183A) { 2408 u32 reg; 2409 int i; 2410 2411 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) { 2412 dep = dwc->eps[i]; 2413 2414 if (!(dep->flags & DWC3_EP_ENABLED)) 2415 continue; 2416 2417 if (!list_empty(&dep->started_list)) 2418 return; 2419 } 2420 2421 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2422 reg |= dwc->u1u2; 2423 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2424 2425 dwc->u1u2 = 0; 2426 } 2427 } 2428 2429 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep, 2430 const struct dwc3_event_depevt *event) 2431 { 2432 dwc3_gadget_endpoint_frame_from_event(dep, event); 2433 __dwc3_gadget_start_isoc(dep); 2434 } 2435 2436 static void dwc3_endpoint_interrupt(struct dwc3 *dwc, 2437 const struct dwc3_event_depevt *event) 2438 { 2439 struct dwc3_ep *dep; 2440 u8 epnum = event->endpoint_number; 2441 u8 cmd; 2442 2443 dep = dwc->eps[epnum]; 2444 2445 if (!(dep->flags & DWC3_EP_ENABLED)) { 2446 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING)) 2447 return; 2448 2449 /* Handle only EPCMDCMPLT when EP disabled */ 2450 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) 2451 return; 2452 } 2453 2454 if (epnum == 0 || epnum == 1) { 2455 dwc3_ep0_interrupt(dwc, event); 2456 return; 2457 } 2458 2459 switch (event->endpoint_event) { 2460 case DWC3_DEPEVT_XFERINPROGRESS: 2461 dwc3_gadget_endpoint_transfer_in_progress(dep, event); 2462 break; 2463 case DWC3_DEPEVT_XFERNOTREADY: 2464 dwc3_gadget_endpoint_transfer_not_ready(dep, event); 2465 break; 2466 case DWC3_DEPEVT_EPCMDCMPLT: 2467 cmd = DEPEVT_PARAMETER_CMD(event->parameters); 2468 2469 if (cmd == DWC3_DEPCMD_ENDTRANSFER) { 2470 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING; 2471 wake_up(&dep->wait_end_transfer); 2472 } 2473 break; 2474 case DWC3_DEPEVT_STREAMEVT: 2475 case DWC3_DEPEVT_XFERCOMPLETE: 2476 case DWC3_DEPEVT_RXTXFIFOEVT: 2477 break; 2478 } 2479 } 2480 2481 static void dwc3_disconnect_gadget(struct dwc3 *dwc) 2482 { 2483 if (dwc->gadget_driver && dwc->gadget_driver->disconnect) { 2484 spin_unlock(&dwc->lock); 2485 dwc->gadget_driver->disconnect(&dwc->gadget); 2486 spin_lock(&dwc->lock); 2487 } 2488 } 2489 2490 static void dwc3_suspend_gadget(struct dwc3 *dwc) 2491 { 2492 if (dwc->gadget_driver && dwc->gadget_driver->suspend) { 2493 spin_unlock(&dwc->lock); 2494 dwc->gadget_driver->suspend(&dwc->gadget); 2495 spin_lock(&dwc->lock); 2496 } 2497 } 2498 2499 static void dwc3_resume_gadget(struct dwc3 *dwc) 2500 { 2501 if (dwc->gadget_driver && dwc->gadget_driver->resume) { 2502 spin_unlock(&dwc->lock); 2503 dwc->gadget_driver->resume(&dwc->gadget); 2504 spin_lock(&dwc->lock); 2505 } 2506 } 2507 2508 static void dwc3_reset_gadget(struct dwc3 *dwc) 2509 { 2510 if (!dwc->gadget_driver) 2511 return; 2512 2513 if (dwc->gadget.speed != USB_SPEED_UNKNOWN) { 2514 spin_unlock(&dwc->lock); 2515 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver); 2516 spin_lock(&dwc->lock); 2517 } 2518 } 2519 2520 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force) 2521 { 2522 struct dwc3 *dwc = dep->dwc; 2523 struct dwc3_gadget_ep_cmd_params params; 2524 u32 cmd; 2525 int ret; 2526 2527 if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) || 2528 !dep->resource_index) 2529 return; 2530 2531 /* 2532 * NOTICE: We are violating what the Databook says about the 2533 * EndTransfer command. Ideally we would _always_ wait for the 2534 * EndTransfer Command Completion IRQ, but that's causing too 2535 * much trouble synchronizing between us and gadget driver. 2536 * 2537 * We have discussed this with the IP Provider and it was 2538 * suggested to giveback all requests here, but give HW some 2539 * extra time to synchronize with the interconnect. We're using 2540 * an arbitrary 100us delay for that. 2541 * 2542 * Note also that a similar handling was tested by Synopsys 2543 * (thanks a lot Paul) and nothing bad has come out of it. 2544 * In short, what we're doing is: 2545 * 2546 * - Issue EndTransfer WITH CMDIOC bit set 2547 * - Wait 100us 2548 * 2549 * As of IP version 3.10a of the DWC_usb3 IP, the controller 2550 * supports a mode to work around the above limitation. The 2551 * software can poll the CMDACT bit in the DEPCMD register 2552 * after issuing a EndTransfer command. This mode is enabled 2553 * by writing GUCTL2[14]. This polling is already done in the 2554 * dwc3_send_gadget_ep_cmd() function so if the mode is 2555 * enabled, the EndTransfer command will have completed upon 2556 * returning from this function and we don't need to delay for 2557 * 100us. 2558 * 2559 * This mode is NOT available on the DWC_usb31 IP. 2560 */ 2561 2562 cmd = DWC3_DEPCMD_ENDTRANSFER; 2563 cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0; 2564 cmd |= DWC3_DEPCMD_CMDIOC; 2565 cmd |= DWC3_DEPCMD_PARAM(dep->resource_index); 2566 memset(¶ms, 0, sizeof(params)); 2567 ret = dwc3_send_gadget_ep_cmd(dep, cmd, ¶ms); 2568 WARN_ON_ONCE(ret); 2569 dep->resource_index = 0; 2570 2571 if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) { 2572 dep->flags |= DWC3_EP_END_TRANSFER_PENDING; 2573 udelay(100); 2574 } 2575 } 2576 2577 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc) 2578 { 2579 u32 epnum; 2580 2581 for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) { 2582 struct dwc3_ep *dep; 2583 int ret; 2584 2585 dep = dwc->eps[epnum]; 2586 if (!dep) 2587 continue; 2588 2589 if (!(dep->flags & DWC3_EP_STALL)) 2590 continue; 2591 2592 dep->flags &= ~DWC3_EP_STALL; 2593 2594 ret = dwc3_send_clear_stall_ep_cmd(dep); 2595 WARN_ON_ONCE(ret); 2596 } 2597 } 2598 2599 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc) 2600 { 2601 int reg; 2602 2603 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2604 reg &= ~DWC3_DCTL_INITU1ENA; 2605 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2606 2607 reg &= ~DWC3_DCTL_INITU2ENA; 2608 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2609 2610 dwc3_disconnect_gadget(dwc); 2611 2612 dwc->gadget.speed = USB_SPEED_UNKNOWN; 2613 dwc->setup_packet_pending = false; 2614 usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED); 2615 2616 dwc->connected = false; 2617 } 2618 2619 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc) 2620 { 2621 u32 reg; 2622 2623 dwc->connected = true; 2624 2625 /* 2626 * WORKAROUND: DWC3 revisions <1.88a have an issue which 2627 * would cause a missing Disconnect Event if there's a 2628 * pending Setup Packet in the FIFO. 2629 * 2630 * There's no suggested workaround on the official Bug 2631 * report, which states that "unless the driver/application 2632 * is doing any special handling of a disconnect event, 2633 * there is no functional issue". 2634 * 2635 * Unfortunately, it turns out that we _do_ some special 2636 * handling of a disconnect event, namely complete all 2637 * pending transfers, notify gadget driver of the 2638 * disconnection, and so on. 2639 * 2640 * Our suggested workaround is to follow the Disconnect 2641 * Event steps here, instead, based on a setup_packet_pending 2642 * flag. Such flag gets set whenever we have a SETUP_PENDING 2643 * status for EP0 TRBs and gets cleared on XferComplete for the 2644 * same endpoint. 2645 * 2646 * Refers to: 2647 * 2648 * STAR#9000466709: RTL: Device : Disconnect event not 2649 * generated if setup packet pending in FIFO 2650 */ 2651 if (dwc->revision < DWC3_REVISION_188A) { 2652 if (dwc->setup_packet_pending) 2653 dwc3_gadget_disconnect_interrupt(dwc); 2654 } 2655 2656 dwc3_reset_gadget(dwc); 2657 2658 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2659 reg &= ~DWC3_DCTL_TSTCTRL_MASK; 2660 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2661 dwc->test_mode = false; 2662 dwc3_clear_stall_all_ep(dwc); 2663 2664 /* Reset device address to zero */ 2665 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2666 reg &= ~(DWC3_DCFG_DEVADDR_MASK); 2667 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2668 } 2669 2670 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc) 2671 { 2672 struct dwc3_ep *dep; 2673 int ret; 2674 u32 reg; 2675 u8 speed; 2676 2677 reg = dwc3_readl(dwc->regs, DWC3_DSTS); 2678 speed = reg & DWC3_DSTS_CONNECTSPD; 2679 dwc->speed = speed; 2680 2681 /* 2682 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed 2683 * each time on Connect Done. 2684 * 2685 * Currently we always use the reset value. If any platform 2686 * wants to set this to a different value, we need to add a 2687 * setting and update GCTL.RAMCLKSEL here. 2688 */ 2689 2690 switch (speed) { 2691 case DWC3_DSTS_SUPERSPEED_PLUS: 2692 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 2693 dwc->gadget.ep0->maxpacket = 512; 2694 dwc->gadget.speed = USB_SPEED_SUPER_PLUS; 2695 break; 2696 case DWC3_DSTS_SUPERSPEED: 2697 /* 2698 * WORKAROUND: DWC3 revisions <1.90a have an issue which 2699 * would cause a missing USB3 Reset event. 2700 * 2701 * In such situations, we should force a USB3 Reset 2702 * event by calling our dwc3_gadget_reset_interrupt() 2703 * routine. 2704 * 2705 * Refers to: 2706 * 2707 * STAR#9000483510: RTL: SS : USB3 reset event may 2708 * not be generated always when the link enters poll 2709 */ 2710 if (dwc->revision < DWC3_REVISION_190A) 2711 dwc3_gadget_reset_interrupt(dwc); 2712 2713 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512); 2714 dwc->gadget.ep0->maxpacket = 512; 2715 dwc->gadget.speed = USB_SPEED_SUPER; 2716 break; 2717 case DWC3_DSTS_HIGHSPEED: 2718 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 2719 dwc->gadget.ep0->maxpacket = 64; 2720 dwc->gadget.speed = USB_SPEED_HIGH; 2721 break; 2722 case DWC3_DSTS_FULLSPEED: 2723 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64); 2724 dwc->gadget.ep0->maxpacket = 64; 2725 dwc->gadget.speed = USB_SPEED_FULL; 2726 break; 2727 case DWC3_DSTS_LOWSPEED: 2728 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8); 2729 dwc->gadget.ep0->maxpacket = 8; 2730 dwc->gadget.speed = USB_SPEED_LOW; 2731 break; 2732 } 2733 2734 dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket; 2735 2736 /* Enable USB2 LPM Capability */ 2737 2738 if ((dwc->revision > DWC3_REVISION_194A) && 2739 (speed != DWC3_DSTS_SUPERSPEED) && 2740 (speed != DWC3_DSTS_SUPERSPEED_PLUS)) { 2741 reg = dwc3_readl(dwc->regs, DWC3_DCFG); 2742 reg |= DWC3_DCFG_LPM_CAP; 2743 dwc3_writel(dwc->regs, DWC3_DCFG, reg); 2744 2745 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2746 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN); 2747 2748 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold); 2749 2750 /* 2751 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and 2752 * DCFG.LPMCap is set, core responses with an ACK and the 2753 * BESL value in the LPM token is less than or equal to LPM 2754 * NYET threshold. 2755 */ 2756 WARN_ONCE(dwc->revision < DWC3_REVISION_240A 2757 && dwc->has_lpm_erratum, 2758 "LPM Erratum not available on dwc3 revisions < 2.40a\n"); 2759 2760 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A) 2761 reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold); 2762 2763 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2764 } else { 2765 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2766 reg &= ~DWC3_DCTL_HIRD_THRES_MASK; 2767 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2768 } 2769 2770 dep = dwc->eps[0]; 2771 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 2772 if (ret) { 2773 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2774 return; 2775 } 2776 2777 dep = dwc->eps[1]; 2778 ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY); 2779 if (ret) { 2780 dev_err(dwc->dev, "failed to enable %s\n", dep->name); 2781 return; 2782 } 2783 2784 /* 2785 * Configure PHY via GUSB3PIPECTLn if required. 2786 * 2787 * Update GTXFIFOSIZn 2788 * 2789 * In both cases reset values should be sufficient. 2790 */ 2791 } 2792 2793 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc) 2794 { 2795 /* 2796 * TODO take core out of low power mode when that's 2797 * implemented. 2798 */ 2799 2800 if (dwc->gadget_driver && dwc->gadget_driver->resume) { 2801 spin_unlock(&dwc->lock); 2802 dwc->gadget_driver->resume(&dwc->gadget); 2803 spin_lock(&dwc->lock); 2804 } 2805 } 2806 2807 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc, 2808 unsigned int evtinfo) 2809 { 2810 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 2811 unsigned int pwropt; 2812 2813 /* 2814 * WORKAROUND: DWC3 < 2.50a have an issue when configured without 2815 * Hibernation mode enabled which would show up when device detects 2816 * host-initiated U3 exit. 2817 * 2818 * In that case, device will generate a Link State Change Interrupt 2819 * from U3 to RESUME which is only necessary if Hibernation is 2820 * configured in. 2821 * 2822 * There are no functional changes due to such spurious event and we 2823 * just need to ignore it. 2824 * 2825 * Refers to: 2826 * 2827 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation 2828 * operational mode 2829 */ 2830 pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1); 2831 if ((dwc->revision < DWC3_REVISION_250A) && 2832 (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) { 2833 if ((dwc->link_state == DWC3_LINK_STATE_U3) && 2834 (next == DWC3_LINK_STATE_RESUME)) { 2835 return; 2836 } 2837 } 2838 2839 /* 2840 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending 2841 * on the link partner, the USB session might do multiple entry/exit 2842 * of low power states before a transfer takes place. 2843 * 2844 * Due to this problem, we might experience lower throughput. The 2845 * suggested workaround is to disable DCTL[12:9] bits if we're 2846 * transitioning from U1/U2 to U0 and enable those bits again 2847 * after a transfer completes and there are no pending transfers 2848 * on any of the enabled endpoints. 2849 * 2850 * This is the first half of that workaround. 2851 * 2852 * Refers to: 2853 * 2854 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us 2855 * core send LGO_Ux entering U0 2856 */ 2857 if (dwc->revision < DWC3_REVISION_183A) { 2858 if (next == DWC3_LINK_STATE_U0) { 2859 u32 u1u2; 2860 u32 reg; 2861 2862 switch (dwc->link_state) { 2863 case DWC3_LINK_STATE_U1: 2864 case DWC3_LINK_STATE_U2: 2865 reg = dwc3_readl(dwc->regs, DWC3_DCTL); 2866 u1u2 = reg & (DWC3_DCTL_INITU2ENA 2867 | DWC3_DCTL_ACCEPTU2ENA 2868 | DWC3_DCTL_INITU1ENA 2869 | DWC3_DCTL_ACCEPTU1ENA); 2870 2871 if (!dwc->u1u2) 2872 dwc->u1u2 = reg & u1u2; 2873 2874 reg &= ~u1u2; 2875 2876 dwc3_writel(dwc->regs, DWC3_DCTL, reg); 2877 break; 2878 default: 2879 /* do nothing */ 2880 break; 2881 } 2882 } 2883 } 2884 2885 switch (next) { 2886 case DWC3_LINK_STATE_U1: 2887 if (dwc->speed == USB_SPEED_SUPER) 2888 dwc3_suspend_gadget(dwc); 2889 break; 2890 case DWC3_LINK_STATE_U2: 2891 case DWC3_LINK_STATE_U3: 2892 dwc3_suspend_gadget(dwc); 2893 break; 2894 case DWC3_LINK_STATE_RESUME: 2895 dwc3_resume_gadget(dwc); 2896 break; 2897 default: 2898 /* do nothing */ 2899 break; 2900 } 2901 2902 dwc->link_state = next; 2903 } 2904 2905 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc, 2906 unsigned int evtinfo) 2907 { 2908 enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK; 2909 2910 if (dwc->link_state != next && next == DWC3_LINK_STATE_U3) 2911 dwc3_suspend_gadget(dwc); 2912 2913 dwc->link_state = next; 2914 } 2915 2916 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc, 2917 unsigned int evtinfo) 2918 { 2919 unsigned int is_ss = evtinfo & BIT(4); 2920 2921 /* 2922 * WORKAROUND: DWC3 revison 2.20a with hibernation support 2923 * have a known issue which can cause USB CV TD.9.23 to fail 2924 * randomly. 2925 * 2926 * Because of this issue, core could generate bogus hibernation 2927 * events which SW needs to ignore. 2928 * 2929 * Refers to: 2930 * 2931 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0 2932 * Device Fallback from SuperSpeed 2933 */ 2934 if (is_ss ^ (dwc->speed == USB_SPEED_SUPER)) 2935 return; 2936 2937 /* enter hibernation here */ 2938 } 2939 2940 static void dwc3_gadget_interrupt(struct dwc3 *dwc, 2941 const struct dwc3_event_devt *event) 2942 { 2943 switch (event->type) { 2944 case DWC3_DEVICE_EVENT_DISCONNECT: 2945 dwc3_gadget_disconnect_interrupt(dwc); 2946 break; 2947 case DWC3_DEVICE_EVENT_RESET: 2948 dwc3_gadget_reset_interrupt(dwc); 2949 break; 2950 case DWC3_DEVICE_EVENT_CONNECT_DONE: 2951 dwc3_gadget_conndone_interrupt(dwc); 2952 break; 2953 case DWC3_DEVICE_EVENT_WAKEUP: 2954 dwc3_gadget_wakeup_interrupt(dwc); 2955 break; 2956 case DWC3_DEVICE_EVENT_HIBER_REQ: 2957 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation, 2958 "unexpected hibernation event\n")) 2959 break; 2960 2961 dwc3_gadget_hibernation_interrupt(dwc, event->event_info); 2962 break; 2963 case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE: 2964 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info); 2965 break; 2966 case DWC3_DEVICE_EVENT_EOPF: 2967 /* It changed to be suspend event for version 2.30a and above */ 2968 if (dwc->revision >= DWC3_REVISION_230A) { 2969 /* 2970 * Ignore suspend event until the gadget enters into 2971 * USB_STATE_CONFIGURED state. 2972 */ 2973 if (dwc->gadget.state >= USB_STATE_CONFIGURED) 2974 dwc3_gadget_suspend_interrupt(dwc, 2975 event->event_info); 2976 } 2977 break; 2978 case DWC3_DEVICE_EVENT_SOF: 2979 case DWC3_DEVICE_EVENT_ERRATIC_ERROR: 2980 case DWC3_DEVICE_EVENT_CMD_CMPL: 2981 case DWC3_DEVICE_EVENT_OVERFLOW: 2982 break; 2983 default: 2984 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type); 2985 } 2986 } 2987 2988 static void dwc3_process_event_entry(struct dwc3 *dwc, 2989 const union dwc3_event *event) 2990 { 2991 trace_dwc3_event(event->raw, dwc); 2992 2993 if (!event->type.is_devspec) 2994 dwc3_endpoint_interrupt(dwc, &event->depevt); 2995 else if (event->type.type == DWC3_EVENT_TYPE_DEV) 2996 dwc3_gadget_interrupt(dwc, &event->devt); 2997 else 2998 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw); 2999 } 3000 3001 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt) 3002 { 3003 struct dwc3 *dwc = evt->dwc; 3004 irqreturn_t ret = IRQ_NONE; 3005 int left; 3006 u32 reg; 3007 3008 left = evt->count; 3009 3010 if (!(evt->flags & DWC3_EVENT_PENDING)) 3011 return IRQ_NONE; 3012 3013 while (left > 0) { 3014 union dwc3_event event; 3015 3016 event.raw = *(u32 *) (evt->cache + evt->lpos); 3017 3018 dwc3_process_event_entry(dwc, &event); 3019 3020 /* 3021 * FIXME we wrap around correctly to the next entry as 3022 * almost all entries are 4 bytes in size. There is one 3023 * entry which has 12 bytes which is a regular entry 3024 * followed by 8 bytes data. ATM I don't know how 3025 * things are organized if we get next to the a 3026 * boundary so I worry about that once we try to handle 3027 * that. 3028 */ 3029 evt->lpos = (evt->lpos + 4) % evt->length; 3030 left -= 4; 3031 } 3032 3033 evt->count = 0; 3034 evt->flags &= ~DWC3_EVENT_PENDING; 3035 ret = IRQ_HANDLED; 3036 3037 /* Unmask interrupt */ 3038 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); 3039 reg &= ~DWC3_GEVNTSIZ_INTMASK; 3040 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); 3041 3042 if (dwc->imod_interval) { 3043 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB); 3044 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval); 3045 } 3046 3047 return ret; 3048 } 3049 3050 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt) 3051 { 3052 struct dwc3_event_buffer *evt = _evt; 3053 struct dwc3 *dwc = evt->dwc; 3054 unsigned long flags; 3055 irqreturn_t ret = IRQ_NONE; 3056 3057 spin_lock_irqsave(&dwc->lock, flags); 3058 ret = dwc3_process_event_buf(evt); 3059 spin_unlock_irqrestore(&dwc->lock, flags); 3060 3061 return ret; 3062 } 3063 3064 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt) 3065 { 3066 struct dwc3 *dwc = evt->dwc; 3067 u32 amount; 3068 u32 count; 3069 u32 reg; 3070 3071 if (pm_runtime_suspended(dwc->dev)) { 3072 pm_runtime_get(dwc->dev); 3073 disable_irq_nosync(dwc->irq_gadget); 3074 dwc->pending_events = true; 3075 return IRQ_HANDLED; 3076 } 3077 3078 /* 3079 * With PCIe legacy interrupt, test shows that top-half irq handler can 3080 * be called again after HW interrupt deassertion. Check if bottom-half 3081 * irq event handler completes before caching new event to prevent 3082 * losing events. 3083 */ 3084 if (evt->flags & DWC3_EVENT_PENDING) 3085 return IRQ_HANDLED; 3086 3087 count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0)); 3088 count &= DWC3_GEVNTCOUNT_MASK; 3089 if (!count) 3090 return IRQ_NONE; 3091 3092 evt->count = count; 3093 evt->flags |= DWC3_EVENT_PENDING; 3094 3095 /* Mask interrupt */ 3096 reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0)); 3097 reg |= DWC3_GEVNTSIZ_INTMASK; 3098 dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg); 3099 3100 amount = min(count, evt->length - evt->lpos); 3101 memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount); 3102 3103 if (amount < count) 3104 memcpy(evt->cache, evt->buf, count - amount); 3105 3106 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count); 3107 3108 return IRQ_WAKE_THREAD; 3109 } 3110 3111 static irqreturn_t dwc3_interrupt(int irq, void *_evt) 3112 { 3113 struct dwc3_event_buffer *evt = _evt; 3114 3115 return dwc3_check_event_buf(evt); 3116 } 3117 3118 static int dwc3_gadget_get_irq(struct dwc3 *dwc) 3119 { 3120 struct platform_device *dwc3_pdev = to_platform_device(dwc->dev); 3121 int irq; 3122 3123 irq = platform_get_irq_byname(dwc3_pdev, "peripheral"); 3124 if (irq > 0) 3125 goto out; 3126 3127 if (irq == -EPROBE_DEFER) 3128 goto out; 3129 3130 irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3"); 3131 if (irq > 0) 3132 goto out; 3133 3134 if (irq == -EPROBE_DEFER) 3135 goto out; 3136 3137 irq = platform_get_irq(dwc3_pdev, 0); 3138 if (irq > 0) 3139 goto out; 3140 3141 if (irq != -EPROBE_DEFER) 3142 dev_err(dwc->dev, "missing peripheral IRQ\n"); 3143 3144 if (!irq) 3145 irq = -EINVAL; 3146 3147 out: 3148 return irq; 3149 } 3150 3151 /** 3152 * dwc3_gadget_init - initializes gadget related registers 3153 * @dwc: pointer to our controller context structure 3154 * 3155 * Returns 0 on success otherwise negative errno. 3156 */ 3157 int dwc3_gadget_init(struct dwc3 *dwc) 3158 { 3159 int ret; 3160 int irq; 3161 3162 irq = dwc3_gadget_get_irq(dwc); 3163 if (irq < 0) { 3164 ret = irq; 3165 goto err0; 3166 } 3167 3168 dwc->irq_gadget = irq; 3169 3170 dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev, 3171 sizeof(*dwc->ep0_trb) * 2, 3172 &dwc->ep0_trb_addr, GFP_KERNEL); 3173 if (!dwc->ep0_trb) { 3174 dev_err(dwc->dev, "failed to allocate ep0 trb\n"); 3175 ret = -ENOMEM; 3176 goto err0; 3177 } 3178 3179 dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL); 3180 if (!dwc->setup_buf) { 3181 ret = -ENOMEM; 3182 goto err1; 3183 } 3184 3185 dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, 3186 &dwc->bounce_addr, GFP_KERNEL); 3187 if (!dwc->bounce) { 3188 ret = -ENOMEM; 3189 goto err2; 3190 } 3191 3192 init_completion(&dwc->ep0_in_setup); 3193 3194 dwc->gadget.ops = &dwc3_gadget_ops; 3195 dwc->gadget.speed = USB_SPEED_UNKNOWN; 3196 dwc->gadget.sg_supported = true; 3197 dwc->gadget.name = "dwc3-gadget"; 3198 dwc->gadget.is_otg = dwc->dr_mode == USB_DR_MODE_OTG; 3199 3200 /* 3201 * FIXME We might be setting max_speed to <SUPER, however versions 3202 * <2.20a of dwc3 have an issue with metastability (documented 3203 * elsewhere in this driver) which tells us we can't set max speed to 3204 * anything lower than SUPER. 3205 * 3206 * Because gadget.max_speed is only used by composite.c and function 3207 * drivers (i.e. it won't go into dwc3's registers) we are allowing this 3208 * to happen so we avoid sending SuperSpeed Capability descriptor 3209 * together with our BOS descriptor as that could confuse host into 3210 * thinking we can handle super speed. 3211 * 3212 * Note that, in fact, we won't even support GetBOS requests when speed 3213 * is less than super speed because we don't have means, yet, to tell 3214 * composite.c that we are USB 2.0 + LPM ECN. 3215 */ 3216 if (dwc->revision < DWC3_REVISION_220A && 3217 !dwc->dis_metastability_quirk) 3218 dev_info(dwc->dev, "changing max_speed on rev %08x\n", 3219 dwc->revision); 3220 3221 dwc->gadget.max_speed = dwc->maximum_speed; 3222 3223 /* 3224 * REVISIT: Here we should clear all pending IRQs to be 3225 * sure we're starting from a well known location. 3226 */ 3227 3228 ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps); 3229 if (ret) 3230 goto err3; 3231 3232 ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget); 3233 if (ret) { 3234 dev_err(dwc->dev, "failed to register udc\n"); 3235 goto err4; 3236 } 3237 3238 return 0; 3239 3240 err4: 3241 dwc3_gadget_free_endpoints(dwc); 3242 3243 err3: 3244 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 3245 dwc->bounce_addr); 3246 3247 err2: 3248 kfree(dwc->setup_buf); 3249 3250 err1: 3251 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 3252 dwc->ep0_trb, dwc->ep0_trb_addr); 3253 3254 err0: 3255 return ret; 3256 } 3257 3258 /* -------------------------------------------------------------------------- */ 3259 3260 void dwc3_gadget_exit(struct dwc3 *dwc) 3261 { 3262 usb_del_gadget_udc(&dwc->gadget); 3263 dwc3_gadget_free_endpoints(dwc); 3264 dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce, 3265 dwc->bounce_addr); 3266 kfree(dwc->setup_buf); 3267 dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2, 3268 dwc->ep0_trb, dwc->ep0_trb_addr); 3269 } 3270 3271 int dwc3_gadget_suspend(struct dwc3 *dwc) 3272 { 3273 if (!dwc->gadget_driver) 3274 return 0; 3275 3276 dwc3_gadget_run_stop(dwc, false, false); 3277 dwc3_disconnect_gadget(dwc); 3278 __dwc3_gadget_stop(dwc); 3279 3280 return 0; 3281 } 3282 3283 int dwc3_gadget_resume(struct dwc3 *dwc) 3284 { 3285 int ret; 3286 3287 if (!dwc->gadget_driver) 3288 return 0; 3289 3290 ret = __dwc3_gadget_start(dwc); 3291 if (ret < 0) 3292 goto err0; 3293 3294 ret = dwc3_gadget_run_stop(dwc, true, false); 3295 if (ret < 0) 3296 goto err1; 3297 3298 return 0; 3299 3300 err1: 3301 __dwc3_gadget_stop(dwc); 3302 3303 err0: 3304 return ret; 3305 } 3306 3307 void dwc3_gadget_process_pending_events(struct dwc3 *dwc) 3308 { 3309 if (dwc->pending_events) { 3310 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf); 3311 dwc->pending_events = false; 3312 enable_irq(dwc->irq_gadget); 3313 } 3314 } 3315