1 /** 2 * Copyright (c) 2011 Samsung Electronics Co., Ltd. 3 * http://www.samsung.com 4 * 5 * Copyright 2008 Openmoko, Inc. 6 * Copyright 2008 Simtec Electronics 7 * Ben Dooks <ben@simtec.co.uk> 8 * http://armlinux.simtec.co.uk/ 9 * 10 * S3C USB2.0 High-speed / OtG driver 11 * 12 * This program is free software; you can redistribute it and/or modify 13 * it under the terms of the GNU General Public License version 2 as 14 * published by the Free Software Foundation. 15 */ 16 17 #include <linux/kernel.h> 18 #include <linux/module.h> 19 #include <linux/spinlock.h> 20 #include <linux/interrupt.h> 21 #include <linux/platform_device.h> 22 #include <linux/dma-mapping.h> 23 #include <linux/mutex.h> 24 #include <linux/seq_file.h> 25 #include <linux/delay.h> 26 #include <linux/io.h> 27 #include <linux/slab.h> 28 #include <linux/of_platform.h> 29 30 #include <linux/usb/ch9.h> 31 #include <linux/usb/gadget.h> 32 #include <linux/usb/phy.h> 33 34 #include "core.h" 35 #include "hw.h" 36 37 /* conversion functions */ 38 static inline struct dwc2_hsotg_req *our_req(struct usb_request *req) 39 { 40 return container_of(req, struct dwc2_hsotg_req, req); 41 } 42 43 static inline struct dwc2_hsotg_ep *our_ep(struct usb_ep *ep) 44 { 45 return container_of(ep, struct dwc2_hsotg_ep, ep); 46 } 47 48 static inline struct dwc2_hsotg *to_hsotg(struct usb_gadget *gadget) 49 { 50 return container_of(gadget, struct dwc2_hsotg, gadget); 51 } 52 53 static inline void __orr32(void __iomem *ptr, u32 val) 54 { 55 dwc2_writel(dwc2_readl(ptr) | val, ptr); 56 } 57 58 static inline void __bic32(void __iomem *ptr, u32 val) 59 { 60 dwc2_writel(dwc2_readl(ptr) & ~val, ptr); 61 } 62 63 static inline struct dwc2_hsotg_ep *index_to_ep(struct dwc2_hsotg *hsotg, 64 u32 ep_index, u32 dir_in) 65 { 66 if (dir_in) 67 return hsotg->eps_in[ep_index]; 68 else 69 return hsotg->eps_out[ep_index]; 70 } 71 72 /* forward declaration of functions */ 73 static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg); 74 75 /** 76 * using_dma - return the DMA status of the driver. 77 * @hsotg: The driver state. 78 * 79 * Return true if we're using DMA. 80 * 81 * Currently, we have the DMA support code worked into everywhere 82 * that needs it, but the AMBA DMA implementation in the hardware can 83 * only DMA from 32bit aligned addresses. This means that gadgets such 84 * as the CDC Ethernet cannot work as they often pass packets which are 85 * not 32bit aligned. 86 * 87 * Unfortunately the choice to use DMA or not is global to the controller 88 * and seems to be only settable when the controller is being put through 89 * a core reset. This means we either need to fix the gadgets to take 90 * account of DMA alignment, or add bounce buffers (yuerk). 91 * 92 * g_using_dma is set depending on dts flag. 93 */ 94 static inline bool using_dma(struct dwc2_hsotg *hsotg) 95 { 96 return hsotg->params.g_dma; 97 } 98 99 /* 100 * using_desc_dma - return the descriptor DMA status of the driver. 101 * @hsotg: The driver state. 102 * 103 * Return true if we're using descriptor DMA. 104 */ 105 static inline bool using_desc_dma(struct dwc2_hsotg *hsotg) 106 { 107 return hsotg->params.g_dma_desc; 108 } 109 110 /** 111 * dwc2_gadget_incr_frame_num - Increments the targeted frame number. 112 * @hs_ep: The endpoint 113 * @increment: The value to increment by 114 * 115 * This function will also check if the frame number overruns DSTS_SOFFN_LIMIT. 116 * If an overrun occurs it will wrap the value and set the frame_overrun flag. 117 */ 118 static inline void dwc2_gadget_incr_frame_num(struct dwc2_hsotg_ep *hs_ep) 119 { 120 hs_ep->target_frame += hs_ep->interval; 121 if (hs_ep->target_frame > DSTS_SOFFN_LIMIT) { 122 hs_ep->frame_overrun = 1; 123 hs_ep->target_frame &= DSTS_SOFFN_LIMIT; 124 } else { 125 hs_ep->frame_overrun = 0; 126 } 127 } 128 129 /** 130 * dwc2_hsotg_en_gsint - enable one or more of the general interrupt 131 * @hsotg: The device state 132 * @ints: A bitmask of the interrupts to enable 133 */ 134 static void dwc2_hsotg_en_gsint(struct dwc2_hsotg *hsotg, u32 ints) 135 { 136 u32 gsintmsk = dwc2_readl(hsotg->regs + GINTMSK); 137 u32 new_gsintmsk; 138 139 new_gsintmsk = gsintmsk | ints; 140 141 if (new_gsintmsk != gsintmsk) { 142 dev_dbg(hsotg->dev, "gsintmsk now 0x%08x\n", new_gsintmsk); 143 dwc2_writel(new_gsintmsk, hsotg->regs + GINTMSK); 144 } 145 } 146 147 /** 148 * dwc2_hsotg_disable_gsint - disable one or more of the general interrupt 149 * @hsotg: The device state 150 * @ints: A bitmask of the interrupts to enable 151 */ 152 static void dwc2_hsotg_disable_gsint(struct dwc2_hsotg *hsotg, u32 ints) 153 { 154 u32 gsintmsk = dwc2_readl(hsotg->regs + GINTMSK); 155 u32 new_gsintmsk; 156 157 new_gsintmsk = gsintmsk & ~ints; 158 159 if (new_gsintmsk != gsintmsk) 160 dwc2_writel(new_gsintmsk, hsotg->regs + GINTMSK); 161 } 162 163 /** 164 * dwc2_hsotg_ctrl_epint - enable/disable an endpoint irq 165 * @hsotg: The device state 166 * @ep: The endpoint index 167 * @dir_in: True if direction is in. 168 * @en: The enable value, true to enable 169 * 170 * Set or clear the mask for an individual endpoint's interrupt 171 * request. 172 */ 173 static void dwc2_hsotg_ctrl_epint(struct dwc2_hsotg *hsotg, 174 unsigned int ep, unsigned int dir_in, 175 unsigned int en) 176 { 177 unsigned long flags; 178 u32 bit = 1 << ep; 179 u32 daint; 180 181 if (!dir_in) 182 bit <<= 16; 183 184 local_irq_save(flags); 185 daint = dwc2_readl(hsotg->regs + DAINTMSK); 186 if (en) 187 daint |= bit; 188 else 189 daint &= ~bit; 190 dwc2_writel(daint, hsotg->regs + DAINTMSK); 191 local_irq_restore(flags); 192 } 193 194 /** 195 * dwc2_hsotg_init_fifo - initialise non-periodic FIFOs 196 * @hsotg: The device instance. 197 */ 198 static void dwc2_hsotg_init_fifo(struct dwc2_hsotg *hsotg) 199 { 200 unsigned int ep; 201 unsigned int addr; 202 int timeout; 203 u32 val; 204 u32 *txfsz = hsotg->params.g_tx_fifo_size; 205 206 /* Reset fifo map if not correctly cleared during previous session */ 207 WARN_ON(hsotg->fifo_map); 208 hsotg->fifo_map = 0; 209 210 /* set RX/NPTX FIFO sizes */ 211 dwc2_writel(hsotg->params.g_rx_fifo_size, hsotg->regs + GRXFSIZ); 212 dwc2_writel((hsotg->params.g_rx_fifo_size << FIFOSIZE_STARTADDR_SHIFT) | 213 (hsotg->params.g_np_tx_fifo_size << FIFOSIZE_DEPTH_SHIFT), 214 hsotg->regs + GNPTXFSIZ); 215 216 /* 217 * arange all the rest of the TX FIFOs, as some versions of this 218 * block have overlapping default addresses. This also ensures 219 * that if the settings have been changed, then they are set to 220 * known values. 221 */ 222 223 /* start at the end of the GNPTXFSIZ, rounded up */ 224 addr = hsotg->params.g_rx_fifo_size + hsotg->params.g_np_tx_fifo_size; 225 226 /* 227 * Configure fifos sizes from provided configuration and assign 228 * them to endpoints dynamically according to maxpacket size value of 229 * given endpoint. 230 */ 231 for (ep = 1; ep < MAX_EPS_CHANNELS; ep++) { 232 if (!txfsz[ep]) 233 continue; 234 val = addr; 235 val |= txfsz[ep] << FIFOSIZE_DEPTH_SHIFT; 236 WARN_ONCE(addr + txfsz[ep] > hsotg->fifo_mem, 237 "insufficient fifo memory"); 238 addr += txfsz[ep]; 239 240 dwc2_writel(val, hsotg->regs + DPTXFSIZN(ep)); 241 val = dwc2_readl(hsotg->regs + DPTXFSIZN(ep)); 242 } 243 244 /* 245 * according to p428 of the design guide, we need to ensure that 246 * all fifos are flushed before continuing 247 */ 248 249 dwc2_writel(GRSTCTL_TXFNUM(0x10) | GRSTCTL_TXFFLSH | 250 GRSTCTL_RXFFLSH, hsotg->regs + GRSTCTL); 251 252 /* wait until the fifos are both flushed */ 253 timeout = 100; 254 while (1) { 255 val = dwc2_readl(hsotg->regs + GRSTCTL); 256 257 if ((val & (GRSTCTL_TXFFLSH | GRSTCTL_RXFFLSH)) == 0) 258 break; 259 260 if (--timeout == 0) { 261 dev_err(hsotg->dev, 262 "%s: timeout flushing fifos (GRSTCTL=%08x)\n", 263 __func__, val); 264 break; 265 } 266 267 udelay(1); 268 } 269 270 dev_dbg(hsotg->dev, "FIFOs reset, timeout at %d\n", timeout); 271 } 272 273 /** 274 * @ep: USB endpoint to allocate request for. 275 * @flags: Allocation flags 276 * 277 * Allocate a new USB request structure appropriate for the specified endpoint 278 */ 279 static struct usb_request *dwc2_hsotg_ep_alloc_request(struct usb_ep *ep, 280 gfp_t flags) 281 { 282 struct dwc2_hsotg_req *req; 283 284 req = kzalloc(sizeof(struct dwc2_hsotg_req), flags); 285 if (!req) 286 return NULL; 287 288 INIT_LIST_HEAD(&req->queue); 289 290 return &req->req; 291 } 292 293 /** 294 * is_ep_periodic - return true if the endpoint is in periodic mode. 295 * @hs_ep: The endpoint to query. 296 * 297 * Returns true if the endpoint is in periodic mode, meaning it is being 298 * used for an Interrupt or ISO transfer. 299 */ 300 static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep) 301 { 302 return hs_ep->periodic; 303 } 304 305 /** 306 * dwc2_hsotg_unmap_dma - unmap the DMA memory being used for the request 307 * @hsotg: The device state. 308 * @hs_ep: The endpoint for the request 309 * @hs_req: The request being processed. 310 * 311 * This is the reverse of dwc2_hsotg_map_dma(), called for the completion 312 * of a request to ensure the buffer is ready for access by the caller. 313 */ 314 static void dwc2_hsotg_unmap_dma(struct dwc2_hsotg *hsotg, 315 struct dwc2_hsotg_ep *hs_ep, 316 struct dwc2_hsotg_req *hs_req) 317 { 318 struct usb_request *req = &hs_req->req; 319 usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->dir_in); 320 } 321 322 /* 323 * dwc2_gadget_alloc_ctrl_desc_chains - allocate DMA descriptor chains 324 * for Control endpoint 325 * @hsotg: The device state. 326 * 327 * This function will allocate 4 descriptor chains for EP 0: 2 for 328 * Setup stage, per one for IN and OUT data/status transactions. 329 */ 330 static int dwc2_gadget_alloc_ctrl_desc_chains(struct dwc2_hsotg *hsotg) 331 { 332 hsotg->setup_desc[0] = 333 dmam_alloc_coherent(hsotg->dev, 334 sizeof(struct dwc2_dma_desc), 335 &hsotg->setup_desc_dma[0], 336 GFP_KERNEL); 337 if (!hsotg->setup_desc[0]) 338 goto fail; 339 340 hsotg->setup_desc[1] = 341 dmam_alloc_coherent(hsotg->dev, 342 sizeof(struct dwc2_dma_desc), 343 &hsotg->setup_desc_dma[1], 344 GFP_KERNEL); 345 if (!hsotg->setup_desc[1]) 346 goto fail; 347 348 hsotg->ctrl_in_desc = 349 dmam_alloc_coherent(hsotg->dev, 350 sizeof(struct dwc2_dma_desc), 351 &hsotg->ctrl_in_desc_dma, 352 GFP_KERNEL); 353 if (!hsotg->ctrl_in_desc) 354 goto fail; 355 356 hsotg->ctrl_out_desc = 357 dmam_alloc_coherent(hsotg->dev, 358 sizeof(struct dwc2_dma_desc), 359 &hsotg->ctrl_out_desc_dma, 360 GFP_KERNEL); 361 if (!hsotg->ctrl_out_desc) 362 goto fail; 363 364 return 0; 365 366 fail: 367 return -ENOMEM; 368 } 369 370 /** 371 * dwc2_hsotg_write_fifo - write packet Data to the TxFIFO 372 * @hsotg: The controller state. 373 * @hs_ep: The endpoint we're going to write for. 374 * @hs_req: The request to write data for. 375 * 376 * This is called when the TxFIFO has some space in it to hold a new 377 * transmission and we have something to give it. The actual setup of 378 * the data size is done elsewhere, so all we have to do is to actually 379 * write the data. 380 * 381 * The return value is zero if there is more space (or nothing was done) 382 * otherwise -ENOSPC is returned if the FIFO space was used up. 383 * 384 * This routine is only needed for PIO 385 */ 386 static int dwc2_hsotg_write_fifo(struct dwc2_hsotg *hsotg, 387 struct dwc2_hsotg_ep *hs_ep, 388 struct dwc2_hsotg_req *hs_req) 389 { 390 bool periodic = is_ep_periodic(hs_ep); 391 u32 gnptxsts = dwc2_readl(hsotg->regs + GNPTXSTS); 392 int buf_pos = hs_req->req.actual; 393 int to_write = hs_ep->size_loaded; 394 void *data; 395 int can_write; 396 int pkt_round; 397 int max_transfer; 398 399 to_write -= (buf_pos - hs_ep->last_load); 400 401 /* if there's nothing to write, get out early */ 402 if (to_write == 0) 403 return 0; 404 405 if (periodic && !hsotg->dedicated_fifos) { 406 u32 epsize = dwc2_readl(hsotg->regs + DIEPTSIZ(hs_ep->index)); 407 int size_left; 408 int size_done; 409 410 /* 411 * work out how much data was loaded so we can calculate 412 * how much data is left in the fifo. 413 */ 414 415 size_left = DXEPTSIZ_XFERSIZE_GET(epsize); 416 417 /* 418 * if shared fifo, we cannot write anything until the 419 * previous data has been completely sent. 420 */ 421 if (hs_ep->fifo_load != 0) { 422 dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP); 423 return -ENOSPC; 424 } 425 426 dev_dbg(hsotg->dev, "%s: left=%d, load=%d, fifo=%d, size %d\n", 427 __func__, size_left, 428 hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size); 429 430 /* how much of the data has moved */ 431 size_done = hs_ep->size_loaded - size_left; 432 433 /* how much data is left in the fifo */ 434 can_write = hs_ep->fifo_load - size_done; 435 dev_dbg(hsotg->dev, "%s: => can_write1=%d\n", 436 __func__, can_write); 437 438 can_write = hs_ep->fifo_size - can_write; 439 dev_dbg(hsotg->dev, "%s: => can_write2=%d\n", 440 __func__, can_write); 441 442 if (can_write <= 0) { 443 dwc2_hsotg_en_gsint(hsotg, GINTSTS_PTXFEMP); 444 return -ENOSPC; 445 } 446 } else if (hsotg->dedicated_fifos && hs_ep->index != 0) { 447 can_write = dwc2_readl(hsotg->regs + 448 DTXFSTS(hs_ep->fifo_index)); 449 450 can_write &= 0xffff; 451 can_write *= 4; 452 } else { 453 if (GNPTXSTS_NP_TXQ_SPC_AVAIL_GET(gnptxsts) == 0) { 454 dev_dbg(hsotg->dev, 455 "%s: no queue slots available (0x%08x)\n", 456 __func__, gnptxsts); 457 458 dwc2_hsotg_en_gsint(hsotg, GINTSTS_NPTXFEMP); 459 return -ENOSPC; 460 } 461 462 can_write = GNPTXSTS_NP_TXF_SPC_AVAIL_GET(gnptxsts); 463 can_write *= 4; /* fifo size is in 32bit quantities. */ 464 } 465 466 max_transfer = hs_ep->ep.maxpacket * hs_ep->mc; 467 468 dev_dbg(hsotg->dev, "%s: GNPTXSTS=%08x, can=%d, to=%d, max_transfer %d\n", 469 __func__, gnptxsts, can_write, to_write, max_transfer); 470 471 /* 472 * limit to 512 bytes of data, it seems at least on the non-periodic 473 * FIFO, requests of >512 cause the endpoint to get stuck with a 474 * fragment of the end of the transfer in it. 475 */ 476 if (can_write > 512 && !periodic) 477 can_write = 512; 478 479 /* 480 * limit the write to one max-packet size worth of data, but allow 481 * the transfer to return that it did not run out of fifo space 482 * doing it. 483 */ 484 if (to_write > max_transfer) { 485 to_write = max_transfer; 486 487 /* it's needed only when we do not use dedicated fifos */ 488 if (!hsotg->dedicated_fifos) 489 dwc2_hsotg_en_gsint(hsotg, 490 periodic ? GINTSTS_PTXFEMP : 491 GINTSTS_NPTXFEMP); 492 } 493 494 /* see if we can write data */ 495 496 if (to_write > can_write) { 497 to_write = can_write; 498 pkt_round = to_write % max_transfer; 499 500 /* 501 * Round the write down to an 502 * exact number of packets. 503 * 504 * Note, we do not currently check to see if we can ever 505 * write a full packet or not to the FIFO. 506 */ 507 508 if (pkt_round) 509 to_write -= pkt_round; 510 511 /* 512 * enable correct FIFO interrupt to alert us when there 513 * is more room left. 514 */ 515 516 /* it's needed only when we do not use dedicated fifos */ 517 if (!hsotg->dedicated_fifos) 518 dwc2_hsotg_en_gsint(hsotg, 519 periodic ? GINTSTS_PTXFEMP : 520 GINTSTS_NPTXFEMP); 521 } 522 523 dev_dbg(hsotg->dev, "write %d/%d, can_write %d, done %d\n", 524 to_write, hs_req->req.length, can_write, buf_pos); 525 526 if (to_write <= 0) 527 return -ENOSPC; 528 529 hs_req->req.actual = buf_pos + to_write; 530 hs_ep->total_data += to_write; 531 532 if (periodic) 533 hs_ep->fifo_load += to_write; 534 535 to_write = DIV_ROUND_UP(to_write, 4); 536 data = hs_req->req.buf + buf_pos; 537 538 iowrite32_rep(hsotg->regs + EPFIFO(hs_ep->index), data, to_write); 539 540 return (to_write >= can_write) ? -ENOSPC : 0; 541 } 542 543 /** 544 * get_ep_limit - get the maximum data legnth for this endpoint 545 * @hs_ep: The endpoint 546 * 547 * Return the maximum data that can be queued in one go on a given endpoint 548 * so that transfers that are too long can be split. 549 */ 550 static unsigned get_ep_limit(struct dwc2_hsotg_ep *hs_ep) 551 { 552 int index = hs_ep->index; 553 unsigned maxsize; 554 unsigned maxpkt; 555 556 if (index != 0) { 557 maxsize = DXEPTSIZ_XFERSIZE_LIMIT + 1; 558 maxpkt = DXEPTSIZ_PKTCNT_LIMIT + 1; 559 } else { 560 maxsize = 64+64; 561 if (hs_ep->dir_in) 562 maxpkt = DIEPTSIZ0_PKTCNT_LIMIT + 1; 563 else 564 maxpkt = 2; 565 } 566 567 /* we made the constant loading easier above by using +1 */ 568 maxpkt--; 569 maxsize--; 570 571 /* 572 * constrain by packet count if maxpkts*pktsize is greater 573 * than the length register size. 574 */ 575 576 if ((maxpkt * hs_ep->ep.maxpacket) < maxsize) 577 maxsize = maxpkt * hs_ep->ep.maxpacket; 578 579 return maxsize; 580 } 581 582 /** 583 * dwc2_hsotg_read_frameno - read current frame number 584 * @hsotg: The device instance 585 * 586 * Return the current frame number 587 */ 588 static u32 dwc2_hsotg_read_frameno(struct dwc2_hsotg *hsotg) 589 { 590 u32 dsts; 591 592 dsts = dwc2_readl(hsotg->regs + DSTS); 593 dsts &= DSTS_SOFFN_MASK; 594 dsts >>= DSTS_SOFFN_SHIFT; 595 596 return dsts; 597 } 598 599 /** 600 * dwc2_gadget_get_chain_limit - get the maximum data payload value of the 601 * DMA descriptor chain prepared for specific endpoint 602 * @hs_ep: The endpoint 603 * 604 * Return the maximum data that can be queued in one go on a given endpoint 605 * depending on its descriptor chain capacity so that transfers that 606 * are too long can be split. 607 */ 608 static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep) 609 { 610 int is_isoc = hs_ep->isochronous; 611 unsigned int maxsize; 612 613 if (is_isoc) 614 maxsize = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT : 615 DEV_DMA_ISOC_RX_NBYTES_LIMIT; 616 else 617 maxsize = DEV_DMA_NBYTES_LIMIT; 618 619 /* Above size of one descriptor was chosen, multiple it */ 620 maxsize *= MAX_DMA_DESC_NUM_GENERIC; 621 622 return maxsize; 623 } 624 625 /* 626 * dwc2_gadget_get_desc_params - get DMA descriptor parameters. 627 * @hs_ep: The endpoint 628 * @mask: RX/TX bytes mask to be defined 629 * 630 * Returns maximum data payload for one descriptor after analyzing endpoint 631 * characteristics. 632 * DMA descriptor transfer bytes limit depends on EP type: 633 * Control out - MPS, 634 * Isochronous - descriptor rx/tx bytes bitfield limit, 635 * Control In/Bulk/Interrupt - multiple of mps. This will allow to not 636 * have concatenations from various descriptors within one packet. 637 * 638 * Selects corresponding mask for RX/TX bytes as well. 639 */ 640 static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask) 641 { 642 u32 mps = hs_ep->ep.maxpacket; 643 int dir_in = hs_ep->dir_in; 644 u32 desc_size = 0; 645 646 if (!hs_ep->index && !dir_in) { 647 desc_size = mps; 648 *mask = DEV_DMA_NBYTES_MASK; 649 } else if (hs_ep->isochronous) { 650 if (dir_in) { 651 desc_size = DEV_DMA_ISOC_TX_NBYTES_LIMIT; 652 *mask = DEV_DMA_ISOC_TX_NBYTES_MASK; 653 } else { 654 desc_size = DEV_DMA_ISOC_RX_NBYTES_LIMIT; 655 *mask = DEV_DMA_ISOC_RX_NBYTES_MASK; 656 } 657 } else { 658 desc_size = DEV_DMA_NBYTES_LIMIT; 659 *mask = DEV_DMA_NBYTES_MASK; 660 661 /* Round down desc_size to be mps multiple */ 662 desc_size -= desc_size % mps; 663 } 664 665 return desc_size; 666 } 667 668 /* 669 * dwc2_gadget_config_nonisoc_xfer_ddma - prepare non ISOC DMA desc chain. 670 * @hs_ep: The endpoint 671 * @dma_buff: DMA address to use 672 * @len: Length of the transfer 673 * 674 * This function will iterate over descriptor chain and fill its entries 675 * with corresponding information based on transfer data. 676 */ 677 static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep, 678 dma_addr_t dma_buff, 679 unsigned int len) 680 { 681 struct dwc2_hsotg *hsotg = hs_ep->parent; 682 int dir_in = hs_ep->dir_in; 683 struct dwc2_dma_desc *desc = hs_ep->desc_list; 684 u32 mps = hs_ep->ep.maxpacket; 685 u32 maxsize = 0; 686 u32 offset = 0; 687 u32 mask = 0; 688 int i; 689 690 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); 691 692 hs_ep->desc_count = (len / maxsize) + 693 ((len % maxsize) ? 1 : 0); 694 if (len == 0) 695 hs_ep->desc_count = 1; 696 697 for (i = 0; i < hs_ep->desc_count; ++i) { 698 desc->status = 0; 699 desc->status |= (DEV_DMA_BUFF_STS_HBUSY 700 << DEV_DMA_BUFF_STS_SHIFT); 701 702 if (len > maxsize) { 703 if (!hs_ep->index && !dir_in) 704 desc->status |= (DEV_DMA_L | DEV_DMA_IOC); 705 706 desc->status |= (maxsize << 707 DEV_DMA_NBYTES_SHIFT & mask); 708 desc->buf = dma_buff + offset; 709 710 len -= maxsize; 711 offset += maxsize; 712 } else { 713 desc->status |= (DEV_DMA_L | DEV_DMA_IOC); 714 715 if (dir_in) 716 desc->status |= (len % mps) ? DEV_DMA_SHORT : 717 ((hs_ep->send_zlp) ? DEV_DMA_SHORT : 0); 718 if (len > maxsize) 719 dev_err(hsotg->dev, "wrong len %d\n", len); 720 721 desc->status |= 722 len << DEV_DMA_NBYTES_SHIFT & mask; 723 desc->buf = dma_buff + offset; 724 } 725 726 desc->status &= ~DEV_DMA_BUFF_STS_MASK; 727 desc->status |= (DEV_DMA_BUFF_STS_HREADY 728 << DEV_DMA_BUFF_STS_SHIFT); 729 desc++; 730 } 731 } 732 733 /* 734 * dwc2_gadget_fill_isoc_desc - fills next isochronous descriptor in chain. 735 * @hs_ep: The isochronous endpoint. 736 * @dma_buff: usb requests dma buffer. 737 * @len: usb request transfer length. 738 * 739 * Finds out index of first free entry either in the bottom or up half of 740 * descriptor chain depend on which is under SW control and not processed 741 * by HW. Then fills that descriptor with the data of the arrived usb request, 742 * frame info, sets Last and IOC bits increments next_desc. If filled 743 * descriptor is not the first one, removes L bit from the previous descriptor 744 * status. 745 */ 746 static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep, 747 dma_addr_t dma_buff, unsigned int len) 748 { 749 struct dwc2_dma_desc *desc; 750 struct dwc2_hsotg *hsotg = hs_ep->parent; 751 u32 index; 752 u32 maxsize = 0; 753 u32 mask = 0; 754 755 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); 756 if (len > maxsize) { 757 dev_err(hsotg->dev, "wrong len %d\n", len); 758 return -EINVAL; 759 } 760 761 /* 762 * If SW has already filled half of chain, then return and wait for 763 * the other chain to be processed by HW. 764 */ 765 if (hs_ep->next_desc == MAX_DMA_DESC_NUM_GENERIC / 2) 766 return -EBUSY; 767 768 /* Increment frame number by interval for IN */ 769 if (hs_ep->dir_in) 770 dwc2_gadget_incr_frame_num(hs_ep); 771 772 index = (MAX_DMA_DESC_NUM_GENERIC / 2) * hs_ep->isoc_chain_num + 773 hs_ep->next_desc; 774 775 /* Sanity check of calculated index */ 776 if ((hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC) || 777 (!hs_ep->isoc_chain_num && index > MAX_DMA_DESC_NUM_GENERIC / 2)) { 778 dev_err(hsotg->dev, "wrong index %d for iso chain\n", index); 779 return -EINVAL; 780 } 781 782 desc = &hs_ep->desc_list[index]; 783 784 /* Clear L bit of previous desc if more than one entries in the chain */ 785 if (hs_ep->next_desc) 786 hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L; 787 788 dev_dbg(hsotg->dev, "%s: Filling ep %d, dir %s isoc desc # %d\n", 789 __func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index); 790 791 desc->status = 0; 792 desc->status |= (DEV_DMA_BUFF_STS_HBUSY << DEV_DMA_BUFF_STS_SHIFT); 793 794 desc->buf = dma_buff; 795 desc->status |= (DEV_DMA_L | DEV_DMA_IOC | 796 ((len << DEV_DMA_NBYTES_SHIFT) & mask)); 797 798 if (hs_ep->dir_in) { 799 desc->status |= ((hs_ep->mc << DEV_DMA_ISOC_PID_SHIFT) & 800 DEV_DMA_ISOC_PID_MASK) | 801 ((len % hs_ep->ep.maxpacket) ? 802 DEV_DMA_SHORT : 0) | 803 ((hs_ep->target_frame << 804 DEV_DMA_ISOC_FRNUM_SHIFT) & 805 DEV_DMA_ISOC_FRNUM_MASK); 806 } 807 808 desc->status &= ~DEV_DMA_BUFF_STS_MASK; 809 desc->status |= (DEV_DMA_BUFF_STS_HREADY << DEV_DMA_BUFF_STS_SHIFT); 810 811 /* Update index of last configured entry in the chain */ 812 hs_ep->next_desc++; 813 814 return 0; 815 } 816 817 /* 818 * dwc2_gadget_start_isoc_ddma - start isochronous transfer in DDMA 819 * @hs_ep: The isochronous endpoint. 820 * 821 * Prepare first descriptor chain for isochronous endpoints. Afterwards 822 * write DMA address to HW and enable the endpoint. 823 * 824 * Switch between descriptor chains via isoc_chain_num to give SW opportunity 825 * to prepare second descriptor chain while first one is being processed by HW. 826 */ 827 static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep) 828 { 829 struct dwc2_hsotg *hsotg = hs_ep->parent; 830 struct dwc2_hsotg_req *hs_req, *treq; 831 int index = hs_ep->index; 832 int ret; 833 u32 dma_reg; 834 u32 depctl; 835 u32 ctrl; 836 837 if (list_empty(&hs_ep->queue)) { 838 dev_dbg(hsotg->dev, "%s: No requests in queue\n", __func__); 839 return; 840 } 841 842 list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) { 843 ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma, 844 hs_req->req.length); 845 if (ret) { 846 dev_dbg(hsotg->dev, "%s: desc chain full\n", __func__); 847 break; 848 } 849 } 850 851 depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); 852 dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index); 853 854 /* write descriptor chain address to control register */ 855 dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg); 856 857 ctrl = dwc2_readl(hsotg->regs + depctl); 858 ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK; 859 dwc2_writel(ctrl, hsotg->regs + depctl); 860 861 /* Switch ISOC descriptor chain number being processed by SW*/ 862 hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1; 863 hs_ep->next_desc = 0; 864 } 865 866 /** 867 * dwc2_hsotg_start_req - start a USB request from an endpoint's queue 868 * @hsotg: The controller state. 869 * @hs_ep: The endpoint to process a request for 870 * @hs_req: The request to start. 871 * @continuing: True if we are doing more for the current request. 872 * 873 * Start the given request running by setting the endpoint registers 874 * appropriately, and writing any data to the FIFOs. 875 */ 876 static void dwc2_hsotg_start_req(struct dwc2_hsotg *hsotg, 877 struct dwc2_hsotg_ep *hs_ep, 878 struct dwc2_hsotg_req *hs_req, 879 bool continuing) 880 { 881 struct usb_request *ureq = &hs_req->req; 882 int index = hs_ep->index; 883 int dir_in = hs_ep->dir_in; 884 u32 epctrl_reg; 885 u32 epsize_reg; 886 u32 epsize; 887 u32 ctrl; 888 unsigned length; 889 unsigned packets; 890 unsigned maxreq; 891 unsigned int dma_reg; 892 893 if (index != 0) { 894 if (hs_ep->req && !continuing) { 895 dev_err(hsotg->dev, "%s: active request\n", __func__); 896 WARN_ON(1); 897 return; 898 } else if (hs_ep->req != hs_req && continuing) { 899 dev_err(hsotg->dev, 900 "%s: continue different req\n", __func__); 901 WARN_ON(1); 902 return; 903 } 904 } 905 906 dma_reg = dir_in ? DIEPDMA(index) : DOEPDMA(index); 907 epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index); 908 epsize_reg = dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index); 909 910 dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x, ep %d, dir %s\n", 911 __func__, dwc2_readl(hsotg->regs + epctrl_reg), index, 912 hs_ep->dir_in ? "in" : "out"); 913 914 /* If endpoint is stalled, we will restart request later */ 915 ctrl = dwc2_readl(hsotg->regs + epctrl_reg); 916 917 if (index && ctrl & DXEPCTL_STALL) { 918 dev_warn(hsotg->dev, "%s: ep%d is stalled\n", __func__, index); 919 return; 920 } 921 922 length = ureq->length - ureq->actual; 923 dev_dbg(hsotg->dev, "ureq->length:%d ureq->actual:%d\n", 924 ureq->length, ureq->actual); 925 926 if (!using_desc_dma(hsotg)) 927 maxreq = get_ep_limit(hs_ep); 928 else 929 maxreq = dwc2_gadget_get_chain_limit(hs_ep); 930 931 if (length > maxreq) { 932 int round = maxreq % hs_ep->ep.maxpacket; 933 934 dev_dbg(hsotg->dev, "%s: length %d, max-req %d, r %d\n", 935 __func__, length, maxreq, round); 936 937 /* round down to multiple of packets */ 938 if (round) 939 maxreq -= round; 940 941 length = maxreq; 942 } 943 944 if (length) 945 packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket); 946 else 947 packets = 1; /* send one packet if length is zero. */ 948 949 if (hs_ep->isochronous && length > (hs_ep->mc * hs_ep->ep.maxpacket)) { 950 dev_err(hsotg->dev, "req length > maxpacket*mc\n"); 951 return; 952 } 953 954 if (dir_in && index != 0) 955 if (hs_ep->isochronous) 956 epsize = DXEPTSIZ_MC(packets); 957 else 958 epsize = DXEPTSIZ_MC(1); 959 else 960 epsize = 0; 961 962 /* 963 * zero length packet should be programmed on its own and should not 964 * be counted in DIEPTSIZ.PktCnt with other packets. 965 */ 966 if (dir_in && ureq->zero && !continuing) { 967 /* Test if zlp is actually required. */ 968 if ((ureq->length >= hs_ep->ep.maxpacket) && 969 !(ureq->length % hs_ep->ep.maxpacket)) 970 hs_ep->send_zlp = 1; 971 } 972 973 epsize |= DXEPTSIZ_PKTCNT(packets); 974 epsize |= DXEPTSIZ_XFERSIZE(length); 975 976 dev_dbg(hsotg->dev, "%s: %d@%d/%d, 0x%08x => 0x%08x\n", 977 __func__, packets, length, ureq->length, epsize, epsize_reg); 978 979 /* store the request as the current one we're doing */ 980 hs_ep->req = hs_req; 981 982 if (using_desc_dma(hsotg)) { 983 u32 offset = 0; 984 u32 mps = hs_ep->ep.maxpacket; 985 986 /* Adjust length: EP0 - MPS, other OUT EPs - multiple of MPS */ 987 if (!dir_in) { 988 if (!index) 989 length = mps; 990 else if (length % mps) 991 length += (mps - (length % mps)); 992 } 993 994 /* 995 * If more data to send, adjust DMA for EP0 out data stage. 996 * ureq->dma stays unchanged, hence increment it by already 997 * passed passed data count before starting new transaction. 998 */ 999 if (!index && hsotg->ep0_state == DWC2_EP0_DATA_OUT && 1000 continuing) 1001 offset = ureq->actual; 1002 1003 /* Fill DDMA chain entries */ 1004 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset, 1005 length); 1006 1007 /* write descriptor chain address to control register */ 1008 dwc2_writel(hs_ep->desc_list_dma, hsotg->regs + dma_reg); 1009 1010 dev_dbg(hsotg->dev, "%s: %08x pad => 0x%08x\n", 1011 __func__, (u32)hs_ep->desc_list_dma, dma_reg); 1012 } else { 1013 /* write size / packets */ 1014 dwc2_writel(epsize, hsotg->regs + epsize_reg); 1015 1016 if (using_dma(hsotg) && !continuing && (length != 0)) { 1017 /* 1018 * write DMA address to control register, buffer 1019 * already synced by dwc2_hsotg_ep_queue(). 1020 */ 1021 1022 dwc2_writel(ureq->dma, hsotg->regs + dma_reg); 1023 1024 dev_dbg(hsotg->dev, "%s: %pad => 0x%08x\n", 1025 __func__, &ureq->dma, dma_reg); 1026 } 1027 } 1028 1029 if (hs_ep->isochronous && hs_ep->interval == 1) { 1030 hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg); 1031 dwc2_gadget_incr_frame_num(hs_ep); 1032 1033 if (hs_ep->target_frame & 0x1) 1034 ctrl |= DXEPCTL_SETODDFR; 1035 else 1036 ctrl |= DXEPCTL_SETEVENFR; 1037 } 1038 1039 ctrl |= DXEPCTL_EPENA; /* ensure ep enabled */ 1040 1041 dev_dbg(hsotg->dev, "ep0 state:%d\n", hsotg->ep0_state); 1042 1043 /* For Setup request do not clear NAK */ 1044 if (!(index == 0 && hsotg->ep0_state == DWC2_EP0_SETUP)) 1045 ctrl |= DXEPCTL_CNAK; /* clear NAK set by core */ 1046 1047 dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl); 1048 dwc2_writel(ctrl, hsotg->regs + epctrl_reg); 1049 1050 /* 1051 * set these, it seems that DMA support increments past the end 1052 * of the packet buffer so we need to calculate the length from 1053 * this information. 1054 */ 1055 hs_ep->size_loaded = length; 1056 hs_ep->last_load = ureq->actual; 1057 1058 if (dir_in && !using_dma(hsotg)) { 1059 /* set these anyway, we may need them for non-periodic in */ 1060 hs_ep->fifo_load = 0; 1061 1062 dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req); 1063 } 1064 1065 /* 1066 * Note, trying to clear the NAK here causes problems with transmit 1067 * on the S3C6400 ending up with the TXFIFO becoming full. 1068 */ 1069 1070 /* check ep is enabled */ 1071 if (!(dwc2_readl(hsotg->regs + epctrl_reg) & DXEPCTL_EPENA)) 1072 dev_dbg(hsotg->dev, 1073 "ep%d: failed to become enabled (DXEPCTL=0x%08x)?\n", 1074 index, dwc2_readl(hsotg->regs + epctrl_reg)); 1075 1076 dev_dbg(hsotg->dev, "%s: DXEPCTL=0x%08x\n", 1077 __func__, dwc2_readl(hsotg->regs + epctrl_reg)); 1078 1079 /* enable ep interrupts */ 1080 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1); 1081 } 1082 1083 /** 1084 * dwc2_hsotg_map_dma - map the DMA memory being used for the request 1085 * @hsotg: The device state. 1086 * @hs_ep: The endpoint the request is on. 1087 * @req: The request being processed. 1088 * 1089 * We've been asked to queue a request, so ensure that the memory buffer 1090 * is correctly setup for DMA. If we've been passed an extant DMA address 1091 * then ensure the buffer has been synced to memory. If our buffer has no 1092 * DMA memory, then we map the memory and mark our request to allow us to 1093 * cleanup on completion. 1094 */ 1095 static int dwc2_hsotg_map_dma(struct dwc2_hsotg *hsotg, 1096 struct dwc2_hsotg_ep *hs_ep, 1097 struct usb_request *req) 1098 { 1099 int ret; 1100 1101 ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in); 1102 if (ret) 1103 goto dma_error; 1104 1105 return 0; 1106 1107 dma_error: 1108 dev_err(hsotg->dev, "%s: failed to map buffer %p, %d bytes\n", 1109 __func__, req->buf, req->length); 1110 1111 return -EIO; 1112 } 1113 1114 static int dwc2_hsotg_handle_unaligned_buf_start(struct dwc2_hsotg *hsotg, 1115 struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req) 1116 { 1117 void *req_buf = hs_req->req.buf; 1118 1119 /* If dma is not being used or buffer is aligned */ 1120 if (!using_dma(hsotg) || !((long)req_buf & 3)) 1121 return 0; 1122 1123 WARN_ON(hs_req->saved_req_buf); 1124 1125 dev_dbg(hsotg->dev, "%s: %s: buf=%p length=%d\n", __func__, 1126 hs_ep->ep.name, req_buf, hs_req->req.length); 1127 1128 hs_req->req.buf = kmalloc(hs_req->req.length, GFP_ATOMIC); 1129 if (!hs_req->req.buf) { 1130 hs_req->req.buf = req_buf; 1131 dev_err(hsotg->dev, 1132 "%s: unable to allocate memory for bounce buffer\n", 1133 __func__); 1134 return -ENOMEM; 1135 } 1136 1137 /* Save actual buffer */ 1138 hs_req->saved_req_buf = req_buf; 1139 1140 if (hs_ep->dir_in) 1141 memcpy(hs_req->req.buf, req_buf, hs_req->req.length); 1142 return 0; 1143 } 1144 1145 static void dwc2_hsotg_handle_unaligned_buf_complete(struct dwc2_hsotg *hsotg, 1146 struct dwc2_hsotg_ep *hs_ep, struct dwc2_hsotg_req *hs_req) 1147 { 1148 /* If dma is not being used or buffer was aligned */ 1149 if (!using_dma(hsotg) || !hs_req->saved_req_buf) 1150 return; 1151 1152 dev_dbg(hsotg->dev, "%s: %s: status=%d actual-length=%d\n", __func__, 1153 hs_ep->ep.name, hs_req->req.status, hs_req->req.actual); 1154 1155 /* Copy data from bounce buffer on successful out transfer */ 1156 if (!hs_ep->dir_in && !hs_req->req.status) 1157 memcpy(hs_req->saved_req_buf, hs_req->req.buf, 1158 hs_req->req.actual); 1159 1160 /* Free bounce buffer */ 1161 kfree(hs_req->req.buf); 1162 1163 hs_req->req.buf = hs_req->saved_req_buf; 1164 hs_req->saved_req_buf = NULL; 1165 } 1166 1167 /** 1168 * dwc2_gadget_target_frame_elapsed - Checks target frame 1169 * @hs_ep: The driver endpoint to check 1170 * 1171 * Returns 1 if targeted frame elapsed. If returned 1 then we need to drop 1172 * corresponding transfer. 1173 */ 1174 static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep) 1175 { 1176 struct dwc2_hsotg *hsotg = hs_ep->parent; 1177 u32 target_frame = hs_ep->target_frame; 1178 u32 current_frame = dwc2_hsotg_read_frameno(hsotg); 1179 bool frame_overrun = hs_ep->frame_overrun; 1180 1181 if (!frame_overrun && current_frame >= target_frame) 1182 return true; 1183 1184 if (frame_overrun && current_frame >= target_frame && 1185 ((current_frame - target_frame) < DSTS_SOFFN_LIMIT / 2)) 1186 return true; 1187 1188 return false; 1189 } 1190 1191 /* 1192 * dwc2_gadget_set_ep0_desc_chain - Set EP's desc chain pointers 1193 * @hsotg: The driver state 1194 * @hs_ep: the ep descriptor chain is for 1195 * 1196 * Called to update EP0 structure's pointers depend on stage of 1197 * control transfer. 1198 */ 1199 static int dwc2_gadget_set_ep0_desc_chain(struct dwc2_hsotg *hsotg, 1200 struct dwc2_hsotg_ep *hs_ep) 1201 { 1202 switch (hsotg->ep0_state) { 1203 case DWC2_EP0_SETUP: 1204 case DWC2_EP0_STATUS_OUT: 1205 hs_ep->desc_list = hsotg->setup_desc[0]; 1206 hs_ep->desc_list_dma = hsotg->setup_desc_dma[0]; 1207 break; 1208 case DWC2_EP0_DATA_IN: 1209 case DWC2_EP0_STATUS_IN: 1210 hs_ep->desc_list = hsotg->ctrl_in_desc; 1211 hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma; 1212 break; 1213 case DWC2_EP0_DATA_OUT: 1214 hs_ep->desc_list = hsotg->ctrl_out_desc; 1215 hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma; 1216 break; 1217 default: 1218 dev_err(hsotg->dev, "invalid EP 0 state in queue %d\n", 1219 hsotg->ep0_state); 1220 return -EINVAL; 1221 } 1222 1223 return 0; 1224 } 1225 1226 static int dwc2_hsotg_ep_queue(struct usb_ep *ep, struct usb_request *req, 1227 gfp_t gfp_flags) 1228 { 1229 struct dwc2_hsotg_req *hs_req = our_req(req); 1230 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); 1231 struct dwc2_hsotg *hs = hs_ep->parent; 1232 bool first; 1233 int ret; 1234 1235 dev_dbg(hs->dev, "%s: req %p: %d@%p, noi=%d, zero=%d, snok=%d\n", 1236 ep->name, req, req->length, req->buf, req->no_interrupt, 1237 req->zero, req->short_not_ok); 1238 1239 /* Prevent new request submission when controller is suspended */ 1240 if (hs->lx_state == DWC2_L2) { 1241 dev_dbg(hs->dev, "%s: don't submit request while suspended\n", 1242 __func__); 1243 return -EAGAIN; 1244 } 1245 1246 /* initialise status of the request */ 1247 INIT_LIST_HEAD(&hs_req->queue); 1248 req->actual = 0; 1249 req->status = -EINPROGRESS; 1250 1251 ret = dwc2_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req); 1252 if (ret) 1253 return ret; 1254 1255 /* if we're using DMA, sync the buffers as necessary */ 1256 if (using_dma(hs)) { 1257 ret = dwc2_hsotg_map_dma(hs, hs_ep, req); 1258 if (ret) 1259 return ret; 1260 } 1261 /* If using descriptor DMA configure EP0 descriptor chain pointers */ 1262 if (using_desc_dma(hs) && !hs_ep->index) { 1263 ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep); 1264 if (ret) 1265 return ret; 1266 } 1267 1268 first = list_empty(&hs_ep->queue); 1269 list_add_tail(&hs_req->queue, &hs_ep->queue); 1270 1271 /* 1272 * Handle DDMA isochronous transfers separately - just add new entry 1273 * to the half of descriptor chain that is not processed by HW. 1274 * Transfer will be started once SW gets either one of NAK or 1275 * OutTknEpDis interrupts. 1276 */ 1277 if (using_desc_dma(hs) && hs_ep->isochronous && 1278 hs_ep->target_frame != TARGET_FRAME_INITIAL) { 1279 ret = dwc2_gadget_fill_isoc_desc(hs_ep, hs_req->req.dma, 1280 hs_req->req.length); 1281 if (ret) 1282 dev_dbg(hs->dev, "%s: ISO desc chain full\n", __func__); 1283 1284 return 0; 1285 } 1286 1287 if (first) { 1288 if (!hs_ep->isochronous) { 1289 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false); 1290 return 0; 1291 } 1292 1293 while (dwc2_gadget_target_frame_elapsed(hs_ep)) 1294 dwc2_gadget_incr_frame_num(hs_ep); 1295 1296 if (hs_ep->target_frame != TARGET_FRAME_INITIAL) 1297 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false); 1298 } 1299 return 0; 1300 } 1301 1302 static int dwc2_hsotg_ep_queue_lock(struct usb_ep *ep, struct usb_request *req, 1303 gfp_t gfp_flags) 1304 { 1305 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); 1306 struct dwc2_hsotg *hs = hs_ep->parent; 1307 unsigned long flags = 0; 1308 int ret = 0; 1309 1310 spin_lock_irqsave(&hs->lock, flags); 1311 ret = dwc2_hsotg_ep_queue(ep, req, gfp_flags); 1312 spin_unlock_irqrestore(&hs->lock, flags); 1313 1314 return ret; 1315 } 1316 1317 static void dwc2_hsotg_ep_free_request(struct usb_ep *ep, 1318 struct usb_request *req) 1319 { 1320 struct dwc2_hsotg_req *hs_req = our_req(req); 1321 1322 kfree(hs_req); 1323 } 1324 1325 /** 1326 * dwc2_hsotg_complete_oursetup - setup completion callback 1327 * @ep: The endpoint the request was on. 1328 * @req: The request completed. 1329 * 1330 * Called on completion of any requests the driver itself 1331 * submitted that need cleaning up. 1332 */ 1333 static void dwc2_hsotg_complete_oursetup(struct usb_ep *ep, 1334 struct usb_request *req) 1335 { 1336 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); 1337 struct dwc2_hsotg *hsotg = hs_ep->parent; 1338 1339 dev_dbg(hsotg->dev, "%s: ep %p, req %p\n", __func__, ep, req); 1340 1341 dwc2_hsotg_ep_free_request(ep, req); 1342 } 1343 1344 /** 1345 * ep_from_windex - convert control wIndex value to endpoint 1346 * @hsotg: The driver state. 1347 * @windex: The control request wIndex field (in host order). 1348 * 1349 * Convert the given wIndex into a pointer to an driver endpoint 1350 * structure, or return NULL if it is not a valid endpoint. 1351 */ 1352 static struct dwc2_hsotg_ep *ep_from_windex(struct dwc2_hsotg *hsotg, 1353 u32 windex) 1354 { 1355 struct dwc2_hsotg_ep *ep; 1356 int dir = (windex & USB_DIR_IN) ? 1 : 0; 1357 int idx = windex & 0x7F; 1358 1359 if (windex >= 0x100) 1360 return NULL; 1361 1362 if (idx > hsotg->num_of_eps) 1363 return NULL; 1364 1365 ep = index_to_ep(hsotg, idx, dir); 1366 1367 if (idx && ep->dir_in != dir) 1368 return NULL; 1369 1370 return ep; 1371 } 1372 1373 /** 1374 * dwc2_hsotg_set_test_mode - Enable usb Test Modes 1375 * @hsotg: The driver state. 1376 * @testmode: requested usb test mode 1377 * Enable usb Test Mode requested by the Host. 1378 */ 1379 int dwc2_hsotg_set_test_mode(struct dwc2_hsotg *hsotg, int testmode) 1380 { 1381 int dctl = dwc2_readl(hsotg->regs + DCTL); 1382 1383 dctl &= ~DCTL_TSTCTL_MASK; 1384 switch (testmode) { 1385 case TEST_J: 1386 case TEST_K: 1387 case TEST_SE0_NAK: 1388 case TEST_PACKET: 1389 case TEST_FORCE_EN: 1390 dctl |= testmode << DCTL_TSTCTL_SHIFT; 1391 break; 1392 default: 1393 return -EINVAL; 1394 } 1395 dwc2_writel(dctl, hsotg->regs + DCTL); 1396 return 0; 1397 } 1398 1399 /** 1400 * dwc2_hsotg_send_reply - send reply to control request 1401 * @hsotg: The device state 1402 * @ep: Endpoint 0 1403 * @buff: Buffer for request 1404 * @length: Length of reply. 1405 * 1406 * Create a request and queue it on the given endpoint. This is useful as 1407 * an internal method of sending replies to certain control requests, etc. 1408 */ 1409 static int dwc2_hsotg_send_reply(struct dwc2_hsotg *hsotg, 1410 struct dwc2_hsotg_ep *ep, 1411 void *buff, 1412 int length) 1413 { 1414 struct usb_request *req; 1415 int ret; 1416 1417 dev_dbg(hsotg->dev, "%s: buff %p, len %d\n", __func__, buff, length); 1418 1419 req = dwc2_hsotg_ep_alloc_request(&ep->ep, GFP_ATOMIC); 1420 hsotg->ep0_reply = req; 1421 if (!req) { 1422 dev_warn(hsotg->dev, "%s: cannot alloc req\n", __func__); 1423 return -ENOMEM; 1424 } 1425 1426 req->buf = hsotg->ep0_buff; 1427 req->length = length; 1428 /* 1429 * zero flag is for sending zlp in DATA IN stage. It has no impact on 1430 * STATUS stage. 1431 */ 1432 req->zero = 0; 1433 req->complete = dwc2_hsotg_complete_oursetup; 1434 1435 if (length) 1436 memcpy(req->buf, buff, length); 1437 1438 ret = dwc2_hsotg_ep_queue(&ep->ep, req, GFP_ATOMIC); 1439 if (ret) { 1440 dev_warn(hsotg->dev, "%s: cannot queue req\n", __func__); 1441 return ret; 1442 } 1443 1444 return 0; 1445 } 1446 1447 /** 1448 * dwc2_hsotg_process_req_status - process request GET_STATUS 1449 * @hsotg: The device state 1450 * @ctrl: USB control request 1451 */ 1452 static int dwc2_hsotg_process_req_status(struct dwc2_hsotg *hsotg, 1453 struct usb_ctrlrequest *ctrl) 1454 { 1455 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0]; 1456 struct dwc2_hsotg_ep *ep; 1457 __le16 reply; 1458 int ret; 1459 1460 dev_dbg(hsotg->dev, "%s: USB_REQ_GET_STATUS\n", __func__); 1461 1462 if (!ep0->dir_in) { 1463 dev_warn(hsotg->dev, "%s: direction out?\n", __func__); 1464 return -EINVAL; 1465 } 1466 1467 switch (ctrl->bRequestType & USB_RECIP_MASK) { 1468 case USB_RECIP_DEVICE: 1469 reply = cpu_to_le16(0); /* bit 0 => self powered, 1470 * bit 1 => remote wakeup */ 1471 break; 1472 1473 case USB_RECIP_INTERFACE: 1474 /* currently, the data result should be zero */ 1475 reply = cpu_to_le16(0); 1476 break; 1477 1478 case USB_RECIP_ENDPOINT: 1479 ep = ep_from_windex(hsotg, le16_to_cpu(ctrl->wIndex)); 1480 if (!ep) 1481 return -ENOENT; 1482 1483 reply = cpu_to_le16(ep->halted ? 1 : 0); 1484 break; 1485 1486 default: 1487 return 0; 1488 } 1489 1490 if (le16_to_cpu(ctrl->wLength) != 2) 1491 return -EINVAL; 1492 1493 ret = dwc2_hsotg_send_reply(hsotg, ep0, &reply, 2); 1494 if (ret) { 1495 dev_err(hsotg->dev, "%s: failed to send reply\n", __func__); 1496 return ret; 1497 } 1498 1499 return 1; 1500 } 1501 1502 static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now); 1503 1504 /** 1505 * get_ep_head - return the first request on the endpoint 1506 * @hs_ep: The controller endpoint to get 1507 * 1508 * Get the first request on the endpoint. 1509 */ 1510 static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep) 1511 { 1512 return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req, 1513 queue); 1514 } 1515 1516 /** 1517 * dwc2_gadget_start_next_request - Starts next request from ep queue 1518 * @hs_ep: Endpoint structure 1519 * 1520 * If queue is empty and EP is ISOC-OUT - unmasks OUTTKNEPDIS which is masked 1521 * in its handler. Hence we need to unmask it here to be able to do 1522 * resynchronization. 1523 */ 1524 static void dwc2_gadget_start_next_request(struct dwc2_hsotg_ep *hs_ep) 1525 { 1526 u32 mask; 1527 struct dwc2_hsotg *hsotg = hs_ep->parent; 1528 int dir_in = hs_ep->dir_in; 1529 struct dwc2_hsotg_req *hs_req; 1530 u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK; 1531 1532 if (!list_empty(&hs_ep->queue)) { 1533 hs_req = get_ep_head(hs_ep); 1534 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, false); 1535 return; 1536 } 1537 if (!hs_ep->isochronous) 1538 return; 1539 1540 if (dir_in) { 1541 dev_dbg(hsotg->dev, "%s: No more ISOC-IN requests\n", 1542 __func__); 1543 } else { 1544 dev_dbg(hsotg->dev, "%s: No more ISOC-OUT requests\n", 1545 __func__); 1546 mask = dwc2_readl(hsotg->regs + epmsk_reg); 1547 mask |= DOEPMSK_OUTTKNEPDISMSK; 1548 dwc2_writel(mask, hsotg->regs + epmsk_reg); 1549 } 1550 } 1551 1552 /** 1553 * dwc2_hsotg_process_req_feature - process request {SET,CLEAR}_FEATURE 1554 * @hsotg: The device state 1555 * @ctrl: USB control request 1556 */ 1557 static int dwc2_hsotg_process_req_feature(struct dwc2_hsotg *hsotg, 1558 struct usb_ctrlrequest *ctrl) 1559 { 1560 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0]; 1561 struct dwc2_hsotg_req *hs_req; 1562 bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE); 1563 struct dwc2_hsotg_ep *ep; 1564 int ret; 1565 bool halted; 1566 u32 recip; 1567 u32 wValue; 1568 u32 wIndex; 1569 1570 dev_dbg(hsotg->dev, "%s: %s_FEATURE\n", 1571 __func__, set ? "SET" : "CLEAR"); 1572 1573 wValue = le16_to_cpu(ctrl->wValue); 1574 wIndex = le16_to_cpu(ctrl->wIndex); 1575 recip = ctrl->bRequestType & USB_RECIP_MASK; 1576 1577 switch (recip) { 1578 case USB_RECIP_DEVICE: 1579 switch (wValue) { 1580 case USB_DEVICE_TEST_MODE: 1581 if ((wIndex & 0xff) != 0) 1582 return -EINVAL; 1583 if (!set) 1584 return -EINVAL; 1585 1586 hsotg->test_mode = wIndex >> 8; 1587 ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0); 1588 if (ret) { 1589 dev_err(hsotg->dev, 1590 "%s: failed to send reply\n", __func__); 1591 return ret; 1592 } 1593 break; 1594 default: 1595 return -ENOENT; 1596 } 1597 break; 1598 1599 case USB_RECIP_ENDPOINT: 1600 ep = ep_from_windex(hsotg, wIndex); 1601 if (!ep) { 1602 dev_dbg(hsotg->dev, "%s: no endpoint for 0x%04x\n", 1603 __func__, wIndex); 1604 return -ENOENT; 1605 } 1606 1607 switch (wValue) { 1608 case USB_ENDPOINT_HALT: 1609 halted = ep->halted; 1610 1611 dwc2_hsotg_ep_sethalt(&ep->ep, set, true); 1612 1613 ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0); 1614 if (ret) { 1615 dev_err(hsotg->dev, 1616 "%s: failed to send reply\n", __func__); 1617 return ret; 1618 } 1619 1620 /* 1621 * we have to complete all requests for ep if it was 1622 * halted, and the halt was cleared by CLEAR_FEATURE 1623 */ 1624 1625 if (!set && halted) { 1626 /* 1627 * If we have request in progress, 1628 * then complete it 1629 */ 1630 if (ep->req) { 1631 hs_req = ep->req; 1632 ep->req = NULL; 1633 list_del_init(&hs_req->queue); 1634 if (hs_req->req.complete) { 1635 spin_unlock(&hsotg->lock); 1636 usb_gadget_giveback_request( 1637 &ep->ep, &hs_req->req); 1638 spin_lock(&hsotg->lock); 1639 } 1640 } 1641 1642 /* If we have pending request, then start it */ 1643 if (!ep->req) { 1644 dwc2_gadget_start_next_request(ep); 1645 } 1646 } 1647 1648 break; 1649 1650 default: 1651 return -ENOENT; 1652 } 1653 break; 1654 default: 1655 return -ENOENT; 1656 } 1657 return 1; 1658 } 1659 1660 static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg); 1661 1662 /** 1663 * dwc2_hsotg_stall_ep0 - stall ep0 1664 * @hsotg: The device state 1665 * 1666 * Set stall for ep0 as response for setup request. 1667 */ 1668 static void dwc2_hsotg_stall_ep0(struct dwc2_hsotg *hsotg) 1669 { 1670 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0]; 1671 u32 reg; 1672 u32 ctrl; 1673 1674 dev_dbg(hsotg->dev, "ep0 stall (dir=%d)\n", ep0->dir_in); 1675 reg = (ep0->dir_in) ? DIEPCTL0 : DOEPCTL0; 1676 1677 /* 1678 * DxEPCTL_Stall will be cleared by EP once it has 1679 * taken effect, so no need to clear later. 1680 */ 1681 1682 ctrl = dwc2_readl(hsotg->regs + reg); 1683 ctrl |= DXEPCTL_STALL; 1684 ctrl |= DXEPCTL_CNAK; 1685 dwc2_writel(ctrl, hsotg->regs + reg); 1686 1687 dev_dbg(hsotg->dev, 1688 "written DXEPCTL=0x%08x to %08x (DXEPCTL=0x%08x)\n", 1689 ctrl, reg, dwc2_readl(hsotg->regs + reg)); 1690 1691 /* 1692 * complete won't be called, so we enqueue 1693 * setup request here 1694 */ 1695 dwc2_hsotg_enqueue_setup(hsotg); 1696 } 1697 1698 /** 1699 * dwc2_hsotg_process_control - process a control request 1700 * @hsotg: The device state 1701 * @ctrl: The control request received 1702 * 1703 * The controller has received the SETUP phase of a control request, and 1704 * needs to work out what to do next (and whether to pass it on to the 1705 * gadget driver). 1706 */ 1707 static void dwc2_hsotg_process_control(struct dwc2_hsotg *hsotg, 1708 struct usb_ctrlrequest *ctrl) 1709 { 1710 struct dwc2_hsotg_ep *ep0 = hsotg->eps_out[0]; 1711 int ret = 0; 1712 u32 dcfg; 1713 1714 dev_dbg(hsotg->dev, 1715 "ctrl Type=%02x, Req=%02x, V=%04x, I=%04x, L=%04x\n", 1716 ctrl->bRequestType, ctrl->bRequest, ctrl->wValue, 1717 ctrl->wIndex, ctrl->wLength); 1718 1719 if (ctrl->wLength == 0) { 1720 ep0->dir_in = 1; 1721 hsotg->ep0_state = DWC2_EP0_STATUS_IN; 1722 } else if (ctrl->bRequestType & USB_DIR_IN) { 1723 ep0->dir_in = 1; 1724 hsotg->ep0_state = DWC2_EP0_DATA_IN; 1725 } else { 1726 ep0->dir_in = 0; 1727 hsotg->ep0_state = DWC2_EP0_DATA_OUT; 1728 } 1729 1730 if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) { 1731 switch (ctrl->bRequest) { 1732 case USB_REQ_SET_ADDRESS: 1733 hsotg->connected = 1; 1734 dcfg = dwc2_readl(hsotg->regs + DCFG); 1735 dcfg &= ~DCFG_DEVADDR_MASK; 1736 dcfg |= (le16_to_cpu(ctrl->wValue) << 1737 DCFG_DEVADDR_SHIFT) & DCFG_DEVADDR_MASK; 1738 dwc2_writel(dcfg, hsotg->regs + DCFG); 1739 1740 dev_info(hsotg->dev, "new address %d\n", ctrl->wValue); 1741 1742 ret = dwc2_hsotg_send_reply(hsotg, ep0, NULL, 0); 1743 return; 1744 1745 case USB_REQ_GET_STATUS: 1746 ret = dwc2_hsotg_process_req_status(hsotg, ctrl); 1747 break; 1748 1749 case USB_REQ_CLEAR_FEATURE: 1750 case USB_REQ_SET_FEATURE: 1751 ret = dwc2_hsotg_process_req_feature(hsotg, ctrl); 1752 break; 1753 } 1754 } 1755 1756 /* as a fallback, try delivering it to the driver to deal with */ 1757 1758 if (ret == 0 && hsotg->driver) { 1759 spin_unlock(&hsotg->lock); 1760 ret = hsotg->driver->setup(&hsotg->gadget, ctrl); 1761 spin_lock(&hsotg->lock); 1762 if (ret < 0) 1763 dev_dbg(hsotg->dev, "driver->setup() ret %d\n", ret); 1764 } 1765 1766 /* 1767 * the request is either unhandlable, or is not formatted correctly 1768 * so respond with a STALL for the status stage to indicate failure. 1769 */ 1770 1771 if (ret < 0) 1772 dwc2_hsotg_stall_ep0(hsotg); 1773 } 1774 1775 /** 1776 * dwc2_hsotg_complete_setup - completion of a setup transfer 1777 * @ep: The endpoint the request was on. 1778 * @req: The request completed. 1779 * 1780 * Called on completion of any requests the driver itself submitted for 1781 * EP0 setup packets 1782 */ 1783 static void dwc2_hsotg_complete_setup(struct usb_ep *ep, 1784 struct usb_request *req) 1785 { 1786 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); 1787 struct dwc2_hsotg *hsotg = hs_ep->parent; 1788 1789 if (req->status < 0) { 1790 dev_dbg(hsotg->dev, "%s: failed %d\n", __func__, req->status); 1791 return; 1792 } 1793 1794 spin_lock(&hsotg->lock); 1795 if (req->actual == 0) 1796 dwc2_hsotg_enqueue_setup(hsotg); 1797 else 1798 dwc2_hsotg_process_control(hsotg, req->buf); 1799 spin_unlock(&hsotg->lock); 1800 } 1801 1802 /** 1803 * dwc2_hsotg_enqueue_setup - start a request for EP0 packets 1804 * @hsotg: The device state. 1805 * 1806 * Enqueue a request on EP0 if necessary to received any SETUP packets 1807 * received from the host. 1808 */ 1809 static void dwc2_hsotg_enqueue_setup(struct dwc2_hsotg *hsotg) 1810 { 1811 struct usb_request *req = hsotg->ctrl_req; 1812 struct dwc2_hsotg_req *hs_req = our_req(req); 1813 int ret; 1814 1815 dev_dbg(hsotg->dev, "%s: queueing setup request\n", __func__); 1816 1817 req->zero = 0; 1818 req->length = 8; 1819 req->buf = hsotg->ctrl_buff; 1820 req->complete = dwc2_hsotg_complete_setup; 1821 1822 if (!list_empty(&hs_req->queue)) { 1823 dev_dbg(hsotg->dev, "%s already queued???\n", __func__); 1824 return; 1825 } 1826 1827 hsotg->eps_out[0]->dir_in = 0; 1828 hsotg->eps_out[0]->send_zlp = 0; 1829 hsotg->ep0_state = DWC2_EP0_SETUP; 1830 1831 ret = dwc2_hsotg_ep_queue(&hsotg->eps_out[0]->ep, req, GFP_ATOMIC); 1832 if (ret < 0) { 1833 dev_err(hsotg->dev, "%s: failed queue (%d)\n", __func__, ret); 1834 /* 1835 * Don't think there's much we can do other than watch the 1836 * driver fail. 1837 */ 1838 } 1839 } 1840 1841 static void dwc2_hsotg_program_zlp(struct dwc2_hsotg *hsotg, 1842 struct dwc2_hsotg_ep *hs_ep) 1843 { 1844 u32 ctrl; 1845 u8 index = hs_ep->index; 1846 u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); 1847 u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index); 1848 1849 if (hs_ep->dir_in) 1850 dev_dbg(hsotg->dev, "Sending zero-length packet on ep%d\n", 1851 index); 1852 else 1853 dev_dbg(hsotg->dev, "Receiving zero-length packet on ep%d\n", 1854 index); 1855 if (using_desc_dma(hsotg)) { 1856 /* Not specific buffer needed for ep0 ZLP */ 1857 dma_addr_t dma = hs_ep->desc_list_dma; 1858 1859 dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep); 1860 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0); 1861 } else { 1862 dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) | 1863 DXEPTSIZ_XFERSIZE(0), hsotg->regs + 1864 epsiz_reg); 1865 } 1866 1867 ctrl = dwc2_readl(hsotg->regs + epctl_reg); 1868 ctrl |= DXEPCTL_CNAK; /* clear NAK set by core */ 1869 ctrl |= DXEPCTL_EPENA; /* ensure ep enabled */ 1870 ctrl |= DXEPCTL_USBACTEP; 1871 dwc2_writel(ctrl, hsotg->regs + epctl_reg); 1872 } 1873 1874 /** 1875 * dwc2_hsotg_complete_request - complete a request given to us 1876 * @hsotg: The device state. 1877 * @hs_ep: The endpoint the request was on. 1878 * @hs_req: The request to complete. 1879 * @result: The result code (0 => Ok, otherwise errno) 1880 * 1881 * The given request has finished, so call the necessary completion 1882 * if it has one and then look to see if we can start a new request 1883 * on the endpoint. 1884 * 1885 * Note, expects the ep to already be locked as appropriate. 1886 */ 1887 static void dwc2_hsotg_complete_request(struct dwc2_hsotg *hsotg, 1888 struct dwc2_hsotg_ep *hs_ep, 1889 struct dwc2_hsotg_req *hs_req, 1890 int result) 1891 { 1892 1893 if (!hs_req) { 1894 dev_dbg(hsotg->dev, "%s: nothing to complete?\n", __func__); 1895 return; 1896 } 1897 1898 dev_dbg(hsotg->dev, "complete: ep %p %s, req %p, %d => %p\n", 1899 hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete); 1900 1901 /* 1902 * only replace the status if we've not already set an error 1903 * from a previous transaction 1904 */ 1905 1906 if (hs_req->req.status == -EINPROGRESS) 1907 hs_req->req.status = result; 1908 1909 if (using_dma(hsotg)) 1910 dwc2_hsotg_unmap_dma(hsotg, hs_ep, hs_req); 1911 1912 dwc2_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req); 1913 1914 hs_ep->req = NULL; 1915 list_del_init(&hs_req->queue); 1916 1917 /* 1918 * call the complete request with the locks off, just in case the 1919 * request tries to queue more work for this endpoint. 1920 */ 1921 1922 if (hs_req->req.complete) { 1923 spin_unlock(&hsotg->lock); 1924 usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req); 1925 spin_lock(&hsotg->lock); 1926 } 1927 1928 /* In DDMA don't need to proceed to starting of next ISOC request */ 1929 if (using_desc_dma(hsotg) && hs_ep->isochronous) 1930 return; 1931 1932 /* 1933 * Look to see if there is anything else to do. Note, the completion 1934 * of the previous request may have caused a new request to be started 1935 * so be careful when doing this. 1936 */ 1937 1938 if (!hs_ep->req && result >= 0) { 1939 dwc2_gadget_start_next_request(hs_ep); 1940 } 1941 } 1942 1943 /* 1944 * dwc2_gadget_complete_isoc_request_ddma - complete an isoc request in DDMA 1945 * @hs_ep: The endpoint the request was on. 1946 * 1947 * Get first request from the ep queue, determine descriptor on which complete 1948 * happened. SW based on isoc_chain_num discovers which half of the descriptor 1949 * chain is currently in use by HW, adjusts dma_address and calculates index 1950 * of completed descriptor based on the value of DEPDMA register. Update actual 1951 * length of request, giveback to gadget. 1952 */ 1953 static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep) 1954 { 1955 struct dwc2_hsotg *hsotg = hs_ep->parent; 1956 struct dwc2_hsotg_req *hs_req; 1957 struct usb_request *ureq; 1958 int index; 1959 dma_addr_t dma_addr; 1960 u32 dma_reg; 1961 u32 depdma; 1962 u32 desc_sts; 1963 u32 mask; 1964 1965 hs_req = get_ep_head(hs_ep); 1966 if (!hs_req) { 1967 dev_warn(hsotg->dev, "%s: ISOC EP queue empty\n", __func__); 1968 return; 1969 } 1970 ureq = &hs_req->req; 1971 1972 dma_addr = hs_ep->desc_list_dma; 1973 1974 /* 1975 * If lower half of descriptor chain is currently use by SW, 1976 * that means higher half is being processed by HW, so shift 1977 * DMA address to higher half of descriptor chain. 1978 */ 1979 if (!hs_ep->isoc_chain_num) 1980 dma_addr += sizeof(struct dwc2_dma_desc) * 1981 (MAX_DMA_DESC_NUM_GENERIC / 2); 1982 1983 dma_reg = hs_ep->dir_in ? DIEPDMA(hs_ep->index) : DOEPDMA(hs_ep->index); 1984 depdma = dwc2_readl(hsotg->regs + dma_reg); 1985 1986 index = (depdma - dma_addr) / sizeof(struct dwc2_dma_desc) - 1; 1987 desc_sts = hs_ep->desc_list[index].status; 1988 1989 mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK : 1990 DEV_DMA_ISOC_RX_NBYTES_MASK; 1991 ureq->actual = ureq->length - 1992 ((desc_sts & mask) >> DEV_DMA_ISOC_NBYTES_SHIFT); 1993 1994 /* Adjust actual length for ISOC Out if length is not align of 4 */ 1995 if (!hs_ep->dir_in && ureq->length & 0x3) 1996 ureq->actual += 4 - (ureq->length & 0x3); 1997 1998 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); 1999 } 2000 2001 /* 2002 * dwc2_gadget_start_next_isoc_ddma - start next isoc request, if any. 2003 * @hs_ep: The isochronous endpoint to be re-enabled. 2004 * 2005 * If ep has been disabled due to last descriptor servicing (IN endpoint) or 2006 * BNA (OUT endpoint) check the status of other half of descriptor chain that 2007 * was under SW control till HW was busy and restart the endpoint if needed. 2008 */ 2009 static void dwc2_gadget_start_next_isoc_ddma(struct dwc2_hsotg_ep *hs_ep) 2010 { 2011 struct dwc2_hsotg *hsotg = hs_ep->parent; 2012 u32 depctl; 2013 u32 dma_reg; 2014 u32 ctrl; 2015 u32 dma_addr = hs_ep->desc_list_dma; 2016 unsigned char index = hs_ep->index; 2017 2018 dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index); 2019 depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); 2020 2021 ctrl = dwc2_readl(hsotg->regs + depctl); 2022 2023 /* 2024 * EP was disabled if HW has processed last descriptor or BNA was set. 2025 * So restart ep if SW has prepared new descriptor chain in ep_queue 2026 * routine while HW was busy. 2027 */ 2028 if (!(ctrl & DXEPCTL_EPENA)) { 2029 if (!hs_ep->next_desc) { 2030 dev_dbg(hsotg->dev, "%s: No more ISOC requests\n", 2031 __func__); 2032 return; 2033 } 2034 2035 dma_addr += sizeof(struct dwc2_dma_desc) * 2036 (MAX_DMA_DESC_NUM_GENERIC / 2) * 2037 hs_ep->isoc_chain_num; 2038 dwc2_writel(dma_addr, hsotg->regs + dma_reg); 2039 2040 ctrl |= DXEPCTL_EPENA | DXEPCTL_CNAK; 2041 dwc2_writel(ctrl, hsotg->regs + depctl); 2042 2043 /* Switch ISOC descriptor chain number being processed by SW*/ 2044 hs_ep->isoc_chain_num = (hs_ep->isoc_chain_num ^ 1) & 0x1; 2045 hs_ep->next_desc = 0; 2046 2047 dev_dbg(hsotg->dev, "%s: Restarted isochronous endpoint\n", 2048 __func__); 2049 } 2050 } 2051 2052 /** 2053 * dwc2_hsotg_rx_data - receive data from the FIFO for an endpoint 2054 * @hsotg: The device state. 2055 * @ep_idx: The endpoint index for the data 2056 * @size: The size of data in the fifo, in bytes 2057 * 2058 * The FIFO status shows there is data to read from the FIFO for a given 2059 * endpoint, so sort out whether we need to read the data into a request 2060 * that has been made for that endpoint. 2061 */ 2062 static void dwc2_hsotg_rx_data(struct dwc2_hsotg *hsotg, int ep_idx, int size) 2063 { 2064 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx]; 2065 struct dwc2_hsotg_req *hs_req = hs_ep->req; 2066 void __iomem *fifo = hsotg->regs + EPFIFO(ep_idx); 2067 int to_read; 2068 int max_req; 2069 int read_ptr; 2070 2071 2072 if (!hs_req) { 2073 u32 epctl = dwc2_readl(hsotg->regs + DOEPCTL(ep_idx)); 2074 int ptr; 2075 2076 dev_dbg(hsotg->dev, 2077 "%s: FIFO %d bytes on ep%d but no req (DXEPCTl=0x%08x)\n", 2078 __func__, size, ep_idx, epctl); 2079 2080 /* dump the data from the FIFO, we've nothing we can do */ 2081 for (ptr = 0; ptr < size; ptr += 4) 2082 (void)dwc2_readl(fifo); 2083 2084 return; 2085 } 2086 2087 to_read = size; 2088 read_ptr = hs_req->req.actual; 2089 max_req = hs_req->req.length - read_ptr; 2090 2091 dev_dbg(hsotg->dev, "%s: read %d/%d, done %d/%d\n", 2092 __func__, to_read, max_req, read_ptr, hs_req->req.length); 2093 2094 if (to_read > max_req) { 2095 /* 2096 * more data appeared than we where willing 2097 * to deal with in this request. 2098 */ 2099 2100 /* currently we don't deal this */ 2101 WARN_ON_ONCE(1); 2102 } 2103 2104 hs_ep->total_data += to_read; 2105 hs_req->req.actual += to_read; 2106 to_read = DIV_ROUND_UP(to_read, 4); 2107 2108 /* 2109 * note, we might over-write the buffer end by 3 bytes depending on 2110 * alignment of the data. 2111 */ 2112 ioread32_rep(fifo, hs_req->req.buf + read_ptr, to_read); 2113 } 2114 2115 /** 2116 * dwc2_hsotg_ep0_zlp - send/receive zero-length packet on control endpoint 2117 * @hsotg: The device instance 2118 * @dir_in: If IN zlp 2119 * 2120 * Generate a zero-length IN packet request for terminating a SETUP 2121 * transaction. 2122 * 2123 * Note, since we don't write any data to the TxFIFO, then it is 2124 * currently believed that we do not need to wait for any space in 2125 * the TxFIFO. 2126 */ 2127 static void dwc2_hsotg_ep0_zlp(struct dwc2_hsotg *hsotg, bool dir_in) 2128 { 2129 /* eps_out[0] is used in both directions */ 2130 hsotg->eps_out[0]->dir_in = dir_in; 2131 hsotg->ep0_state = dir_in ? DWC2_EP0_STATUS_IN : DWC2_EP0_STATUS_OUT; 2132 2133 dwc2_hsotg_program_zlp(hsotg, hsotg->eps_out[0]); 2134 } 2135 2136 static void dwc2_hsotg_change_ep_iso_parity(struct dwc2_hsotg *hsotg, 2137 u32 epctl_reg) 2138 { 2139 u32 ctrl; 2140 2141 ctrl = dwc2_readl(hsotg->regs + epctl_reg); 2142 if (ctrl & DXEPCTL_EOFRNUM) 2143 ctrl |= DXEPCTL_SETEVENFR; 2144 else 2145 ctrl |= DXEPCTL_SETODDFR; 2146 dwc2_writel(ctrl, hsotg->regs + epctl_reg); 2147 } 2148 2149 /* 2150 * dwc2_gadget_get_xfersize_ddma - get transferred bytes amount from desc 2151 * @hs_ep - The endpoint on which transfer went 2152 * 2153 * Iterate over endpoints descriptor chain and get info on bytes remained 2154 * in DMA descriptors after transfer has completed. Used for non isoc EPs. 2155 */ 2156 static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep) 2157 { 2158 struct dwc2_hsotg *hsotg = hs_ep->parent; 2159 unsigned int bytes_rem = 0; 2160 struct dwc2_dma_desc *desc = hs_ep->desc_list; 2161 int i; 2162 u32 status; 2163 2164 if (!desc) 2165 return -EINVAL; 2166 2167 for (i = 0; i < hs_ep->desc_count; ++i) { 2168 status = desc->status; 2169 bytes_rem += status & DEV_DMA_NBYTES_MASK; 2170 2171 if (status & DEV_DMA_STS_MASK) 2172 dev_err(hsotg->dev, "descriptor %d closed with %x\n", 2173 i, status & DEV_DMA_STS_MASK); 2174 } 2175 2176 return bytes_rem; 2177 } 2178 2179 /** 2180 * dwc2_hsotg_handle_outdone - handle receiving OutDone/SetupDone from RXFIFO 2181 * @hsotg: The device instance 2182 * @epnum: The endpoint received from 2183 * 2184 * The RXFIFO has delivered an OutDone event, which means that the data 2185 * transfer for an OUT endpoint has been completed, either by a short 2186 * packet or by the finish of a transfer. 2187 */ 2188 static void dwc2_hsotg_handle_outdone(struct dwc2_hsotg *hsotg, int epnum) 2189 { 2190 u32 epsize = dwc2_readl(hsotg->regs + DOEPTSIZ(epnum)); 2191 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum]; 2192 struct dwc2_hsotg_req *hs_req = hs_ep->req; 2193 struct usb_request *req = &hs_req->req; 2194 unsigned size_left = DXEPTSIZ_XFERSIZE_GET(epsize); 2195 int result = 0; 2196 2197 if (!hs_req) { 2198 dev_dbg(hsotg->dev, "%s: no request active\n", __func__); 2199 return; 2200 } 2201 2202 if (epnum == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_OUT) { 2203 dev_dbg(hsotg->dev, "zlp packet received\n"); 2204 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); 2205 dwc2_hsotg_enqueue_setup(hsotg); 2206 return; 2207 } 2208 2209 if (using_desc_dma(hsotg)) 2210 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); 2211 2212 if (using_dma(hsotg)) { 2213 unsigned size_done; 2214 2215 /* 2216 * Calculate the size of the transfer by checking how much 2217 * is left in the endpoint size register and then working it 2218 * out from the amount we loaded for the transfer. 2219 * 2220 * We need to do this as DMA pointers are always 32bit aligned 2221 * so may overshoot/undershoot the transfer. 2222 */ 2223 2224 size_done = hs_ep->size_loaded - size_left; 2225 size_done += hs_ep->last_load; 2226 2227 req->actual = size_done; 2228 } 2229 2230 /* if there is more request to do, schedule new transfer */ 2231 if (req->actual < req->length && size_left == 0) { 2232 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true); 2233 return; 2234 } 2235 2236 if (req->actual < req->length && req->short_not_ok) { 2237 dev_dbg(hsotg->dev, "%s: got %d/%d (short not ok) => error\n", 2238 __func__, req->actual, req->length); 2239 2240 /* 2241 * todo - what should we return here? there's no one else 2242 * even bothering to check the status. 2243 */ 2244 } 2245 2246 /* DDMA IN status phase will start from StsPhseRcvd interrupt */ 2247 if (!using_desc_dma(hsotg) && epnum == 0 && 2248 hsotg->ep0_state == DWC2_EP0_DATA_OUT) { 2249 /* Move to STATUS IN */ 2250 dwc2_hsotg_ep0_zlp(hsotg, true); 2251 return; 2252 } 2253 2254 /* 2255 * Slave mode OUT transfers do not go through XferComplete so 2256 * adjust the ISOC parity here. 2257 */ 2258 if (!using_dma(hsotg)) { 2259 if (hs_ep->isochronous && hs_ep->interval == 1) 2260 dwc2_hsotg_change_ep_iso_parity(hsotg, DOEPCTL(epnum)); 2261 else if (hs_ep->isochronous && hs_ep->interval > 1) 2262 dwc2_gadget_incr_frame_num(hs_ep); 2263 } 2264 2265 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, result); 2266 } 2267 2268 /** 2269 * dwc2_hsotg_handle_rx - RX FIFO has data 2270 * @hsotg: The device instance 2271 * 2272 * The IRQ handler has detected that the RX FIFO has some data in it 2273 * that requires processing, so find out what is in there and do the 2274 * appropriate read. 2275 * 2276 * The RXFIFO is a true FIFO, the packets coming out are still in packet 2277 * chunks, so if you have x packets received on an endpoint you'll get x 2278 * FIFO events delivered, each with a packet's worth of data in it. 2279 * 2280 * When using DMA, we should not be processing events from the RXFIFO 2281 * as the actual data should be sent to the memory directly and we turn 2282 * on the completion interrupts to get notifications of transfer completion. 2283 */ 2284 static void dwc2_hsotg_handle_rx(struct dwc2_hsotg *hsotg) 2285 { 2286 u32 grxstsr = dwc2_readl(hsotg->regs + GRXSTSP); 2287 u32 epnum, status, size; 2288 2289 WARN_ON(using_dma(hsotg)); 2290 2291 epnum = grxstsr & GRXSTS_EPNUM_MASK; 2292 status = grxstsr & GRXSTS_PKTSTS_MASK; 2293 2294 size = grxstsr & GRXSTS_BYTECNT_MASK; 2295 size >>= GRXSTS_BYTECNT_SHIFT; 2296 2297 dev_dbg(hsotg->dev, "%s: GRXSTSP=0x%08x (%d@%d)\n", 2298 __func__, grxstsr, size, epnum); 2299 2300 switch ((status & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT) { 2301 case GRXSTS_PKTSTS_GLOBALOUTNAK: 2302 dev_dbg(hsotg->dev, "GLOBALOUTNAK\n"); 2303 break; 2304 2305 case GRXSTS_PKTSTS_OUTDONE: 2306 dev_dbg(hsotg->dev, "OutDone (Frame=0x%08x)\n", 2307 dwc2_hsotg_read_frameno(hsotg)); 2308 2309 if (!using_dma(hsotg)) 2310 dwc2_hsotg_handle_outdone(hsotg, epnum); 2311 break; 2312 2313 case GRXSTS_PKTSTS_SETUPDONE: 2314 dev_dbg(hsotg->dev, 2315 "SetupDone (Frame=0x%08x, DOPEPCTL=0x%08x)\n", 2316 dwc2_hsotg_read_frameno(hsotg), 2317 dwc2_readl(hsotg->regs + DOEPCTL(0))); 2318 /* 2319 * Call dwc2_hsotg_handle_outdone here if it was not called from 2320 * GRXSTS_PKTSTS_OUTDONE. That is, if the core didn't 2321 * generate GRXSTS_PKTSTS_OUTDONE for setup packet. 2322 */ 2323 if (hsotg->ep0_state == DWC2_EP0_SETUP) 2324 dwc2_hsotg_handle_outdone(hsotg, epnum); 2325 break; 2326 2327 case GRXSTS_PKTSTS_OUTRX: 2328 dwc2_hsotg_rx_data(hsotg, epnum, size); 2329 break; 2330 2331 case GRXSTS_PKTSTS_SETUPRX: 2332 dev_dbg(hsotg->dev, 2333 "SetupRX (Frame=0x%08x, DOPEPCTL=0x%08x)\n", 2334 dwc2_hsotg_read_frameno(hsotg), 2335 dwc2_readl(hsotg->regs + DOEPCTL(0))); 2336 2337 WARN_ON(hsotg->ep0_state != DWC2_EP0_SETUP); 2338 2339 dwc2_hsotg_rx_data(hsotg, epnum, size); 2340 break; 2341 2342 default: 2343 dev_warn(hsotg->dev, "%s: unknown status %08x\n", 2344 __func__, grxstsr); 2345 2346 dwc2_hsotg_dump(hsotg); 2347 break; 2348 } 2349 } 2350 2351 /** 2352 * dwc2_hsotg_ep0_mps - turn max packet size into register setting 2353 * @mps: The maximum packet size in bytes. 2354 */ 2355 static u32 dwc2_hsotg_ep0_mps(unsigned int mps) 2356 { 2357 switch (mps) { 2358 case 64: 2359 return D0EPCTL_MPS_64; 2360 case 32: 2361 return D0EPCTL_MPS_32; 2362 case 16: 2363 return D0EPCTL_MPS_16; 2364 case 8: 2365 return D0EPCTL_MPS_8; 2366 } 2367 2368 /* bad max packet size, warn and return invalid result */ 2369 WARN_ON(1); 2370 return (u32)-1; 2371 } 2372 2373 /** 2374 * dwc2_hsotg_set_ep_maxpacket - set endpoint's max-packet field 2375 * @hsotg: The driver state. 2376 * @ep: The index number of the endpoint 2377 * @mps: The maximum packet size in bytes 2378 * @mc: The multicount value 2379 * 2380 * Configure the maximum packet size for the given endpoint, updating 2381 * the hardware control registers to reflect this. 2382 */ 2383 static void dwc2_hsotg_set_ep_maxpacket(struct dwc2_hsotg *hsotg, 2384 unsigned int ep, unsigned int mps, 2385 unsigned int mc, unsigned int dir_in) 2386 { 2387 struct dwc2_hsotg_ep *hs_ep; 2388 void __iomem *regs = hsotg->regs; 2389 u32 reg; 2390 2391 hs_ep = index_to_ep(hsotg, ep, dir_in); 2392 if (!hs_ep) 2393 return; 2394 2395 if (ep == 0) { 2396 u32 mps_bytes = mps; 2397 2398 /* EP0 is a special case */ 2399 mps = dwc2_hsotg_ep0_mps(mps_bytes); 2400 if (mps > 3) 2401 goto bad_mps; 2402 hs_ep->ep.maxpacket = mps_bytes; 2403 hs_ep->mc = 1; 2404 } else { 2405 if (mps > 1024) 2406 goto bad_mps; 2407 hs_ep->mc = mc; 2408 if (mc > 3) 2409 goto bad_mps; 2410 hs_ep->ep.maxpacket = mps; 2411 } 2412 2413 if (dir_in) { 2414 reg = dwc2_readl(regs + DIEPCTL(ep)); 2415 reg &= ~DXEPCTL_MPS_MASK; 2416 reg |= mps; 2417 dwc2_writel(reg, regs + DIEPCTL(ep)); 2418 } else { 2419 reg = dwc2_readl(regs + DOEPCTL(ep)); 2420 reg &= ~DXEPCTL_MPS_MASK; 2421 reg |= mps; 2422 dwc2_writel(reg, regs + DOEPCTL(ep)); 2423 } 2424 2425 return; 2426 2427 bad_mps: 2428 dev_err(hsotg->dev, "ep%d: bad mps of %d\n", ep, mps); 2429 } 2430 2431 /** 2432 * dwc2_hsotg_txfifo_flush - flush Tx FIFO 2433 * @hsotg: The driver state 2434 * @idx: The index for the endpoint (0..15) 2435 */ 2436 static void dwc2_hsotg_txfifo_flush(struct dwc2_hsotg *hsotg, unsigned int idx) 2437 { 2438 int timeout; 2439 int val; 2440 2441 dwc2_writel(GRSTCTL_TXFNUM(idx) | GRSTCTL_TXFFLSH, 2442 hsotg->regs + GRSTCTL); 2443 2444 /* wait until the fifo is flushed */ 2445 timeout = 100; 2446 2447 while (1) { 2448 val = dwc2_readl(hsotg->regs + GRSTCTL); 2449 2450 if ((val & (GRSTCTL_TXFFLSH)) == 0) 2451 break; 2452 2453 if (--timeout == 0) { 2454 dev_err(hsotg->dev, 2455 "%s: timeout flushing fifo (GRSTCTL=%08x)\n", 2456 __func__, val); 2457 break; 2458 } 2459 2460 udelay(1); 2461 } 2462 } 2463 2464 /** 2465 * dwc2_hsotg_trytx - check to see if anything needs transmitting 2466 * @hsotg: The driver state 2467 * @hs_ep: The driver endpoint to check. 2468 * 2469 * Check to see if there is a request that has data to send, and if so 2470 * make an attempt to write data into the FIFO. 2471 */ 2472 static int dwc2_hsotg_trytx(struct dwc2_hsotg *hsotg, 2473 struct dwc2_hsotg_ep *hs_ep) 2474 { 2475 struct dwc2_hsotg_req *hs_req = hs_ep->req; 2476 2477 if (!hs_ep->dir_in || !hs_req) { 2478 /** 2479 * if request is not enqueued, we disable interrupts 2480 * for endpoints, excepting ep0 2481 */ 2482 if (hs_ep->index != 0) 2483 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, 2484 hs_ep->dir_in, 0); 2485 return 0; 2486 } 2487 2488 if (hs_req->req.actual < hs_req->req.length) { 2489 dev_dbg(hsotg->dev, "trying to write more for ep%d\n", 2490 hs_ep->index); 2491 return dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req); 2492 } 2493 2494 return 0; 2495 } 2496 2497 /** 2498 * dwc2_hsotg_complete_in - complete IN transfer 2499 * @hsotg: The device state. 2500 * @hs_ep: The endpoint that has just completed. 2501 * 2502 * An IN transfer has been completed, update the transfer's state and then 2503 * call the relevant completion routines. 2504 */ 2505 static void dwc2_hsotg_complete_in(struct dwc2_hsotg *hsotg, 2506 struct dwc2_hsotg_ep *hs_ep) 2507 { 2508 struct dwc2_hsotg_req *hs_req = hs_ep->req; 2509 u32 epsize = dwc2_readl(hsotg->regs + DIEPTSIZ(hs_ep->index)); 2510 int size_left, size_done; 2511 2512 if (!hs_req) { 2513 dev_dbg(hsotg->dev, "XferCompl but no req\n"); 2514 return; 2515 } 2516 2517 /* Finish ZLP handling for IN EP0 transactions */ 2518 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) { 2519 dev_dbg(hsotg->dev, "zlp packet sent\n"); 2520 2521 /* 2522 * While send zlp for DWC2_EP0_STATUS_IN EP direction was 2523 * changed to IN. Change back to complete OUT transfer request 2524 */ 2525 hs_ep->dir_in = 0; 2526 2527 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); 2528 if (hsotg->test_mode) { 2529 int ret; 2530 2531 ret = dwc2_hsotg_set_test_mode(hsotg, hsotg->test_mode); 2532 if (ret < 0) { 2533 dev_dbg(hsotg->dev, "Invalid Test #%d\n", 2534 hsotg->test_mode); 2535 dwc2_hsotg_stall_ep0(hsotg); 2536 return; 2537 } 2538 } 2539 dwc2_hsotg_enqueue_setup(hsotg); 2540 return; 2541 } 2542 2543 /* 2544 * Calculate the size of the transfer by checking how much is left 2545 * in the endpoint size register and then working it out from 2546 * the amount we loaded for the transfer. 2547 * 2548 * We do this even for DMA, as the transfer may have incremented 2549 * past the end of the buffer (DMA transfers are always 32bit 2550 * aligned). 2551 */ 2552 if (using_desc_dma(hsotg)) { 2553 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); 2554 if (size_left < 0) 2555 dev_err(hsotg->dev, "error parsing DDMA results %d\n", 2556 size_left); 2557 } else { 2558 size_left = DXEPTSIZ_XFERSIZE_GET(epsize); 2559 } 2560 2561 size_done = hs_ep->size_loaded - size_left; 2562 size_done += hs_ep->last_load; 2563 2564 if (hs_req->req.actual != size_done) 2565 dev_dbg(hsotg->dev, "%s: adjusting size done %d => %d\n", 2566 __func__, hs_req->req.actual, size_done); 2567 2568 hs_req->req.actual = size_done; 2569 dev_dbg(hsotg->dev, "req->length:%d req->actual:%d req->zero:%d\n", 2570 hs_req->req.length, hs_req->req.actual, hs_req->req.zero); 2571 2572 if (!size_left && hs_req->req.actual < hs_req->req.length) { 2573 dev_dbg(hsotg->dev, "%s trying more for req...\n", __func__); 2574 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true); 2575 return; 2576 } 2577 2578 /* Zlp for all endpoints, for ep0 only in DATA IN stage */ 2579 if (hs_ep->send_zlp) { 2580 dwc2_hsotg_program_zlp(hsotg, hs_ep); 2581 hs_ep->send_zlp = 0; 2582 /* transfer will be completed on next complete interrupt */ 2583 return; 2584 } 2585 2586 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) { 2587 /* Move to STATUS OUT */ 2588 dwc2_hsotg_ep0_zlp(hsotg, false); 2589 return; 2590 } 2591 2592 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); 2593 } 2594 2595 /** 2596 * dwc2_gadget_read_ep_interrupts - reads interrupts for given ep 2597 * @hsotg: The device state. 2598 * @idx: Index of ep. 2599 * @dir_in: Endpoint direction 1-in 0-out. 2600 * 2601 * Reads for endpoint with given index and direction, by masking 2602 * epint_reg with coresponding mask. 2603 */ 2604 static u32 dwc2_gadget_read_ep_interrupts(struct dwc2_hsotg *hsotg, 2605 unsigned int idx, int dir_in) 2606 { 2607 u32 epmsk_reg = dir_in ? DIEPMSK : DOEPMSK; 2608 u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx); 2609 u32 ints; 2610 u32 mask; 2611 u32 diepempmsk; 2612 2613 mask = dwc2_readl(hsotg->regs + epmsk_reg); 2614 diepempmsk = dwc2_readl(hsotg->regs + DIEPEMPMSK); 2615 mask |= ((diepempmsk >> idx) & 0x1) ? DIEPMSK_TXFIFOEMPTY : 0; 2616 mask |= DXEPINT_SETUP_RCVD; 2617 2618 ints = dwc2_readl(hsotg->regs + epint_reg); 2619 ints &= mask; 2620 return ints; 2621 } 2622 2623 /** 2624 * dwc2_gadget_handle_ep_disabled - handle DXEPINT_EPDISBLD 2625 * @hs_ep: The endpoint on which interrupt is asserted. 2626 * 2627 * This interrupt indicates that the endpoint has been disabled per the 2628 * application's request. 2629 * 2630 * For IN endpoints flushes txfifo, in case of BULK clears DCTL_CGNPINNAK, 2631 * in case of ISOC completes current request. 2632 * 2633 * For ISOC-OUT endpoints completes expired requests. If there is remaining 2634 * request starts it. 2635 */ 2636 static void dwc2_gadget_handle_ep_disabled(struct dwc2_hsotg_ep *hs_ep) 2637 { 2638 struct dwc2_hsotg *hsotg = hs_ep->parent; 2639 struct dwc2_hsotg_req *hs_req; 2640 unsigned char idx = hs_ep->index; 2641 int dir_in = hs_ep->dir_in; 2642 u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx); 2643 int dctl = dwc2_readl(hsotg->regs + DCTL); 2644 2645 dev_dbg(hsotg->dev, "%s: EPDisbld\n", __func__); 2646 2647 if (dir_in) { 2648 int epctl = dwc2_readl(hsotg->regs + epctl_reg); 2649 2650 dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index); 2651 2652 if (hs_ep->isochronous) { 2653 dwc2_hsotg_complete_in(hsotg, hs_ep); 2654 return; 2655 } 2656 2657 if ((epctl & DXEPCTL_STALL) && (epctl & DXEPCTL_EPTYPE_BULK)) { 2658 int dctl = dwc2_readl(hsotg->regs + DCTL); 2659 2660 dctl |= DCTL_CGNPINNAK; 2661 dwc2_writel(dctl, hsotg->regs + DCTL); 2662 } 2663 return; 2664 } 2665 2666 if (dctl & DCTL_GOUTNAKSTS) { 2667 dctl |= DCTL_CGOUTNAK; 2668 dwc2_writel(dctl, hsotg->regs + DCTL); 2669 } 2670 2671 if (!hs_ep->isochronous) 2672 return; 2673 2674 if (list_empty(&hs_ep->queue)) { 2675 dev_dbg(hsotg->dev, "%s: complete_ep 0x%p, ep->queue empty!\n", 2676 __func__, hs_ep); 2677 return; 2678 } 2679 2680 do { 2681 hs_req = get_ep_head(hs_ep); 2682 if (hs_req) 2683 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 2684 -ENODATA); 2685 dwc2_gadget_incr_frame_num(hs_ep); 2686 } while (dwc2_gadget_target_frame_elapsed(hs_ep)); 2687 2688 dwc2_gadget_start_next_request(hs_ep); 2689 } 2690 2691 /** 2692 * dwc2_gadget_handle_out_token_ep_disabled - handle DXEPINT_OUTTKNEPDIS 2693 * @hs_ep: The endpoint on which interrupt is asserted. 2694 * 2695 * This is starting point for ISOC-OUT transfer, synchronization done with 2696 * first out token received from host while corresponding EP is disabled. 2697 * 2698 * Device does not know initial frame in which out token will come. For this 2699 * HW generates OUTTKNEPDIS - out token is received while EP is disabled. Upon 2700 * getting this interrupt SW starts calculation for next transfer frame. 2701 */ 2702 static void dwc2_gadget_handle_out_token_ep_disabled(struct dwc2_hsotg_ep *ep) 2703 { 2704 struct dwc2_hsotg *hsotg = ep->parent; 2705 int dir_in = ep->dir_in; 2706 u32 doepmsk; 2707 u32 tmp; 2708 2709 if (dir_in || !ep->isochronous) 2710 return; 2711 2712 /* 2713 * Store frame in which irq was asserted here, as 2714 * it can change while completing request below. 2715 */ 2716 tmp = dwc2_hsotg_read_frameno(hsotg); 2717 2718 dwc2_hsotg_complete_request(hsotg, ep, get_ep_head(ep), -ENODATA); 2719 2720 if (using_desc_dma(hsotg)) { 2721 if (ep->target_frame == TARGET_FRAME_INITIAL) { 2722 /* Start first ISO Out */ 2723 ep->target_frame = tmp; 2724 dwc2_gadget_start_isoc_ddma(ep); 2725 } 2726 return; 2727 } 2728 2729 if (ep->interval > 1 && 2730 ep->target_frame == TARGET_FRAME_INITIAL) { 2731 u32 dsts; 2732 u32 ctrl; 2733 2734 dsts = dwc2_readl(hsotg->regs + DSTS); 2735 ep->target_frame = dwc2_hsotg_read_frameno(hsotg); 2736 dwc2_gadget_incr_frame_num(ep); 2737 2738 ctrl = dwc2_readl(hsotg->regs + DOEPCTL(ep->index)); 2739 if (ep->target_frame & 0x1) 2740 ctrl |= DXEPCTL_SETODDFR; 2741 else 2742 ctrl |= DXEPCTL_SETEVENFR; 2743 2744 dwc2_writel(ctrl, hsotg->regs + DOEPCTL(ep->index)); 2745 } 2746 2747 dwc2_gadget_start_next_request(ep); 2748 doepmsk = dwc2_readl(hsotg->regs + DOEPMSK); 2749 doepmsk &= ~DOEPMSK_OUTTKNEPDISMSK; 2750 dwc2_writel(doepmsk, hsotg->regs + DOEPMSK); 2751 } 2752 2753 /** 2754 * dwc2_gadget_handle_nak - handle NAK interrupt 2755 * @hs_ep: The endpoint on which interrupt is asserted. 2756 * 2757 * This is starting point for ISOC-IN transfer, synchronization done with 2758 * first IN token received from host while corresponding EP is disabled. 2759 * 2760 * Device does not know when first one token will arrive from host. On first 2761 * token arrival HW generates 2 interrupts: 'in token received while FIFO empty' 2762 * and 'NAK'. NAK interrupt for ISOC-IN means that token has arrived and ZLP was 2763 * sent in response to that as there was no data in FIFO. SW is basing on this 2764 * interrupt to obtain frame in which token has come and then based on the 2765 * interval calculates next frame for transfer. 2766 */ 2767 static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep) 2768 { 2769 struct dwc2_hsotg *hsotg = hs_ep->parent; 2770 int dir_in = hs_ep->dir_in; 2771 2772 if (!dir_in || !hs_ep->isochronous) 2773 return; 2774 2775 if (hs_ep->target_frame == TARGET_FRAME_INITIAL) { 2776 hs_ep->target_frame = dwc2_hsotg_read_frameno(hsotg); 2777 2778 if (using_desc_dma(hsotg)) { 2779 dwc2_gadget_start_isoc_ddma(hs_ep); 2780 return; 2781 } 2782 2783 if (hs_ep->interval > 1) { 2784 u32 ctrl = dwc2_readl(hsotg->regs + 2785 DIEPCTL(hs_ep->index)); 2786 if (hs_ep->target_frame & 0x1) 2787 ctrl |= DXEPCTL_SETODDFR; 2788 else 2789 ctrl |= DXEPCTL_SETEVENFR; 2790 2791 dwc2_writel(ctrl, hsotg->regs + DIEPCTL(hs_ep->index)); 2792 } 2793 2794 dwc2_hsotg_complete_request(hsotg, hs_ep, 2795 get_ep_head(hs_ep), 0); 2796 } 2797 2798 dwc2_gadget_incr_frame_num(hs_ep); 2799 } 2800 2801 /** 2802 * dwc2_hsotg_epint - handle an in/out endpoint interrupt 2803 * @hsotg: The driver state 2804 * @idx: The index for the endpoint (0..15) 2805 * @dir_in: Set if this is an IN endpoint 2806 * 2807 * Process and clear any interrupt pending for an individual endpoint 2808 */ 2809 static void dwc2_hsotg_epint(struct dwc2_hsotg *hsotg, unsigned int idx, 2810 int dir_in) 2811 { 2812 struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in); 2813 u32 epint_reg = dir_in ? DIEPINT(idx) : DOEPINT(idx); 2814 u32 epctl_reg = dir_in ? DIEPCTL(idx) : DOEPCTL(idx); 2815 u32 epsiz_reg = dir_in ? DIEPTSIZ(idx) : DOEPTSIZ(idx); 2816 u32 ints; 2817 u32 ctrl; 2818 2819 ints = dwc2_gadget_read_ep_interrupts(hsotg, idx, dir_in); 2820 ctrl = dwc2_readl(hsotg->regs + epctl_reg); 2821 2822 /* Clear endpoint interrupts */ 2823 dwc2_writel(ints, hsotg->regs + epint_reg); 2824 2825 if (!hs_ep) { 2826 dev_err(hsotg->dev, "%s:Interrupt for unconfigured ep%d(%s)\n", 2827 __func__, idx, dir_in ? "in" : "out"); 2828 return; 2829 } 2830 2831 dev_dbg(hsotg->dev, "%s: ep%d(%s) DxEPINT=0x%08x\n", 2832 __func__, idx, dir_in ? "in" : "out", ints); 2833 2834 /* Don't process XferCompl interrupt if it is a setup packet */ 2835 if (idx == 0 && (ints & (DXEPINT_SETUP | DXEPINT_SETUP_RCVD))) 2836 ints &= ~DXEPINT_XFERCOMPL; 2837 2838 /* 2839 * Don't process XferCompl interrupt in DDMA if EP0 is still in SETUP 2840 * stage and xfercomplete was generated without SETUP phase done 2841 * interrupt. SW should parse received setup packet only after host's 2842 * exit from setup phase of control transfer. 2843 */ 2844 if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in && 2845 hsotg->ep0_state == DWC2_EP0_SETUP && !(ints & DXEPINT_SETUP)) 2846 ints &= ~DXEPINT_XFERCOMPL; 2847 2848 if (ints & DXEPINT_XFERCOMPL) { 2849 dev_dbg(hsotg->dev, 2850 "%s: XferCompl: DxEPCTL=0x%08x, DXEPTSIZ=%08x\n", 2851 __func__, dwc2_readl(hsotg->regs + epctl_reg), 2852 dwc2_readl(hsotg->regs + epsiz_reg)); 2853 2854 /* In DDMA handle isochronous requests separately */ 2855 if (using_desc_dma(hsotg) && hs_ep->isochronous) { 2856 dwc2_gadget_complete_isoc_request_ddma(hs_ep); 2857 /* Try to start next isoc request */ 2858 dwc2_gadget_start_next_isoc_ddma(hs_ep); 2859 } else if (dir_in) { 2860 /* 2861 * We get OutDone from the FIFO, so we only 2862 * need to look at completing IN requests here 2863 * if operating slave mode 2864 */ 2865 if (hs_ep->isochronous && hs_ep->interval > 1) 2866 dwc2_gadget_incr_frame_num(hs_ep); 2867 2868 dwc2_hsotg_complete_in(hsotg, hs_ep); 2869 if (ints & DXEPINT_NAKINTRPT) 2870 ints &= ~DXEPINT_NAKINTRPT; 2871 2872 if (idx == 0 && !hs_ep->req) 2873 dwc2_hsotg_enqueue_setup(hsotg); 2874 } else if (using_dma(hsotg)) { 2875 /* 2876 * We're using DMA, we need to fire an OutDone here 2877 * as we ignore the RXFIFO. 2878 */ 2879 if (hs_ep->isochronous && hs_ep->interval > 1) 2880 dwc2_gadget_incr_frame_num(hs_ep); 2881 2882 dwc2_hsotg_handle_outdone(hsotg, idx); 2883 } 2884 } 2885 2886 if (ints & DXEPINT_EPDISBLD) 2887 dwc2_gadget_handle_ep_disabled(hs_ep); 2888 2889 if (ints & DXEPINT_OUTTKNEPDIS) 2890 dwc2_gadget_handle_out_token_ep_disabled(hs_ep); 2891 2892 if (ints & DXEPINT_NAKINTRPT) 2893 dwc2_gadget_handle_nak(hs_ep); 2894 2895 if (ints & DXEPINT_AHBERR) 2896 dev_dbg(hsotg->dev, "%s: AHBErr\n", __func__); 2897 2898 if (ints & DXEPINT_SETUP) { /* Setup or Timeout */ 2899 dev_dbg(hsotg->dev, "%s: Setup/Timeout\n", __func__); 2900 2901 if (using_dma(hsotg) && idx == 0) { 2902 /* 2903 * this is the notification we've received a 2904 * setup packet. In non-DMA mode we'd get this 2905 * from the RXFIFO, instead we need to process 2906 * the setup here. 2907 */ 2908 2909 if (dir_in) 2910 WARN_ON_ONCE(1); 2911 else 2912 dwc2_hsotg_handle_outdone(hsotg, 0); 2913 } 2914 } 2915 2916 if (ints & DXEPINT_STSPHSERCVD) { 2917 dev_dbg(hsotg->dev, "%s: StsPhseRcvd\n", __func__); 2918 2919 /* Move to STATUS IN for DDMA */ 2920 if (using_desc_dma(hsotg)) 2921 dwc2_hsotg_ep0_zlp(hsotg, true); 2922 } 2923 2924 if (ints & DXEPINT_BACK2BACKSETUP) 2925 dev_dbg(hsotg->dev, "%s: B2BSetup/INEPNakEff\n", __func__); 2926 2927 if (ints & DXEPINT_BNAINTR) { 2928 dev_dbg(hsotg->dev, "%s: BNA interrupt\n", __func__); 2929 2930 /* 2931 * Try to start next isoc request, if any. 2932 * Sometimes the endpoint remains enabled after BNA interrupt 2933 * assertion, which is not expected, hence we can enter here 2934 * couple of times. 2935 */ 2936 if (hs_ep->isochronous) 2937 dwc2_gadget_start_next_isoc_ddma(hs_ep); 2938 } 2939 2940 if (dir_in && !hs_ep->isochronous) { 2941 /* not sure if this is important, but we'll clear it anyway */ 2942 if (ints & DXEPINT_INTKNTXFEMP) { 2943 dev_dbg(hsotg->dev, "%s: ep%d: INTknTXFEmpMsk\n", 2944 __func__, idx); 2945 } 2946 2947 /* this probably means something bad is happening */ 2948 if (ints & DXEPINT_INTKNEPMIS) { 2949 dev_warn(hsotg->dev, "%s: ep%d: INTknEP\n", 2950 __func__, idx); 2951 } 2952 2953 /* FIFO has space or is empty (see GAHBCFG) */ 2954 if (hsotg->dedicated_fifos && 2955 ints & DXEPINT_TXFEMP) { 2956 dev_dbg(hsotg->dev, "%s: ep%d: TxFIFOEmpty\n", 2957 __func__, idx); 2958 if (!using_dma(hsotg)) 2959 dwc2_hsotg_trytx(hsotg, hs_ep); 2960 } 2961 } 2962 } 2963 2964 /** 2965 * dwc2_hsotg_irq_enumdone - Handle EnumDone interrupt (enumeration done) 2966 * @hsotg: The device state. 2967 * 2968 * Handle updating the device settings after the enumeration phase has 2969 * been completed. 2970 */ 2971 static void dwc2_hsotg_irq_enumdone(struct dwc2_hsotg *hsotg) 2972 { 2973 u32 dsts = dwc2_readl(hsotg->regs + DSTS); 2974 int ep0_mps = 0, ep_mps = 8; 2975 2976 /* 2977 * This should signal the finish of the enumeration phase 2978 * of the USB handshaking, so we should now know what rate 2979 * we connected at. 2980 */ 2981 2982 dev_dbg(hsotg->dev, "EnumDone (DSTS=0x%08x)\n", dsts); 2983 2984 /* 2985 * note, since we're limited by the size of transfer on EP0, and 2986 * it seems IN transfers must be a even number of packets we do 2987 * not advertise a 64byte MPS on EP0. 2988 */ 2989 2990 /* catch both EnumSpd_FS and EnumSpd_FS48 */ 2991 switch ((dsts & DSTS_ENUMSPD_MASK) >> DSTS_ENUMSPD_SHIFT) { 2992 case DSTS_ENUMSPD_FS: 2993 case DSTS_ENUMSPD_FS48: 2994 hsotg->gadget.speed = USB_SPEED_FULL; 2995 ep0_mps = EP0_MPS_LIMIT; 2996 ep_mps = 1023; 2997 break; 2998 2999 case DSTS_ENUMSPD_HS: 3000 hsotg->gadget.speed = USB_SPEED_HIGH; 3001 ep0_mps = EP0_MPS_LIMIT; 3002 ep_mps = 1024; 3003 break; 3004 3005 case DSTS_ENUMSPD_LS: 3006 hsotg->gadget.speed = USB_SPEED_LOW; 3007 ep0_mps = 8; 3008 ep_mps = 8; 3009 /* 3010 * note, we don't actually support LS in this driver at the 3011 * moment, and the documentation seems to imply that it isn't 3012 * supported by the PHYs on some of the devices. 3013 */ 3014 break; 3015 } 3016 dev_info(hsotg->dev, "new device is %s\n", 3017 usb_speed_string(hsotg->gadget.speed)); 3018 3019 /* 3020 * we should now know the maximum packet size for an 3021 * endpoint, so set the endpoints to a default value. 3022 */ 3023 3024 if (ep0_mps) { 3025 int i; 3026 /* Initialize ep0 for both in and out directions */ 3027 dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 1); 3028 dwc2_hsotg_set_ep_maxpacket(hsotg, 0, ep0_mps, 0, 0); 3029 for (i = 1; i < hsotg->num_of_eps; i++) { 3030 if (hsotg->eps_in[i]) 3031 dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, 3032 0, 1); 3033 if (hsotg->eps_out[i]) 3034 dwc2_hsotg_set_ep_maxpacket(hsotg, i, ep_mps, 3035 0, 0); 3036 } 3037 } 3038 3039 /* ensure after enumeration our EP0 is active */ 3040 3041 dwc2_hsotg_enqueue_setup(hsotg); 3042 3043 dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n", 3044 dwc2_readl(hsotg->regs + DIEPCTL0), 3045 dwc2_readl(hsotg->regs + DOEPCTL0)); 3046 } 3047 3048 /** 3049 * kill_all_requests - remove all requests from the endpoint's queue 3050 * @hsotg: The device state. 3051 * @ep: The endpoint the requests may be on. 3052 * @result: The result code to use. 3053 * 3054 * Go through the requests on the given endpoint and mark them 3055 * completed with the given result code. 3056 */ 3057 static void kill_all_requests(struct dwc2_hsotg *hsotg, 3058 struct dwc2_hsotg_ep *ep, 3059 int result) 3060 { 3061 struct dwc2_hsotg_req *req, *treq; 3062 unsigned size; 3063 3064 ep->req = NULL; 3065 3066 list_for_each_entry_safe(req, treq, &ep->queue, queue) 3067 dwc2_hsotg_complete_request(hsotg, ep, req, 3068 result); 3069 3070 if (!hsotg->dedicated_fifos) 3071 return; 3072 size = (dwc2_readl(hsotg->regs + DTXFSTS(ep->fifo_index)) & 0xffff) * 4; 3073 if (size < ep->fifo_size) 3074 dwc2_hsotg_txfifo_flush(hsotg, ep->fifo_index); 3075 } 3076 3077 /** 3078 * dwc2_hsotg_disconnect - disconnect service 3079 * @hsotg: The device state. 3080 * 3081 * The device has been disconnected. Remove all current 3082 * transactions and signal the gadget driver that this 3083 * has happened. 3084 */ 3085 void dwc2_hsotg_disconnect(struct dwc2_hsotg *hsotg) 3086 { 3087 unsigned ep; 3088 3089 if (!hsotg->connected) 3090 return; 3091 3092 hsotg->connected = 0; 3093 hsotg->test_mode = 0; 3094 3095 for (ep = 0; ep < hsotg->num_of_eps; ep++) { 3096 if (hsotg->eps_in[ep]) 3097 kill_all_requests(hsotg, hsotg->eps_in[ep], 3098 -ESHUTDOWN); 3099 if (hsotg->eps_out[ep]) 3100 kill_all_requests(hsotg, hsotg->eps_out[ep], 3101 -ESHUTDOWN); 3102 } 3103 3104 call_gadget(hsotg, disconnect); 3105 hsotg->lx_state = DWC2_L3; 3106 } 3107 3108 /** 3109 * dwc2_hsotg_irq_fifoempty - TX FIFO empty interrupt handler 3110 * @hsotg: The device state: 3111 * @periodic: True if this is a periodic FIFO interrupt 3112 */ 3113 static void dwc2_hsotg_irq_fifoempty(struct dwc2_hsotg *hsotg, bool periodic) 3114 { 3115 struct dwc2_hsotg_ep *ep; 3116 int epno, ret; 3117 3118 /* look through for any more data to transmit */ 3119 for (epno = 0; epno < hsotg->num_of_eps; epno++) { 3120 ep = index_to_ep(hsotg, epno, 1); 3121 3122 if (!ep) 3123 continue; 3124 3125 if (!ep->dir_in) 3126 continue; 3127 3128 if ((periodic && !ep->periodic) || 3129 (!periodic && ep->periodic)) 3130 continue; 3131 3132 ret = dwc2_hsotg_trytx(hsotg, ep); 3133 if (ret < 0) 3134 break; 3135 } 3136 } 3137 3138 /* IRQ flags which will trigger a retry around the IRQ loop */ 3139 #define IRQ_RETRY_MASK (GINTSTS_NPTXFEMP | \ 3140 GINTSTS_PTXFEMP | \ 3141 GINTSTS_RXFLVL) 3142 3143 /** 3144 * dwc2_hsotg_core_init - issue softreset to the core 3145 * @hsotg: The device state 3146 * 3147 * Issue a soft reset to the core, and await the core finishing it. 3148 */ 3149 void dwc2_hsotg_core_init_disconnected(struct dwc2_hsotg *hsotg, 3150 bool is_usb_reset) 3151 { 3152 u32 intmsk; 3153 u32 val; 3154 u32 usbcfg; 3155 u32 dcfg = 0; 3156 3157 /* Kill any ep0 requests as controller will be reinitialized */ 3158 kill_all_requests(hsotg, hsotg->eps_out[0], -ECONNRESET); 3159 3160 if (!is_usb_reset) 3161 if (dwc2_core_reset(hsotg)) 3162 return; 3163 3164 /* 3165 * we must now enable ep0 ready for host detection and then 3166 * set configuration. 3167 */ 3168 3169 /* keep other bits untouched (so e.g. forced modes are not lost) */ 3170 usbcfg = dwc2_readl(hsotg->regs + GUSBCFG); 3171 usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP | 3172 GUSBCFG_HNPCAP); 3173 3174 if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS && 3175 (hsotg->params.speed == DWC2_SPEED_PARAM_FULL || 3176 hsotg->params.speed == DWC2_SPEED_PARAM_LOW)) { 3177 /* FS/LS Dedicated Transceiver Interface */ 3178 usbcfg |= GUSBCFG_PHYSEL; 3179 } else { 3180 /* set the PLL on, remove the HNP/SRP and set the PHY */ 3181 val = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5; 3182 usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) | 3183 (val << GUSBCFG_USBTRDTIM_SHIFT); 3184 } 3185 dwc2_writel(usbcfg, hsotg->regs + GUSBCFG); 3186 3187 dwc2_hsotg_init_fifo(hsotg); 3188 3189 if (!is_usb_reset) 3190 __orr32(hsotg->regs + DCTL, DCTL_SFTDISCON); 3191 3192 dcfg |= DCFG_EPMISCNT(1); 3193 3194 switch (hsotg->params.speed) { 3195 case DWC2_SPEED_PARAM_LOW: 3196 dcfg |= DCFG_DEVSPD_LS; 3197 break; 3198 case DWC2_SPEED_PARAM_FULL: 3199 if (hsotg->params.phy_type == DWC2_PHY_TYPE_PARAM_FS) 3200 dcfg |= DCFG_DEVSPD_FS48; 3201 else 3202 dcfg |= DCFG_DEVSPD_FS; 3203 break; 3204 default: 3205 dcfg |= DCFG_DEVSPD_HS; 3206 } 3207 3208 dwc2_writel(dcfg, hsotg->regs + DCFG); 3209 3210 /* Clear any pending OTG interrupts */ 3211 dwc2_writel(0xffffffff, hsotg->regs + GOTGINT); 3212 3213 /* Clear any pending interrupts */ 3214 dwc2_writel(0xffffffff, hsotg->regs + GINTSTS); 3215 intmsk = GINTSTS_ERLYSUSP | GINTSTS_SESSREQINT | 3216 GINTSTS_GOUTNAKEFF | GINTSTS_GINNAKEFF | 3217 GINTSTS_USBRST | GINTSTS_RESETDET | 3218 GINTSTS_ENUMDONE | GINTSTS_OTGINT | 3219 GINTSTS_USBSUSP | GINTSTS_WKUPINT; 3220 3221 if (!using_desc_dma(hsotg)) 3222 intmsk |= GINTSTS_INCOMPL_SOIN | GINTSTS_INCOMPL_SOOUT; 3223 3224 if (hsotg->params.external_id_pin_ctl <= 0) 3225 intmsk |= GINTSTS_CONIDSTSCHNG; 3226 3227 dwc2_writel(intmsk, hsotg->regs + GINTMSK); 3228 3229 if (using_dma(hsotg)) { 3230 dwc2_writel(GAHBCFG_GLBL_INTR_EN | GAHBCFG_DMA_EN | 3231 (GAHBCFG_HBSTLEN_INCR4 << GAHBCFG_HBSTLEN_SHIFT), 3232 hsotg->regs + GAHBCFG); 3233 3234 /* Set DDMA mode support in the core if needed */ 3235 if (using_desc_dma(hsotg)) 3236 __orr32(hsotg->regs + DCFG, DCFG_DESCDMA_EN); 3237 3238 } else { 3239 dwc2_writel(((hsotg->dedicated_fifos) ? 3240 (GAHBCFG_NP_TXF_EMP_LVL | 3241 GAHBCFG_P_TXF_EMP_LVL) : 0) | 3242 GAHBCFG_GLBL_INTR_EN, hsotg->regs + GAHBCFG); 3243 } 3244 3245 /* 3246 * If INTknTXFEmpMsk is enabled, it's important to disable ep interrupts 3247 * when we have no data to transfer. Otherwise we get being flooded by 3248 * interrupts. 3249 */ 3250 3251 dwc2_writel(((hsotg->dedicated_fifos && !using_dma(hsotg)) ? 3252 DIEPMSK_TXFIFOEMPTY | DIEPMSK_INTKNTXFEMPMSK : 0) | 3253 DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK | 3254 DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK, 3255 hsotg->regs + DIEPMSK); 3256 3257 /* 3258 * don't need XferCompl, we get that from RXFIFO in slave mode. In 3259 * DMA mode we may need this and StsPhseRcvd. 3260 */ 3261 dwc2_writel((using_dma(hsotg) ? (DIEPMSK_XFERCOMPLMSK | 3262 DOEPMSK_STSPHSERCVDMSK) : 0) | 3263 DOEPMSK_EPDISBLDMSK | DOEPMSK_AHBERRMSK | 3264 DOEPMSK_SETUPMSK, 3265 hsotg->regs + DOEPMSK); 3266 3267 /* Enable BNA interrupt for DDMA */ 3268 if (using_desc_dma(hsotg)) 3269 __orr32(hsotg->regs + DOEPMSK, DOEPMSK_BNAMSK); 3270 3271 dwc2_writel(0, hsotg->regs + DAINTMSK); 3272 3273 dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n", 3274 dwc2_readl(hsotg->regs + DIEPCTL0), 3275 dwc2_readl(hsotg->regs + DOEPCTL0)); 3276 3277 /* enable in and out endpoint interrupts */ 3278 dwc2_hsotg_en_gsint(hsotg, GINTSTS_OEPINT | GINTSTS_IEPINT); 3279 3280 /* 3281 * Enable the RXFIFO when in slave mode, as this is how we collect 3282 * the data. In DMA mode, we get events from the FIFO but also 3283 * things we cannot process, so do not use it. 3284 */ 3285 if (!using_dma(hsotg)) 3286 dwc2_hsotg_en_gsint(hsotg, GINTSTS_RXFLVL); 3287 3288 /* Enable interrupts for EP0 in and out */ 3289 dwc2_hsotg_ctrl_epint(hsotg, 0, 0, 1); 3290 dwc2_hsotg_ctrl_epint(hsotg, 0, 1, 1); 3291 3292 if (!is_usb_reset) { 3293 __orr32(hsotg->regs + DCTL, DCTL_PWRONPRGDONE); 3294 udelay(10); /* see openiboot */ 3295 __bic32(hsotg->regs + DCTL, DCTL_PWRONPRGDONE); 3296 } 3297 3298 dev_dbg(hsotg->dev, "DCTL=0x%08x\n", dwc2_readl(hsotg->regs + DCTL)); 3299 3300 /* 3301 * DxEPCTL_USBActEp says RO in manual, but seems to be set by 3302 * writing to the EPCTL register.. 3303 */ 3304 3305 /* set to read 1 8byte packet */ 3306 dwc2_writel(DXEPTSIZ_MC(1) | DXEPTSIZ_PKTCNT(1) | 3307 DXEPTSIZ_XFERSIZE(8), hsotg->regs + DOEPTSIZ0); 3308 3309 dwc2_writel(dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) | 3310 DXEPCTL_CNAK | DXEPCTL_EPENA | 3311 DXEPCTL_USBACTEP, 3312 hsotg->regs + DOEPCTL0); 3313 3314 /* enable, but don't activate EP0in */ 3315 dwc2_writel(dwc2_hsotg_ep0_mps(hsotg->eps_out[0]->ep.maxpacket) | 3316 DXEPCTL_USBACTEP, hsotg->regs + DIEPCTL0); 3317 3318 dwc2_hsotg_enqueue_setup(hsotg); 3319 3320 dev_dbg(hsotg->dev, "EP0: DIEPCTL0=0x%08x, DOEPCTL0=0x%08x\n", 3321 dwc2_readl(hsotg->regs + DIEPCTL0), 3322 dwc2_readl(hsotg->regs + DOEPCTL0)); 3323 3324 /* clear global NAKs */ 3325 val = DCTL_CGOUTNAK | DCTL_CGNPINNAK; 3326 if (!is_usb_reset) 3327 val |= DCTL_SFTDISCON; 3328 __orr32(hsotg->regs + DCTL, val); 3329 3330 /* must be at-least 3ms to allow bus to see disconnect */ 3331 mdelay(3); 3332 3333 hsotg->lx_state = DWC2_L0; 3334 } 3335 3336 static void dwc2_hsotg_core_disconnect(struct dwc2_hsotg *hsotg) 3337 { 3338 /* set the soft-disconnect bit */ 3339 __orr32(hsotg->regs + DCTL, DCTL_SFTDISCON); 3340 } 3341 3342 void dwc2_hsotg_core_connect(struct dwc2_hsotg *hsotg) 3343 { 3344 /* remove the soft-disconnect and let's go */ 3345 __bic32(hsotg->regs + DCTL, DCTL_SFTDISCON); 3346 } 3347 3348 /** 3349 * dwc2_gadget_handle_incomplete_isoc_in - handle incomplete ISO IN Interrupt. 3350 * @hsotg: The device state: 3351 * 3352 * This interrupt indicates one of the following conditions occurred while 3353 * transmitting an ISOC transaction. 3354 * - Corrupted IN Token for ISOC EP. 3355 * - Packet not complete in FIFO. 3356 * 3357 * The following actions will be taken: 3358 * - Determine the EP 3359 * - Disable EP; when 'Endpoint Disabled' interrupt is received Flush FIFO 3360 */ 3361 static void dwc2_gadget_handle_incomplete_isoc_in(struct dwc2_hsotg *hsotg) 3362 { 3363 struct dwc2_hsotg_ep *hs_ep; 3364 u32 epctrl; 3365 u32 idx; 3366 3367 dev_dbg(hsotg->dev, "Incomplete isoc in interrupt received:\n"); 3368 3369 for (idx = 1; idx <= hsotg->num_of_eps; idx++) { 3370 hs_ep = hsotg->eps_in[idx]; 3371 epctrl = dwc2_readl(hsotg->regs + DIEPCTL(idx)); 3372 if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous && 3373 dwc2_gadget_target_frame_elapsed(hs_ep)) { 3374 epctrl |= DXEPCTL_SNAK; 3375 epctrl |= DXEPCTL_EPDIS; 3376 dwc2_writel(epctrl, hsotg->regs + DIEPCTL(idx)); 3377 } 3378 } 3379 3380 /* Clear interrupt */ 3381 dwc2_writel(GINTSTS_INCOMPL_SOIN, hsotg->regs + GINTSTS); 3382 } 3383 3384 /** 3385 * dwc2_gadget_handle_incomplete_isoc_out - handle incomplete ISO OUT Interrupt 3386 * @hsotg: The device state: 3387 * 3388 * This interrupt indicates one of the following conditions occurred while 3389 * transmitting an ISOC transaction. 3390 * - Corrupted OUT Token for ISOC EP. 3391 * - Packet not complete in FIFO. 3392 * 3393 * The following actions will be taken: 3394 * - Determine the EP 3395 * - Set DCTL_SGOUTNAK and unmask GOUTNAKEFF if target frame elapsed. 3396 */ 3397 static void dwc2_gadget_handle_incomplete_isoc_out(struct dwc2_hsotg *hsotg) 3398 { 3399 u32 gintsts; 3400 u32 gintmsk; 3401 u32 epctrl; 3402 struct dwc2_hsotg_ep *hs_ep; 3403 int idx; 3404 3405 dev_dbg(hsotg->dev, "%s: GINTSTS_INCOMPL_SOOUT\n", __func__); 3406 3407 for (idx = 1; idx <= hsotg->num_of_eps; idx++) { 3408 hs_ep = hsotg->eps_out[idx]; 3409 epctrl = dwc2_readl(hsotg->regs + DOEPCTL(idx)); 3410 if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous && 3411 dwc2_gadget_target_frame_elapsed(hs_ep)) { 3412 /* Unmask GOUTNAKEFF interrupt */ 3413 gintmsk = dwc2_readl(hsotg->regs + GINTMSK); 3414 gintmsk |= GINTSTS_GOUTNAKEFF; 3415 dwc2_writel(gintmsk, hsotg->regs + GINTMSK); 3416 3417 gintsts = dwc2_readl(hsotg->regs + GINTSTS); 3418 if (!(gintsts & GINTSTS_GOUTNAKEFF)) 3419 __orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK); 3420 } 3421 } 3422 3423 /* Clear interrupt */ 3424 dwc2_writel(GINTSTS_INCOMPL_SOOUT, hsotg->regs + GINTSTS); 3425 } 3426 3427 /** 3428 * dwc2_hsotg_irq - handle device interrupt 3429 * @irq: The IRQ number triggered 3430 * @pw: The pw value when registered the handler. 3431 */ 3432 static irqreturn_t dwc2_hsotg_irq(int irq, void *pw) 3433 { 3434 struct dwc2_hsotg *hsotg = pw; 3435 int retry_count = 8; 3436 u32 gintsts; 3437 u32 gintmsk; 3438 3439 if (!dwc2_is_device_mode(hsotg)) 3440 return IRQ_NONE; 3441 3442 spin_lock(&hsotg->lock); 3443 irq_retry: 3444 gintsts = dwc2_readl(hsotg->regs + GINTSTS); 3445 gintmsk = dwc2_readl(hsotg->regs + GINTMSK); 3446 3447 dev_dbg(hsotg->dev, "%s: %08x %08x (%08x) retry %d\n", 3448 __func__, gintsts, gintsts & gintmsk, gintmsk, retry_count); 3449 3450 gintsts &= gintmsk; 3451 3452 if (gintsts & GINTSTS_RESETDET) { 3453 dev_dbg(hsotg->dev, "%s: USBRstDet\n", __func__); 3454 3455 dwc2_writel(GINTSTS_RESETDET, hsotg->regs + GINTSTS); 3456 3457 /* This event must be used only if controller is suspended */ 3458 if (hsotg->lx_state == DWC2_L2) { 3459 dwc2_exit_hibernation(hsotg, true); 3460 hsotg->lx_state = DWC2_L0; 3461 } 3462 } 3463 3464 if (gintsts & (GINTSTS_USBRST | GINTSTS_RESETDET)) { 3465 3466 u32 usb_status = dwc2_readl(hsotg->regs + GOTGCTL); 3467 u32 connected = hsotg->connected; 3468 3469 dev_dbg(hsotg->dev, "%s: USBRst\n", __func__); 3470 dev_dbg(hsotg->dev, "GNPTXSTS=%08x\n", 3471 dwc2_readl(hsotg->regs + GNPTXSTS)); 3472 3473 dwc2_writel(GINTSTS_USBRST, hsotg->regs + GINTSTS); 3474 3475 /* Report disconnection if it is not already done. */ 3476 dwc2_hsotg_disconnect(hsotg); 3477 3478 if (usb_status & GOTGCTL_BSESVLD && connected) 3479 dwc2_hsotg_core_init_disconnected(hsotg, true); 3480 } 3481 3482 if (gintsts & GINTSTS_ENUMDONE) { 3483 dwc2_writel(GINTSTS_ENUMDONE, hsotg->regs + GINTSTS); 3484 3485 dwc2_hsotg_irq_enumdone(hsotg); 3486 } 3487 3488 if (gintsts & (GINTSTS_OEPINT | GINTSTS_IEPINT)) { 3489 u32 daint = dwc2_readl(hsotg->regs + DAINT); 3490 u32 daintmsk = dwc2_readl(hsotg->regs + DAINTMSK); 3491 u32 daint_out, daint_in; 3492 int ep; 3493 3494 daint &= daintmsk; 3495 daint_out = daint >> DAINT_OUTEP_SHIFT; 3496 daint_in = daint & ~(daint_out << DAINT_OUTEP_SHIFT); 3497 3498 dev_dbg(hsotg->dev, "%s: daint=%08x\n", __func__, daint); 3499 3500 for (ep = 0; ep < hsotg->num_of_eps && daint_out; 3501 ep++, daint_out >>= 1) { 3502 if (daint_out & 1) 3503 dwc2_hsotg_epint(hsotg, ep, 0); 3504 } 3505 3506 for (ep = 0; ep < hsotg->num_of_eps && daint_in; 3507 ep++, daint_in >>= 1) { 3508 if (daint_in & 1) 3509 dwc2_hsotg_epint(hsotg, ep, 1); 3510 } 3511 } 3512 3513 /* check both FIFOs */ 3514 3515 if (gintsts & GINTSTS_NPTXFEMP) { 3516 dev_dbg(hsotg->dev, "NPTxFEmp\n"); 3517 3518 /* 3519 * Disable the interrupt to stop it happening again 3520 * unless one of these endpoint routines decides that 3521 * it needs re-enabling 3522 */ 3523 3524 dwc2_hsotg_disable_gsint(hsotg, GINTSTS_NPTXFEMP); 3525 dwc2_hsotg_irq_fifoempty(hsotg, false); 3526 } 3527 3528 if (gintsts & GINTSTS_PTXFEMP) { 3529 dev_dbg(hsotg->dev, "PTxFEmp\n"); 3530 3531 /* See note in GINTSTS_NPTxFEmp */ 3532 3533 dwc2_hsotg_disable_gsint(hsotg, GINTSTS_PTXFEMP); 3534 dwc2_hsotg_irq_fifoempty(hsotg, true); 3535 } 3536 3537 if (gintsts & GINTSTS_RXFLVL) { 3538 /* 3539 * note, since GINTSTS_RxFLvl doubles as FIFO-not-empty, 3540 * we need to retry dwc2_hsotg_handle_rx if this is still 3541 * set. 3542 */ 3543 3544 dwc2_hsotg_handle_rx(hsotg); 3545 } 3546 3547 if (gintsts & GINTSTS_ERLYSUSP) { 3548 dev_dbg(hsotg->dev, "GINTSTS_ErlySusp\n"); 3549 dwc2_writel(GINTSTS_ERLYSUSP, hsotg->regs + GINTSTS); 3550 } 3551 3552 /* 3553 * these next two seem to crop-up occasionally causing the core 3554 * to shutdown the USB transfer, so try clearing them and logging 3555 * the occurrence. 3556 */ 3557 3558 if (gintsts & GINTSTS_GOUTNAKEFF) { 3559 u8 idx; 3560 u32 epctrl; 3561 u32 gintmsk; 3562 struct dwc2_hsotg_ep *hs_ep; 3563 3564 /* Mask this interrupt */ 3565 gintmsk = dwc2_readl(hsotg->regs + GINTMSK); 3566 gintmsk &= ~GINTSTS_GOUTNAKEFF; 3567 dwc2_writel(gintmsk, hsotg->regs + GINTMSK); 3568 3569 dev_dbg(hsotg->dev, "GOUTNakEff triggered\n"); 3570 for (idx = 1; idx <= hsotg->num_of_eps; idx++) { 3571 hs_ep = hsotg->eps_out[idx]; 3572 epctrl = dwc2_readl(hsotg->regs + DOEPCTL(idx)); 3573 3574 if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous) { 3575 epctrl |= DXEPCTL_SNAK; 3576 epctrl |= DXEPCTL_EPDIS; 3577 dwc2_writel(epctrl, hsotg->regs + DOEPCTL(idx)); 3578 } 3579 } 3580 3581 /* This interrupt bit is cleared in DXEPINT_EPDISBLD handler */ 3582 } 3583 3584 if (gintsts & GINTSTS_GINNAKEFF) { 3585 dev_info(hsotg->dev, "GINNakEff triggered\n"); 3586 3587 __orr32(hsotg->regs + DCTL, DCTL_CGNPINNAK); 3588 3589 dwc2_hsotg_dump(hsotg); 3590 } 3591 3592 if (gintsts & GINTSTS_INCOMPL_SOIN) 3593 dwc2_gadget_handle_incomplete_isoc_in(hsotg); 3594 3595 if (gintsts & GINTSTS_INCOMPL_SOOUT) 3596 dwc2_gadget_handle_incomplete_isoc_out(hsotg); 3597 3598 /* 3599 * if we've had fifo events, we should try and go around the 3600 * loop again to see if there's any point in returning yet. 3601 */ 3602 3603 if (gintsts & IRQ_RETRY_MASK && --retry_count > 0) 3604 goto irq_retry; 3605 3606 spin_unlock(&hsotg->lock); 3607 3608 return IRQ_HANDLED; 3609 } 3610 3611 static int dwc2_hsotg_wait_bit_set(struct dwc2_hsotg *hs_otg, u32 reg, 3612 u32 bit, u32 timeout) 3613 { 3614 u32 i; 3615 3616 for (i = 0; i < timeout; i++) { 3617 if (dwc2_readl(hs_otg->regs + reg) & bit) 3618 return 0; 3619 udelay(1); 3620 } 3621 3622 return -ETIMEDOUT; 3623 } 3624 3625 static void dwc2_hsotg_ep_stop_xfr(struct dwc2_hsotg *hsotg, 3626 struct dwc2_hsotg_ep *hs_ep) 3627 { 3628 u32 epctrl_reg; 3629 u32 epint_reg; 3630 3631 epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) : 3632 DOEPCTL(hs_ep->index); 3633 epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) : 3634 DOEPINT(hs_ep->index); 3635 3636 dev_dbg(hsotg->dev, "%s: stopping transfer on %s\n", __func__, 3637 hs_ep->name); 3638 3639 if (hs_ep->dir_in) { 3640 if (hsotg->dedicated_fifos || hs_ep->periodic) { 3641 __orr32(hsotg->regs + epctrl_reg, DXEPCTL_SNAK); 3642 /* Wait for Nak effect */ 3643 if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, 3644 DXEPINT_INEPNAKEFF, 100)) 3645 dev_warn(hsotg->dev, 3646 "%s: timeout DIEPINT.NAKEFF\n", 3647 __func__); 3648 } else { 3649 __orr32(hsotg->regs + DCTL, DCTL_SGNPINNAK); 3650 /* Wait for Nak effect */ 3651 if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS, 3652 GINTSTS_GINNAKEFF, 100)) 3653 dev_warn(hsotg->dev, 3654 "%s: timeout GINTSTS.GINNAKEFF\n", 3655 __func__); 3656 } 3657 } else { 3658 if (!(dwc2_readl(hsotg->regs + GINTSTS) & GINTSTS_GOUTNAKEFF)) 3659 __orr32(hsotg->regs + DCTL, DCTL_SGOUTNAK); 3660 3661 /* Wait for global nak to take effect */ 3662 if (dwc2_hsotg_wait_bit_set(hsotg, GINTSTS, 3663 GINTSTS_GOUTNAKEFF, 100)) 3664 dev_warn(hsotg->dev, "%s: timeout GINTSTS.GOUTNAKEFF\n", 3665 __func__); 3666 } 3667 3668 /* Disable ep */ 3669 __orr32(hsotg->regs + epctrl_reg, DXEPCTL_EPDIS | DXEPCTL_SNAK); 3670 3671 /* Wait for ep to be disabled */ 3672 if (dwc2_hsotg_wait_bit_set(hsotg, epint_reg, DXEPINT_EPDISBLD, 100)) 3673 dev_warn(hsotg->dev, 3674 "%s: timeout DOEPCTL.EPDisable\n", __func__); 3675 3676 /* Clear EPDISBLD interrupt */ 3677 __orr32(hsotg->regs + epint_reg, DXEPINT_EPDISBLD); 3678 3679 if (hs_ep->dir_in) { 3680 unsigned short fifo_index; 3681 3682 if (hsotg->dedicated_fifos || hs_ep->periodic) 3683 fifo_index = hs_ep->fifo_index; 3684 else 3685 fifo_index = 0; 3686 3687 /* Flush TX FIFO */ 3688 dwc2_flush_tx_fifo(hsotg, fifo_index); 3689 3690 /* Clear Global In NP NAK in Shared FIFO for non periodic ep */ 3691 if (!hsotg->dedicated_fifos && !hs_ep->periodic) 3692 __orr32(hsotg->regs + DCTL, DCTL_CGNPINNAK); 3693 3694 } else { 3695 /* Remove global NAKs */ 3696 __orr32(hsotg->regs + DCTL, DCTL_CGOUTNAK); 3697 } 3698 } 3699 3700 /** 3701 * dwc2_hsotg_ep_enable - enable the given endpoint 3702 * @ep: The USB endpint to configure 3703 * @desc: The USB endpoint descriptor to configure with. 3704 * 3705 * This is called from the USB gadget code's usb_ep_enable(). 3706 */ 3707 static int dwc2_hsotg_ep_enable(struct usb_ep *ep, 3708 const struct usb_endpoint_descriptor *desc) 3709 { 3710 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); 3711 struct dwc2_hsotg *hsotg = hs_ep->parent; 3712 unsigned long flags; 3713 unsigned int index = hs_ep->index; 3714 u32 epctrl_reg; 3715 u32 epctrl; 3716 u32 mps; 3717 u32 mc; 3718 u32 mask; 3719 unsigned int dir_in; 3720 unsigned int i, val, size; 3721 int ret = 0; 3722 3723 dev_dbg(hsotg->dev, 3724 "%s: ep %s: a 0x%02x, attr 0x%02x, mps 0x%04x, intr %d\n", 3725 __func__, ep->name, desc->bEndpointAddress, desc->bmAttributes, 3726 desc->wMaxPacketSize, desc->bInterval); 3727 3728 /* not to be called for EP0 */ 3729 if (index == 0) { 3730 dev_err(hsotg->dev, "%s: called for EP 0\n", __func__); 3731 return -EINVAL; 3732 } 3733 3734 dir_in = (desc->bEndpointAddress & USB_ENDPOINT_DIR_MASK) ? 1 : 0; 3735 if (dir_in != hs_ep->dir_in) { 3736 dev_err(hsotg->dev, "%s: direction mismatch!\n", __func__); 3737 return -EINVAL; 3738 } 3739 3740 mps = usb_endpoint_maxp(desc); 3741 mc = usb_endpoint_maxp_mult(desc); 3742 3743 /* note, we handle this here instead of dwc2_hsotg_set_ep_maxpacket */ 3744 3745 epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index); 3746 epctrl = dwc2_readl(hsotg->regs + epctrl_reg); 3747 3748 dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x from 0x%08x\n", 3749 __func__, epctrl, epctrl_reg); 3750 3751 /* Allocate DMA descriptor chain for non-ctrl endpoints */ 3752 if (using_desc_dma(hsotg)) { 3753 hs_ep->desc_list = dma_alloc_coherent(hsotg->dev, 3754 MAX_DMA_DESC_NUM_GENERIC * 3755 sizeof(struct dwc2_dma_desc), 3756 &hs_ep->desc_list_dma, GFP_KERNEL); 3757 if (!hs_ep->desc_list) { 3758 ret = -ENOMEM; 3759 goto error2; 3760 } 3761 } 3762 3763 spin_lock_irqsave(&hsotg->lock, flags); 3764 3765 epctrl &= ~(DXEPCTL_EPTYPE_MASK | DXEPCTL_MPS_MASK); 3766 epctrl |= DXEPCTL_MPS(mps); 3767 3768 /* 3769 * mark the endpoint as active, otherwise the core may ignore 3770 * transactions entirely for this endpoint 3771 */ 3772 epctrl |= DXEPCTL_USBACTEP; 3773 3774 /* update the endpoint state */ 3775 dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in); 3776 3777 /* default, set to non-periodic */ 3778 hs_ep->isochronous = 0; 3779 hs_ep->periodic = 0; 3780 hs_ep->halted = 0; 3781 hs_ep->interval = desc->bInterval; 3782 3783 switch (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) { 3784 case USB_ENDPOINT_XFER_ISOC: 3785 epctrl |= DXEPCTL_EPTYPE_ISO; 3786 epctrl |= DXEPCTL_SETEVENFR; 3787 hs_ep->isochronous = 1; 3788 hs_ep->interval = 1 << (desc->bInterval - 1); 3789 hs_ep->target_frame = TARGET_FRAME_INITIAL; 3790 hs_ep->isoc_chain_num = 0; 3791 hs_ep->next_desc = 0; 3792 if (dir_in) { 3793 hs_ep->periodic = 1; 3794 mask = dwc2_readl(hsotg->regs + DIEPMSK); 3795 mask |= DIEPMSK_NAKMSK; 3796 dwc2_writel(mask, hsotg->regs + DIEPMSK); 3797 } else { 3798 mask = dwc2_readl(hsotg->regs + DOEPMSK); 3799 mask |= DOEPMSK_OUTTKNEPDISMSK; 3800 dwc2_writel(mask, hsotg->regs + DOEPMSK); 3801 } 3802 break; 3803 3804 case USB_ENDPOINT_XFER_BULK: 3805 epctrl |= DXEPCTL_EPTYPE_BULK; 3806 break; 3807 3808 case USB_ENDPOINT_XFER_INT: 3809 if (dir_in) 3810 hs_ep->periodic = 1; 3811 3812 if (hsotg->gadget.speed == USB_SPEED_HIGH) 3813 hs_ep->interval = 1 << (desc->bInterval - 1); 3814 3815 epctrl |= DXEPCTL_EPTYPE_INTERRUPT; 3816 break; 3817 3818 case USB_ENDPOINT_XFER_CONTROL: 3819 epctrl |= DXEPCTL_EPTYPE_CONTROL; 3820 break; 3821 } 3822 3823 /* 3824 * if the hardware has dedicated fifos, we must give each IN EP 3825 * a unique tx-fifo even if it is non-periodic. 3826 */ 3827 if (dir_in && hsotg->dedicated_fifos) { 3828 u32 fifo_index = 0; 3829 u32 fifo_size = UINT_MAX; 3830 size = hs_ep->ep.maxpacket*hs_ep->mc; 3831 for (i = 1; i < hsotg->num_of_eps; ++i) { 3832 if (hsotg->fifo_map & (1<<i)) 3833 continue; 3834 val = dwc2_readl(hsotg->regs + DPTXFSIZN(i)); 3835 val = (val >> FIFOSIZE_DEPTH_SHIFT)*4; 3836 if (val < size) 3837 continue; 3838 /* Search for smallest acceptable fifo */ 3839 if (val < fifo_size) { 3840 fifo_size = val; 3841 fifo_index = i; 3842 } 3843 } 3844 if (!fifo_index) { 3845 dev_err(hsotg->dev, 3846 "%s: No suitable fifo found\n", __func__); 3847 ret = -ENOMEM; 3848 goto error1; 3849 } 3850 hsotg->fifo_map |= 1 << fifo_index; 3851 epctrl |= DXEPCTL_TXFNUM(fifo_index); 3852 hs_ep->fifo_index = fifo_index; 3853 hs_ep->fifo_size = fifo_size; 3854 } 3855 3856 /* for non control endpoints, set PID to D0 */ 3857 if (index && !hs_ep->isochronous) 3858 epctrl |= DXEPCTL_SETD0PID; 3859 3860 dev_dbg(hsotg->dev, "%s: write DxEPCTL=0x%08x\n", 3861 __func__, epctrl); 3862 3863 dwc2_writel(epctrl, hsotg->regs + epctrl_reg); 3864 dev_dbg(hsotg->dev, "%s: read DxEPCTL=0x%08x\n", 3865 __func__, dwc2_readl(hsotg->regs + epctrl_reg)); 3866 3867 /* enable the endpoint interrupt */ 3868 dwc2_hsotg_ctrl_epint(hsotg, index, dir_in, 1); 3869 3870 error1: 3871 spin_unlock_irqrestore(&hsotg->lock, flags); 3872 3873 error2: 3874 if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) { 3875 dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC * 3876 sizeof(struct dwc2_dma_desc), 3877 hs_ep->desc_list, hs_ep->desc_list_dma); 3878 hs_ep->desc_list = NULL; 3879 } 3880 3881 return ret; 3882 } 3883 3884 /** 3885 * dwc2_hsotg_ep_disable - disable given endpoint 3886 * @ep: The endpoint to disable. 3887 */ 3888 static int dwc2_hsotg_ep_disable(struct usb_ep *ep) 3889 { 3890 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); 3891 struct dwc2_hsotg *hsotg = hs_ep->parent; 3892 int dir_in = hs_ep->dir_in; 3893 int index = hs_ep->index; 3894 unsigned long flags; 3895 u32 epctrl_reg; 3896 u32 ctrl; 3897 3898 dev_dbg(hsotg->dev, "%s(ep %p)\n", __func__, ep); 3899 3900 if (ep == &hsotg->eps_out[0]->ep) { 3901 dev_err(hsotg->dev, "%s: called for ep0\n", __func__); 3902 return -EINVAL; 3903 } 3904 3905 /* Remove DMA memory allocated for non-control Endpoints */ 3906 if (using_desc_dma(hsotg)) { 3907 dma_free_coherent(hsotg->dev, MAX_DMA_DESC_NUM_GENERIC * 3908 sizeof(struct dwc2_dma_desc), 3909 hs_ep->desc_list, hs_ep->desc_list_dma); 3910 hs_ep->desc_list = NULL; 3911 } 3912 3913 epctrl_reg = dir_in ? DIEPCTL(index) : DOEPCTL(index); 3914 3915 spin_lock_irqsave(&hsotg->lock, flags); 3916 3917 ctrl = dwc2_readl(hsotg->regs + epctrl_reg); 3918 3919 if (ctrl & DXEPCTL_EPENA) 3920 dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep); 3921 3922 ctrl &= ~DXEPCTL_EPENA; 3923 ctrl &= ~DXEPCTL_USBACTEP; 3924 ctrl |= DXEPCTL_SNAK; 3925 3926 dev_dbg(hsotg->dev, "%s: DxEPCTL=0x%08x\n", __func__, ctrl); 3927 dwc2_writel(ctrl, hsotg->regs + epctrl_reg); 3928 3929 /* disable endpoint interrupts */ 3930 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0); 3931 3932 /* terminate all requests with shutdown */ 3933 kill_all_requests(hsotg, hs_ep, -ESHUTDOWN); 3934 3935 hsotg->fifo_map &= ~(1 << hs_ep->fifo_index); 3936 hs_ep->fifo_index = 0; 3937 hs_ep->fifo_size = 0; 3938 3939 spin_unlock_irqrestore(&hsotg->lock, flags); 3940 return 0; 3941 } 3942 3943 /** 3944 * on_list - check request is on the given endpoint 3945 * @ep: The endpoint to check. 3946 * @test: The request to test if it is on the endpoint. 3947 */ 3948 static bool on_list(struct dwc2_hsotg_ep *ep, struct dwc2_hsotg_req *test) 3949 { 3950 struct dwc2_hsotg_req *req, *treq; 3951 3952 list_for_each_entry_safe(req, treq, &ep->queue, queue) { 3953 if (req == test) 3954 return true; 3955 } 3956 3957 return false; 3958 } 3959 3960 /** 3961 * dwc2_hsotg_ep_dequeue - dequeue given endpoint 3962 * @ep: The endpoint to dequeue. 3963 * @req: The request to be removed from a queue. 3964 */ 3965 static int dwc2_hsotg_ep_dequeue(struct usb_ep *ep, struct usb_request *req) 3966 { 3967 struct dwc2_hsotg_req *hs_req = our_req(req); 3968 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); 3969 struct dwc2_hsotg *hs = hs_ep->parent; 3970 unsigned long flags; 3971 3972 dev_dbg(hs->dev, "ep_dequeue(%p,%p)\n", ep, req); 3973 3974 spin_lock_irqsave(&hs->lock, flags); 3975 3976 if (!on_list(hs_ep, hs_req)) { 3977 spin_unlock_irqrestore(&hs->lock, flags); 3978 return -EINVAL; 3979 } 3980 3981 /* Dequeue already started request */ 3982 if (req == &hs_ep->req->req) 3983 dwc2_hsotg_ep_stop_xfr(hs, hs_ep); 3984 3985 dwc2_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET); 3986 spin_unlock_irqrestore(&hs->lock, flags); 3987 3988 return 0; 3989 } 3990 3991 /** 3992 * dwc2_hsotg_ep_sethalt - set halt on a given endpoint 3993 * @ep: The endpoint to set halt. 3994 * @value: Set or unset the halt. 3995 * @now: If true, stall the endpoint now. Otherwise return -EAGAIN if 3996 * the endpoint is busy processing requests. 3997 * 3998 * We need to stall the endpoint immediately if request comes from set_feature 3999 * protocol command handler. 4000 */ 4001 static int dwc2_hsotg_ep_sethalt(struct usb_ep *ep, int value, bool now) 4002 { 4003 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); 4004 struct dwc2_hsotg *hs = hs_ep->parent; 4005 int index = hs_ep->index; 4006 u32 epreg; 4007 u32 epctl; 4008 u32 xfertype; 4009 4010 dev_info(hs->dev, "%s(ep %p %s, %d)\n", __func__, ep, ep->name, value); 4011 4012 if (index == 0) { 4013 if (value) 4014 dwc2_hsotg_stall_ep0(hs); 4015 else 4016 dev_warn(hs->dev, 4017 "%s: can't clear halt on ep0\n", __func__); 4018 return 0; 4019 } 4020 4021 if (hs_ep->isochronous) { 4022 dev_err(hs->dev, "%s is Isochronous Endpoint\n", ep->name); 4023 return -EINVAL; 4024 } 4025 4026 if (!now && value && !list_empty(&hs_ep->queue)) { 4027 dev_dbg(hs->dev, "%s request is pending, cannot halt\n", 4028 ep->name); 4029 return -EAGAIN; 4030 } 4031 4032 if (hs_ep->dir_in) { 4033 epreg = DIEPCTL(index); 4034 epctl = dwc2_readl(hs->regs + epreg); 4035 4036 if (value) { 4037 epctl |= DXEPCTL_STALL | DXEPCTL_SNAK; 4038 if (epctl & DXEPCTL_EPENA) 4039 epctl |= DXEPCTL_EPDIS; 4040 } else { 4041 epctl &= ~DXEPCTL_STALL; 4042 xfertype = epctl & DXEPCTL_EPTYPE_MASK; 4043 if (xfertype == DXEPCTL_EPTYPE_BULK || 4044 xfertype == DXEPCTL_EPTYPE_INTERRUPT) 4045 epctl |= DXEPCTL_SETD0PID; 4046 } 4047 dwc2_writel(epctl, hs->regs + epreg); 4048 } else { 4049 4050 epreg = DOEPCTL(index); 4051 epctl = dwc2_readl(hs->regs + epreg); 4052 4053 if (value) 4054 epctl |= DXEPCTL_STALL; 4055 else { 4056 epctl &= ~DXEPCTL_STALL; 4057 xfertype = epctl & DXEPCTL_EPTYPE_MASK; 4058 if (xfertype == DXEPCTL_EPTYPE_BULK || 4059 xfertype == DXEPCTL_EPTYPE_INTERRUPT) 4060 epctl |= DXEPCTL_SETD0PID; 4061 } 4062 dwc2_writel(epctl, hs->regs + epreg); 4063 } 4064 4065 hs_ep->halted = value; 4066 4067 return 0; 4068 } 4069 4070 /** 4071 * dwc2_hsotg_ep_sethalt_lock - set halt on a given endpoint with lock held 4072 * @ep: The endpoint to set halt. 4073 * @value: Set or unset the halt. 4074 */ 4075 static int dwc2_hsotg_ep_sethalt_lock(struct usb_ep *ep, int value) 4076 { 4077 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); 4078 struct dwc2_hsotg *hs = hs_ep->parent; 4079 unsigned long flags = 0; 4080 int ret = 0; 4081 4082 spin_lock_irqsave(&hs->lock, flags); 4083 ret = dwc2_hsotg_ep_sethalt(ep, value, false); 4084 spin_unlock_irqrestore(&hs->lock, flags); 4085 4086 return ret; 4087 } 4088 4089 static struct usb_ep_ops dwc2_hsotg_ep_ops = { 4090 .enable = dwc2_hsotg_ep_enable, 4091 .disable = dwc2_hsotg_ep_disable, 4092 .alloc_request = dwc2_hsotg_ep_alloc_request, 4093 .free_request = dwc2_hsotg_ep_free_request, 4094 .queue = dwc2_hsotg_ep_queue_lock, 4095 .dequeue = dwc2_hsotg_ep_dequeue, 4096 .set_halt = dwc2_hsotg_ep_sethalt_lock, 4097 /* note, don't believe we have any call for the fifo routines */ 4098 }; 4099 4100 /** 4101 * dwc2_hsotg_init - initalize the usb core 4102 * @hsotg: The driver state 4103 */ 4104 static void dwc2_hsotg_init(struct dwc2_hsotg *hsotg) 4105 { 4106 u32 trdtim; 4107 u32 usbcfg; 4108 /* unmask subset of endpoint interrupts */ 4109 4110 dwc2_writel(DIEPMSK_TIMEOUTMSK | DIEPMSK_AHBERRMSK | 4111 DIEPMSK_EPDISBLDMSK | DIEPMSK_XFERCOMPLMSK, 4112 hsotg->regs + DIEPMSK); 4113 4114 dwc2_writel(DOEPMSK_SETUPMSK | DOEPMSK_AHBERRMSK | 4115 DOEPMSK_EPDISBLDMSK | DOEPMSK_XFERCOMPLMSK, 4116 hsotg->regs + DOEPMSK); 4117 4118 dwc2_writel(0, hsotg->regs + DAINTMSK); 4119 4120 /* Be in disconnected state until gadget is registered */ 4121 __orr32(hsotg->regs + DCTL, DCTL_SFTDISCON); 4122 4123 /* setup fifos */ 4124 4125 dev_dbg(hsotg->dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n", 4126 dwc2_readl(hsotg->regs + GRXFSIZ), 4127 dwc2_readl(hsotg->regs + GNPTXFSIZ)); 4128 4129 dwc2_hsotg_init_fifo(hsotg); 4130 4131 /* keep other bits untouched (so e.g. forced modes are not lost) */ 4132 usbcfg = dwc2_readl(hsotg->regs + GUSBCFG); 4133 usbcfg &= ~(GUSBCFG_TOUTCAL_MASK | GUSBCFG_PHYIF16 | GUSBCFG_SRPCAP | 4134 GUSBCFG_HNPCAP); 4135 4136 /* set the PLL on, remove the HNP/SRP and set the PHY */ 4137 trdtim = (hsotg->phyif == GUSBCFG_PHYIF8) ? 9 : 5; 4138 usbcfg |= hsotg->phyif | GUSBCFG_TOUTCAL(7) | 4139 (trdtim << GUSBCFG_USBTRDTIM_SHIFT); 4140 dwc2_writel(usbcfg, hsotg->regs + GUSBCFG); 4141 4142 if (using_dma(hsotg)) 4143 __orr32(hsotg->regs + GAHBCFG, GAHBCFG_DMA_EN); 4144 } 4145 4146 /** 4147 * dwc2_hsotg_udc_start - prepare the udc for work 4148 * @gadget: The usb gadget state 4149 * @driver: The usb gadget driver 4150 * 4151 * Perform initialization to prepare udc device and driver 4152 * to work. 4153 */ 4154 static int dwc2_hsotg_udc_start(struct usb_gadget *gadget, 4155 struct usb_gadget_driver *driver) 4156 { 4157 struct dwc2_hsotg *hsotg = to_hsotg(gadget); 4158 unsigned long flags; 4159 int ret; 4160 4161 if (!hsotg) { 4162 pr_err("%s: called with no device\n", __func__); 4163 return -ENODEV; 4164 } 4165 4166 if (!driver) { 4167 dev_err(hsotg->dev, "%s: no driver\n", __func__); 4168 return -EINVAL; 4169 } 4170 4171 if (driver->max_speed < USB_SPEED_FULL) 4172 dev_err(hsotg->dev, "%s: bad speed\n", __func__); 4173 4174 if (!driver->setup) { 4175 dev_err(hsotg->dev, "%s: missing entry points\n", __func__); 4176 return -EINVAL; 4177 } 4178 4179 WARN_ON(hsotg->driver); 4180 4181 driver->driver.bus = NULL; 4182 hsotg->driver = driver; 4183 hsotg->gadget.dev.of_node = hsotg->dev->of_node; 4184 hsotg->gadget.speed = USB_SPEED_UNKNOWN; 4185 4186 if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL) { 4187 ret = dwc2_lowlevel_hw_enable(hsotg); 4188 if (ret) 4189 goto err; 4190 } 4191 4192 if (!IS_ERR_OR_NULL(hsotg->uphy)) 4193 otg_set_peripheral(hsotg->uphy->otg, &hsotg->gadget); 4194 4195 spin_lock_irqsave(&hsotg->lock, flags); 4196 if (dwc2_hw_is_device(hsotg)) { 4197 dwc2_hsotg_init(hsotg); 4198 dwc2_hsotg_core_init_disconnected(hsotg, false); 4199 } 4200 4201 hsotg->enabled = 0; 4202 spin_unlock_irqrestore(&hsotg->lock, flags); 4203 4204 dev_info(hsotg->dev, "bound driver %s\n", driver->driver.name); 4205 4206 return 0; 4207 4208 err: 4209 hsotg->driver = NULL; 4210 return ret; 4211 } 4212 4213 /** 4214 * dwc2_hsotg_udc_stop - stop the udc 4215 * @gadget: The usb gadget state 4216 * @driver: The usb gadget driver 4217 * 4218 * Stop udc hw block and stay tunned for future transmissions 4219 */ 4220 static int dwc2_hsotg_udc_stop(struct usb_gadget *gadget) 4221 { 4222 struct dwc2_hsotg *hsotg = to_hsotg(gadget); 4223 unsigned long flags = 0; 4224 int ep; 4225 4226 if (!hsotg) 4227 return -ENODEV; 4228 4229 /* all endpoints should be shutdown */ 4230 for (ep = 1; ep < hsotg->num_of_eps; ep++) { 4231 if (hsotg->eps_in[ep]) 4232 dwc2_hsotg_ep_disable(&hsotg->eps_in[ep]->ep); 4233 if (hsotg->eps_out[ep]) 4234 dwc2_hsotg_ep_disable(&hsotg->eps_out[ep]->ep); 4235 } 4236 4237 spin_lock_irqsave(&hsotg->lock, flags); 4238 4239 hsotg->driver = NULL; 4240 hsotg->gadget.speed = USB_SPEED_UNKNOWN; 4241 hsotg->enabled = 0; 4242 4243 spin_unlock_irqrestore(&hsotg->lock, flags); 4244 4245 if (!IS_ERR_OR_NULL(hsotg->uphy)) 4246 otg_set_peripheral(hsotg->uphy->otg, NULL); 4247 4248 if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL) 4249 dwc2_lowlevel_hw_disable(hsotg); 4250 4251 return 0; 4252 } 4253 4254 /** 4255 * dwc2_hsotg_gadget_getframe - read the frame number 4256 * @gadget: The usb gadget state 4257 * 4258 * Read the {micro} frame number 4259 */ 4260 static int dwc2_hsotg_gadget_getframe(struct usb_gadget *gadget) 4261 { 4262 return dwc2_hsotg_read_frameno(to_hsotg(gadget)); 4263 } 4264 4265 /** 4266 * dwc2_hsotg_pullup - connect/disconnect the USB PHY 4267 * @gadget: The usb gadget state 4268 * @is_on: Current state of the USB PHY 4269 * 4270 * Connect/Disconnect the USB PHY pullup 4271 */ 4272 static int dwc2_hsotg_pullup(struct usb_gadget *gadget, int is_on) 4273 { 4274 struct dwc2_hsotg *hsotg = to_hsotg(gadget); 4275 unsigned long flags = 0; 4276 4277 dev_dbg(hsotg->dev, "%s: is_on: %d op_state: %d\n", __func__, is_on, 4278 hsotg->op_state); 4279 4280 /* Don't modify pullup state while in host mode */ 4281 if (hsotg->op_state != OTG_STATE_B_PERIPHERAL) { 4282 hsotg->enabled = is_on; 4283 return 0; 4284 } 4285 4286 spin_lock_irqsave(&hsotg->lock, flags); 4287 if (is_on) { 4288 hsotg->enabled = 1; 4289 dwc2_hsotg_core_init_disconnected(hsotg, false); 4290 dwc2_hsotg_core_connect(hsotg); 4291 } else { 4292 dwc2_hsotg_core_disconnect(hsotg); 4293 dwc2_hsotg_disconnect(hsotg); 4294 hsotg->enabled = 0; 4295 } 4296 4297 hsotg->gadget.speed = USB_SPEED_UNKNOWN; 4298 spin_unlock_irqrestore(&hsotg->lock, flags); 4299 4300 return 0; 4301 } 4302 4303 static int dwc2_hsotg_vbus_session(struct usb_gadget *gadget, int is_active) 4304 { 4305 struct dwc2_hsotg *hsotg = to_hsotg(gadget); 4306 unsigned long flags; 4307 4308 dev_dbg(hsotg->dev, "%s: is_active: %d\n", __func__, is_active); 4309 spin_lock_irqsave(&hsotg->lock, flags); 4310 4311 /* 4312 * If controller is hibernated, it must exit from hibernation 4313 * before being initialized / de-initialized 4314 */ 4315 if (hsotg->lx_state == DWC2_L2) 4316 dwc2_exit_hibernation(hsotg, false); 4317 4318 if (is_active) { 4319 hsotg->op_state = OTG_STATE_B_PERIPHERAL; 4320 4321 dwc2_hsotg_core_init_disconnected(hsotg, false); 4322 if (hsotg->enabled) 4323 dwc2_hsotg_core_connect(hsotg); 4324 } else { 4325 dwc2_hsotg_core_disconnect(hsotg); 4326 dwc2_hsotg_disconnect(hsotg); 4327 } 4328 4329 spin_unlock_irqrestore(&hsotg->lock, flags); 4330 return 0; 4331 } 4332 4333 /** 4334 * dwc2_hsotg_vbus_draw - report bMaxPower field 4335 * @gadget: The usb gadget state 4336 * @mA: Amount of current 4337 * 4338 * Report how much power the device may consume to the phy. 4339 */ 4340 static int dwc2_hsotg_vbus_draw(struct usb_gadget *gadget, unsigned mA) 4341 { 4342 struct dwc2_hsotg *hsotg = to_hsotg(gadget); 4343 4344 if (IS_ERR_OR_NULL(hsotg->uphy)) 4345 return -ENOTSUPP; 4346 return usb_phy_set_power(hsotg->uphy, mA); 4347 } 4348 4349 static const struct usb_gadget_ops dwc2_hsotg_gadget_ops = { 4350 .get_frame = dwc2_hsotg_gadget_getframe, 4351 .udc_start = dwc2_hsotg_udc_start, 4352 .udc_stop = dwc2_hsotg_udc_stop, 4353 .pullup = dwc2_hsotg_pullup, 4354 .vbus_session = dwc2_hsotg_vbus_session, 4355 .vbus_draw = dwc2_hsotg_vbus_draw, 4356 }; 4357 4358 /** 4359 * dwc2_hsotg_initep - initialise a single endpoint 4360 * @hsotg: The device state. 4361 * @hs_ep: The endpoint to be initialised. 4362 * @epnum: The endpoint number 4363 * 4364 * Initialise the given endpoint (as part of the probe and device state 4365 * creation) to give to the gadget driver. Setup the endpoint name, any 4366 * direction information and other state that may be required. 4367 */ 4368 static void dwc2_hsotg_initep(struct dwc2_hsotg *hsotg, 4369 struct dwc2_hsotg_ep *hs_ep, 4370 int epnum, 4371 bool dir_in) 4372 { 4373 char *dir; 4374 4375 if (epnum == 0) 4376 dir = ""; 4377 else if (dir_in) 4378 dir = "in"; 4379 else 4380 dir = "out"; 4381 4382 hs_ep->dir_in = dir_in; 4383 hs_ep->index = epnum; 4384 4385 snprintf(hs_ep->name, sizeof(hs_ep->name), "ep%d%s", epnum, dir); 4386 4387 INIT_LIST_HEAD(&hs_ep->queue); 4388 INIT_LIST_HEAD(&hs_ep->ep.ep_list); 4389 4390 /* add to the list of endpoints known by the gadget driver */ 4391 if (epnum) 4392 list_add_tail(&hs_ep->ep.ep_list, &hsotg->gadget.ep_list); 4393 4394 hs_ep->parent = hsotg; 4395 hs_ep->ep.name = hs_ep->name; 4396 4397 if (hsotg->params.speed == DWC2_SPEED_PARAM_LOW) 4398 usb_ep_set_maxpacket_limit(&hs_ep->ep, 8); 4399 else 4400 usb_ep_set_maxpacket_limit(&hs_ep->ep, 4401 epnum ? 1024 : EP0_MPS_LIMIT); 4402 hs_ep->ep.ops = &dwc2_hsotg_ep_ops; 4403 4404 if (epnum == 0) { 4405 hs_ep->ep.caps.type_control = true; 4406 } else { 4407 if (hsotg->params.speed != DWC2_SPEED_PARAM_LOW) { 4408 hs_ep->ep.caps.type_iso = true; 4409 hs_ep->ep.caps.type_bulk = true; 4410 } 4411 hs_ep->ep.caps.type_int = true; 4412 } 4413 4414 if (dir_in) 4415 hs_ep->ep.caps.dir_in = true; 4416 else 4417 hs_ep->ep.caps.dir_out = true; 4418 4419 /* 4420 * if we're using dma, we need to set the next-endpoint pointer 4421 * to be something valid. 4422 */ 4423 4424 if (using_dma(hsotg)) { 4425 u32 next = DXEPCTL_NEXTEP((epnum + 1) % 15); 4426 if (dir_in) 4427 dwc2_writel(next, hsotg->regs + DIEPCTL(epnum)); 4428 else 4429 dwc2_writel(next, hsotg->regs + DOEPCTL(epnum)); 4430 } 4431 } 4432 4433 /** 4434 * dwc2_hsotg_hw_cfg - read HW configuration registers 4435 * @param: The device state 4436 * 4437 * Read the USB core HW configuration registers 4438 */ 4439 static int dwc2_hsotg_hw_cfg(struct dwc2_hsotg *hsotg) 4440 { 4441 u32 cfg; 4442 u32 ep_type; 4443 u32 i; 4444 4445 /* check hardware configuration */ 4446 4447 hsotg->num_of_eps = hsotg->hw_params.num_dev_ep; 4448 4449 /* Add ep0 */ 4450 hsotg->num_of_eps++; 4451 4452 hsotg->eps_in[0] = devm_kzalloc(hsotg->dev, sizeof(struct dwc2_hsotg_ep), 4453 GFP_KERNEL); 4454 if (!hsotg->eps_in[0]) 4455 return -ENOMEM; 4456 /* Same dwc2_hsotg_ep is used in both directions for ep0 */ 4457 hsotg->eps_out[0] = hsotg->eps_in[0]; 4458 4459 cfg = hsotg->hw_params.dev_ep_dirs; 4460 for (i = 1, cfg >>= 2; i < hsotg->num_of_eps; i++, cfg >>= 2) { 4461 ep_type = cfg & 3; 4462 /* Direction in or both */ 4463 if (!(ep_type & 2)) { 4464 hsotg->eps_in[i] = devm_kzalloc(hsotg->dev, 4465 sizeof(struct dwc2_hsotg_ep), GFP_KERNEL); 4466 if (!hsotg->eps_in[i]) 4467 return -ENOMEM; 4468 } 4469 /* Direction out or both */ 4470 if (!(ep_type & 1)) { 4471 hsotg->eps_out[i] = devm_kzalloc(hsotg->dev, 4472 sizeof(struct dwc2_hsotg_ep), GFP_KERNEL); 4473 if (!hsotg->eps_out[i]) 4474 return -ENOMEM; 4475 } 4476 } 4477 4478 hsotg->fifo_mem = hsotg->hw_params.total_fifo_size; 4479 hsotg->dedicated_fifos = hsotg->hw_params.en_multiple_tx_fifo; 4480 4481 dev_info(hsotg->dev, "EPs: %d, %s fifos, %d entries in SPRAM\n", 4482 hsotg->num_of_eps, 4483 hsotg->dedicated_fifos ? "dedicated" : "shared", 4484 hsotg->fifo_mem); 4485 return 0; 4486 } 4487 4488 /** 4489 * dwc2_hsotg_dump - dump state of the udc 4490 * @param: The device state 4491 */ 4492 static void dwc2_hsotg_dump(struct dwc2_hsotg *hsotg) 4493 { 4494 #ifdef DEBUG 4495 struct device *dev = hsotg->dev; 4496 void __iomem *regs = hsotg->regs; 4497 u32 val; 4498 int idx; 4499 4500 dev_info(dev, "DCFG=0x%08x, DCTL=0x%08x, DIEPMSK=%08x\n", 4501 dwc2_readl(regs + DCFG), dwc2_readl(regs + DCTL), 4502 dwc2_readl(regs + DIEPMSK)); 4503 4504 dev_info(dev, "GAHBCFG=0x%08x, GHWCFG1=0x%08x\n", 4505 dwc2_readl(regs + GAHBCFG), dwc2_readl(regs + GHWCFG1)); 4506 4507 dev_info(dev, "GRXFSIZ=0x%08x, GNPTXFSIZ=0x%08x\n", 4508 dwc2_readl(regs + GRXFSIZ), dwc2_readl(regs + GNPTXFSIZ)); 4509 4510 /* show periodic fifo settings */ 4511 4512 for (idx = 1; idx < hsotg->num_of_eps; idx++) { 4513 val = dwc2_readl(regs + DPTXFSIZN(idx)); 4514 dev_info(dev, "DPTx[%d] FSize=%d, StAddr=0x%08x\n", idx, 4515 val >> FIFOSIZE_DEPTH_SHIFT, 4516 val & FIFOSIZE_STARTADDR_MASK); 4517 } 4518 4519 for (idx = 0; idx < hsotg->num_of_eps; idx++) { 4520 dev_info(dev, 4521 "ep%d-in: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n", idx, 4522 dwc2_readl(regs + DIEPCTL(idx)), 4523 dwc2_readl(regs + DIEPTSIZ(idx)), 4524 dwc2_readl(regs + DIEPDMA(idx))); 4525 4526 val = dwc2_readl(regs + DOEPCTL(idx)); 4527 dev_info(dev, 4528 "ep%d-out: EPCTL=0x%08x, SIZ=0x%08x, DMA=0x%08x\n", 4529 idx, dwc2_readl(regs + DOEPCTL(idx)), 4530 dwc2_readl(regs + DOEPTSIZ(idx)), 4531 dwc2_readl(regs + DOEPDMA(idx))); 4532 4533 } 4534 4535 dev_info(dev, "DVBUSDIS=0x%08x, DVBUSPULSE=%08x\n", 4536 dwc2_readl(regs + DVBUSDIS), dwc2_readl(regs + DVBUSPULSE)); 4537 #endif 4538 } 4539 4540 /** 4541 * dwc2_gadget_init - init function for gadget 4542 * @dwc2: The data structure for the DWC2 driver. 4543 * @irq: The IRQ number for the controller. 4544 */ 4545 int dwc2_gadget_init(struct dwc2_hsotg *hsotg, int irq) 4546 { 4547 struct device *dev = hsotg->dev; 4548 int epnum; 4549 int ret; 4550 4551 /* Dump fifo information */ 4552 dev_dbg(dev, "NonPeriodic TXFIFO size: %d\n", 4553 hsotg->params.g_np_tx_fifo_size); 4554 dev_dbg(dev, "RXFIFO size: %d\n", hsotg->params.g_rx_fifo_size); 4555 4556 hsotg->gadget.max_speed = USB_SPEED_HIGH; 4557 hsotg->gadget.ops = &dwc2_hsotg_gadget_ops; 4558 hsotg->gadget.name = dev_name(dev); 4559 if (hsotg->dr_mode == USB_DR_MODE_OTG) 4560 hsotg->gadget.is_otg = 1; 4561 else if (hsotg->dr_mode == USB_DR_MODE_PERIPHERAL) 4562 hsotg->op_state = OTG_STATE_B_PERIPHERAL; 4563 4564 ret = dwc2_hsotg_hw_cfg(hsotg); 4565 if (ret) { 4566 dev_err(hsotg->dev, "Hardware configuration failed: %d\n", ret); 4567 return ret; 4568 } 4569 4570 hsotg->ctrl_buff = devm_kzalloc(hsotg->dev, 4571 DWC2_CTRL_BUFF_SIZE, GFP_KERNEL); 4572 if (!hsotg->ctrl_buff) 4573 return -ENOMEM; 4574 4575 hsotg->ep0_buff = devm_kzalloc(hsotg->dev, 4576 DWC2_CTRL_BUFF_SIZE, GFP_KERNEL); 4577 if (!hsotg->ep0_buff) 4578 return -ENOMEM; 4579 4580 if (using_desc_dma(hsotg)) { 4581 ret = dwc2_gadget_alloc_ctrl_desc_chains(hsotg); 4582 if (ret < 0) 4583 return ret; 4584 } 4585 4586 ret = devm_request_irq(hsotg->dev, irq, dwc2_hsotg_irq, IRQF_SHARED, 4587 dev_name(hsotg->dev), hsotg); 4588 if (ret < 0) { 4589 dev_err(dev, "cannot claim IRQ for gadget\n"); 4590 return ret; 4591 } 4592 4593 /* hsotg->num_of_eps holds number of EPs other than ep0 */ 4594 4595 if (hsotg->num_of_eps == 0) { 4596 dev_err(dev, "wrong number of EPs (zero)\n"); 4597 return -EINVAL; 4598 } 4599 4600 /* setup endpoint information */ 4601 4602 INIT_LIST_HEAD(&hsotg->gadget.ep_list); 4603 hsotg->gadget.ep0 = &hsotg->eps_out[0]->ep; 4604 4605 /* allocate EP0 request */ 4606 4607 hsotg->ctrl_req = dwc2_hsotg_ep_alloc_request(&hsotg->eps_out[0]->ep, 4608 GFP_KERNEL); 4609 if (!hsotg->ctrl_req) { 4610 dev_err(dev, "failed to allocate ctrl req\n"); 4611 return -ENOMEM; 4612 } 4613 4614 /* initialise the endpoints now the core has been initialised */ 4615 for (epnum = 0; epnum < hsotg->num_of_eps; epnum++) { 4616 if (hsotg->eps_in[epnum]) 4617 dwc2_hsotg_initep(hsotg, hsotg->eps_in[epnum], 4618 epnum, 1); 4619 if (hsotg->eps_out[epnum]) 4620 dwc2_hsotg_initep(hsotg, hsotg->eps_out[epnum], 4621 epnum, 0); 4622 } 4623 4624 ret = usb_add_gadget_udc(dev, &hsotg->gadget); 4625 if (ret) 4626 return ret; 4627 4628 dwc2_hsotg_dump(hsotg); 4629 4630 return 0; 4631 } 4632 4633 /** 4634 * dwc2_hsotg_remove - remove function for hsotg driver 4635 * @pdev: The platform information for the driver 4636 */ 4637 int dwc2_hsotg_remove(struct dwc2_hsotg *hsotg) 4638 { 4639 usb_del_gadget_udc(&hsotg->gadget); 4640 4641 return 0; 4642 } 4643 4644 int dwc2_hsotg_suspend(struct dwc2_hsotg *hsotg) 4645 { 4646 unsigned long flags; 4647 4648 if (hsotg->lx_state != DWC2_L0) 4649 return 0; 4650 4651 if (hsotg->driver) { 4652 int ep; 4653 4654 dev_info(hsotg->dev, "suspending usb gadget %s\n", 4655 hsotg->driver->driver.name); 4656 4657 spin_lock_irqsave(&hsotg->lock, flags); 4658 if (hsotg->enabled) 4659 dwc2_hsotg_core_disconnect(hsotg); 4660 dwc2_hsotg_disconnect(hsotg); 4661 hsotg->gadget.speed = USB_SPEED_UNKNOWN; 4662 spin_unlock_irqrestore(&hsotg->lock, flags); 4663 4664 for (ep = 0; ep < hsotg->num_of_eps; ep++) { 4665 if (hsotg->eps_in[ep]) 4666 dwc2_hsotg_ep_disable(&hsotg->eps_in[ep]->ep); 4667 if (hsotg->eps_out[ep]) 4668 dwc2_hsotg_ep_disable(&hsotg->eps_out[ep]->ep); 4669 } 4670 } 4671 4672 return 0; 4673 } 4674 4675 int dwc2_hsotg_resume(struct dwc2_hsotg *hsotg) 4676 { 4677 unsigned long flags; 4678 4679 if (hsotg->lx_state == DWC2_L2) 4680 return 0; 4681 4682 if (hsotg->driver) { 4683 dev_info(hsotg->dev, "resuming usb gadget %s\n", 4684 hsotg->driver->driver.name); 4685 4686 spin_lock_irqsave(&hsotg->lock, flags); 4687 dwc2_hsotg_core_init_disconnected(hsotg, false); 4688 if (hsotg->enabled) 4689 dwc2_hsotg_core_connect(hsotg); 4690 spin_unlock_irqrestore(&hsotg->lock, flags); 4691 } 4692 4693 return 0; 4694 } 4695 4696 /** 4697 * dwc2_backup_device_registers() - Backup controller device registers. 4698 * When suspending usb bus, registers needs to be backuped 4699 * if controller power is disabled once suspended. 4700 * 4701 * @hsotg: Programming view of the DWC_otg controller 4702 */ 4703 int dwc2_backup_device_registers(struct dwc2_hsotg *hsotg) 4704 { 4705 struct dwc2_dregs_backup *dr; 4706 int i; 4707 4708 dev_dbg(hsotg->dev, "%s\n", __func__); 4709 4710 /* Backup dev regs */ 4711 dr = &hsotg->dr_backup; 4712 4713 dr->dcfg = dwc2_readl(hsotg->regs + DCFG); 4714 dr->dctl = dwc2_readl(hsotg->regs + DCTL); 4715 dr->daintmsk = dwc2_readl(hsotg->regs + DAINTMSK); 4716 dr->diepmsk = dwc2_readl(hsotg->regs + DIEPMSK); 4717 dr->doepmsk = dwc2_readl(hsotg->regs + DOEPMSK); 4718 4719 for (i = 0; i < hsotg->num_of_eps; i++) { 4720 /* Backup IN EPs */ 4721 dr->diepctl[i] = dwc2_readl(hsotg->regs + DIEPCTL(i)); 4722 4723 /* Ensure DATA PID is correctly configured */ 4724 if (dr->diepctl[i] & DXEPCTL_DPID) 4725 dr->diepctl[i] |= DXEPCTL_SETD1PID; 4726 else 4727 dr->diepctl[i] |= DXEPCTL_SETD0PID; 4728 4729 dr->dieptsiz[i] = dwc2_readl(hsotg->regs + DIEPTSIZ(i)); 4730 dr->diepdma[i] = dwc2_readl(hsotg->regs + DIEPDMA(i)); 4731 4732 /* Backup OUT EPs */ 4733 dr->doepctl[i] = dwc2_readl(hsotg->regs + DOEPCTL(i)); 4734 4735 /* Ensure DATA PID is correctly configured */ 4736 if (dr->doepctl[i] & DXEPCTL_DPID) 4737 dr->doepctl[i] |= DXEPCTL_SETD1PID; 4738 else 4739 dr->doepctl[i] |= DXEPCTL_SETD0PID; 4740 4741 dr->doeptsiz[i] = dwc2_readl(hsotg->regs + DOEPTSIZ(i)); 4742 dr->doepdma[i] = dwc2_readl(hsotg->regs + DOEPDMA(i)); 4743 } 4744 dr->valid = true; 4745 return 0; 4746 } 4747 4748 /** 4749 * dwc2_restore_device_registers() - Restore controller device registers. 4750 * When resuming usb bus, device registers needs to be restored 4751 * if controller power were disabled. 4752 * 4753 * @hsotg: Programming view of the DWC_otg controller 4754 */ 4755 int dwc2_restore_device_registers(struct dwc2_hsotg *hsotg) 4756 { 4757 struct dwc2_dregs_backup *dr; 4758 u32 dctl; 4759 int i; 4760 4761 dev_dbg(hsotg->dev, "%s\n", __func__); 4762 4763 /* Restore dev regs */ 4764 dr = &hsotg->dr_backup; 4765 if (!dr->valid) { 4766 dev_err(hsotg->dev, "%s: no device registers to restore\n", 4767 __func__); 4768 return -EINVAL; 4769 } 4770 dr->valid = false; 4771 4772 dwc2_writel(dr->dcfg, hsotg->regs + DCFG); 4773 dwc2_writel(dr->dctl, hsotg->regs + DCTL); 4774 dwc2_writel(dr->daintmsk, hsotg->regs + DAINTMSK); 4775 dwc2_writel(dr->diepmsk, hsotg->regs + DIEPMSK); 4776 dwc2_writel(dr->doepmsk, hsotg->regs + DOEPMSK); 4777 4778 for (i = 0; i < hsotg->num_of_eps; i++) { 4779 /* Restore IN EPs */ 4780 dwc2_writel(dr->diepctl[i], hsotg->regs + DIEPCTL(i)); 4781 dwc2_writel(dr->dieptsiz[i], hsotg->regs + DIEPTSIZ(i)); 4782 dwc2_writel(dr->diepdma[i], hsotg->regs + DIEPDMA(i)); 4783 4784 /* Restore OUT EPs */ 4785 dwc2_writel(dr->doepctl[i], hsotg->regs + DOEPCTL(i)); 4786 dwc2_writel(dr->doeptsiz[i], hsotg->regs + DOEPTSIZ(i)); 4787 dwc2_writel(dr->doepdma[i], hsotg->regs + DOEPDMA(i)); 4788 } 4789 4790 /* Set the Power-On Programming done bit */ 4791 dctl = dwc2_readl(hsotg->regs + DCTL); 4792 dctl |= DCTL_PWRONPRGDONE; 4793 dwc2_writel(dctl, hsotg->regs + DCTL); 4794 4795 return 0; 4796 } 4797