1 /* 2 * Copyright (c) 2001-2004 by David Brownell 3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers 4 * 5 * This program is free software; you can redistribute it and/or modify it 6 * under the terms of the GNU General Public License as published by the 7 * Free Software Foundation; either version 2 of the License, or (at your 8 * option) any later version. 9 * 10 * This program is distributed in the hope that it will be useful, but 11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY 12 * or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License 13 * for more details. 14 * 15 * You should have received a copy of the GNU General Public License 16 * along with this program; if not, write to the Free Software Foundation, 17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA. 18 */ 19 20 /* this file is part of ehci-hcd.c */ 21 22 /*-------------------------------------------------------------------------*/ 23 24 /* 25 * EHCI scheduled transaction support: interrupt, iso, split iso 26 * These are called "periodic" transactions in the EHCI spec. 27 * 28 * Note that for interrupt transfers, the QH/QTD manipulation is shared 29 * with the "asynchronous" transaction support (control/bulk transfers). 30 * The only real difference is in how interrupt transfers are scheduled. 31 * 32 * For ISO, we make an "iso_stream" head to serve the same role as a QH. 33 * It keeps track of every ITD (or SITD) that's linked, and holds enough 34 * pre-calculated schedule data to make appending to the queue be quick. 35 */ 36 37 static int ehci_get_frame(struct usb_hcd *hcd); 38 39 /* 40 * periodic_next_shadow - return "next" pointer on shadow list 41 * @periodic: host pointer to qh/itd/sitd 42 * @tag: hardware tag for type of this record 43 */ 44 static union ehci_shadow * 45 periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic, 46 __hc32 tag) 47 { 48 switch (hc32_to_cpu(ehci, tag)) { 49 case Q_TYPE_QH: 50 return &periodic->qh->qh_next; 51 case Q_TYPE_FSTN: 52 return &periodic->fstn->fstn_next; 53 case Q_TYPE_ITD: 54 return &periodic->itd->itd_next; 55 /* case Q_TYPE_SITD: */ 56 default: 57 return &periodic->sitd->sitd_next; 58 } 59 } 60 61 static __hc32 * 62 shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic, 63 __hc32 tag) 64 { 65 switch (hc32_to_cpu(ehci, tag)) { 66 /* our ehci_shadow.qh is actually software part */ 67 case Q_TYPE_QH: 68 return &periodic->qh->hw->hw_next; 69 /* others are hw parts */ 70 default: 71 return periodic->hw_next; 72 } 73 } 74 75 /* caller must hold ehci->lock */ 76 static void periodic_unlink(struct ehci_hcd *ehci, unsigned frame, void *ptr) 77 { 78 union ehci_shadow *prev_p = &ehci->pshadow[frame]; 79 __hc32 *hw_p = &ehci->periodic[frame]; 80 union ehci_shadow here = *prev_p; 81 82 /* find predecessor of "ptr"; hw and shadow lists are in sync */ 83 while (here.ptr && here.ptr != ptr) { 84 prev_p = periodic_next_shadow(ehci, prev_p, 85 Q_NEXT_TYPE(ehci, *hw_p)); 86 hw_p = shadow_next_periodic(ehci, &here, 87 Q_NEXT_TYPE(ehci, *hw_p)); 88 here = *prev_p; 89 } 90 /* an interrupt entry (at list end) could have been shared */ 91 if (!here.ptr) 92 return; 93 94 /* update shadow and hardware lists ... the old "next" pointers 95 * from ptr may still be in use, the caller updates them. 96 */ 97 *prev_p = *periodic_next_shadow(ehci, &here, 98 Q_NEXT_TYPE(ehci, *hw_p)); 99 100 if (!ehci->use_dummy_qh || 101 *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p)) 102 != EHCI_LIST_END(ehci)) 103 *hw_p = *shadow_next_periodic(ehci, &here, 104 Q_NEXT_TYPE(ehci, *hw_p)); 105 else 106 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma); 107 } 108 109 /*-------------------------------------------------------------------------*/ 110 111 /* Bandwidth and TT management */ 112 113 /* Find the TT data structure for this device; create it if necessary */ 114 static struct ehci_tt *find_tt(struct usb_device *udev) 115 { 116 struct usb_tt *utt = udev->tt; 117 struct ehci_tt *tt, **tt_index, **ptt; 118 unsigned port; 119 bool allocated_index = false; 120 121 if (!utt) 122 return NULL; /* Not below a TT */ 123 124 /* 125 * Find/create our data structure. 126 * For hubs with a single TT, we get it directly. 127 * For hubs with multiple TTs, there's an extra level of pointers. 128 */ 129 tt_index = NULL; 130 if (utt->multi) { 131 tt_index = utt->hcpriv; 132 if (!tt_index) { /* Create the index array */ 133 tt_index = kzalloc(utt->hub->maxchild * 134 sizeof(*tt_index), GFP_ATOMIC); 135 if (!tt_index) 136 return ERR_PTR(-ENOMEM); 137 utt->hcpriv = tt_index; 138 allocated_index = true; 139 } 140 port = udev->ttport - 1; 141 ptt = &tt_index[port]; 142 } else { 143 port = 0; 144 ptt = (struct ehci_tt **) &utt->hcpriv; 145 } 146 147 tt = *ptt; 148 if (!tt) { /* Create the ehci_tt */ 149 struct ehci_hcd *ehci = 150 hcd_to_ehci(bus_to_hcd(udev->bus)); 151 152 tt = kzalloc(sizeof(*tt), GFP_ATOMIC); 153 if (!tt) { 154 if (allocated_index) { 155 utt->hcpriv = NULL; 156 kfree(tt_index); 157 } 158 return ERR_PTR(-ENOMEM); 159 } 160 list_add_tail(&tt->tt_list, &ehci->tt_list); 161 INIT_LIST_HEAD(&tt->ps_list); 162 tt->usb_tt = utt; 163 tt->tt_port = port; 164 *ptt = tt; 165 } 166 167 return tt; 168 } 169 170 /* Release the TT above udev, if it's not in use */ 171 static void drop_tt(struct usb_device *udev) 172 { 173 struct usb_tt *utt = udev->tt; 174 struct ehci_tt *tt, **tt_index, **ptt; 175 int cnt, i; 176 177 if (!utt || !utt->hcpriv) 178 return; /* Not below a TT, or never allocated */ 179 180 cnt = 0; 181 if (utt->multi) { 182 tt_index = utt->hcpriv; 183 ptt = &tt_index[udev->ttport - 1]; 184 185 /* How many entries are left in tt_index? */ 186 for (i = 0; i < utt->hub->maxchild; ++i) 187 cnt += !!tt_index[i]; 188 } else { 189 tt_index = NULL; 190 ptt = (struct ehci_tt **) &utt->hcpriv; 191 } 192 193 tt = *ptt; 194 if (!tt || !list_empty(&tt->ps_list)) 195 return; /* never allocated, or still in use */ 196 197 list_del(&tt->tt_list); 198 *ptt = NULL; 199 kfree(tt); 200 if (cnt == 1) { 201 utt->hcpriv = NULL; 202 kfree(tt_index); 203 } 204 } 205 206 static void bandwidth_dbg(struct ehci_hcd *ehci, int sign, char *type, 207 struct ehci_per_sched *ps) 208 { 209 dev_dbg(&ps->udev->dev, 210 "ep %02x: %s %s @ %u+%u (%u.%u+%u) [%u/%u us] mask %04x\n", 211 ps->ep->desc.bEndpointAddress, 212 (sign >= 0 ? "reserve" : "release"), type, 213 (ps->bw_phase << 3) + ps->phase_uf, ps->bw_uperiod, 214 ps->phase, ps->phase_uf, ps->period, 215 ps->usecs, ps->c_usecs, ps->cs_mask); 216 } 217 218 static void reserve_release_intr_bandwidth(struct ehci_hcd *ehci, 219 struct ehci_qh *qh, int sign) 220 { 221 unsigned start_uf; 222 unsigned i, j, m; 223 int usecs = qh->ps.usecs; 224 int c_usecs = qh->ps.c_usecs; 225 int tt_usecs = qh->ps.tt_usecs; 226 struct ehci_tt *tt; 227 228 if (qh->ps.phase == NO_FRAME) /* Bandwidth wasn't reserved */ 229 return; 230 start_uf = qh->ps.bw_phase << 3; 231 232 bandwidth_dbg(ehci, sign, "intr", &qh->ps); 233 234 if (sign < 0) { /* Release bandwidth */ 235 usecs = -usecs; 236 c_usecs = -c_usecs; 237 tt_usecs = -tt_usecs; 238 } 239 240 /* Entire transaction (high speed) or start-split (full/low speed) */ 241 for (i = start_uf + qh->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE; 242 i += qh->ps.bw_uperiod) 243 ehci->bandwidth[i] += usecs; 244 245 /* Complete-split (full/low speed) */ 246 if (qh->ps.c_usecs) { 247 /* NOTE: adjustments needed for FSTN */ 248 for (i = start_uf; i < EHCI_BANDWIDTH_SIZE; 249 i += qh->ps.bw_uperiod) { 250 for ((j = 2, m = 1 << (j+8)); j < 8; (++j, m <<= 1)) { 251 if (qh->ps.cs_mask & m) 252 ehci->bandwidth[i+j] += c_usecs; 253 } 254 } 255 } 256 257 /* FS/LS bus bandwidth */ 258 if (tt_usecs) { 259 tt = find_tt(qh->ps.udev); 260 if (sign > 0) 261 list_add_tail(&qh->ps.ps_list, &tt->ps_list); 262 else 263 list_del(&qh->ps.ps_list); 264 265 for (i = start_uf >> 3; i < EHCI_BANDWIDTH_FRAMES; 266 i += qh->ps.bw_period) 267 tt->bandwidth[i] += tt_usecs; 268 } 269 } 270 271 /*-------------------------------------------------------------------------*/ 272 273 static void compute_tt_budget(u8 budget_table[EHCI_BANDWIDTH_SIZE], 274 struct ehci_tt *tt) 275 { 276 struct ehci_per_sched *ps; 277 unsigned uframe, uf, x; 278 u8 *budget_line; 279 280 if (!tt) 281 return; 282 memset(budget_table, 0, EHCI_BANDWIDTH_SIZE); 283 284 /* Add up the contributions from all the endpoints using this TT */ 285 list_for_each_entry(ps, &tt->ps_list, ps_list) { 286 for (uframe = ps->bw_phase << 3; uframe < EHCI_BANDWIDTH_SIZE; 287 uframe += ps->bw_uperiod) { 288 budget_line = &budget_table[uframe]; 289 x = ps->tt_usecs; 290 291 /* propagate the time forward */ 292 for (uf = ps->phase_uf; uf < 8; ++uf) { 293 x += budget_line[uf]; 294 295 /* Each microframe lasts 125 us */ 296 if (x <= 125) { 297 budget_line[uf] = x; 298 break; 299 } 300 budget_line[uf] = 125; 301 x -= 125; 302 } 303 } 304 } 305 } 306 307 static int __maybe_unused same_tt(struct usb_device *dev1, 308 struct usb_device *dev2) 309 { 310 if (!dev1->tt || !dev2->tt) 311 return 0; 312 if (dev1->tt != dev2->tt) 313 return 0; 314 if (dev1->tt->multi) 315 return dev1->ttport == dev2->ttport; 316 else 317 return 1; 318 } 319 320 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED 321 322 /* Which uframe does the low/fullspeed transfer start in? 323 * 324 * The parameter is the mask of ssplits in "H-frame" terms 325 * and this returns the transfer start uframe in "B-frame" terms, 326 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0 327 * will cause a transfer in "B-frame" uframe 0. "B-frames" lag 328 * "H-frames" by 1 uframe. See the EHCI spec sec 4.5 and figure 4.7. 329 */ 330 static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask) 331 { 332 unsigned char smask = hc32_to_cpu(ehci, mask) & QH_SMASK; 333 334 if (!smask) { 335 ehci_err(ehci, "invalid empty smask!\n"); 336 /* uframe 7 can't have bw so this will indicate failure */ 337 return 7; 338 } 339 return ffs(smask) - 1; 340 } 341 342 static const unsigned char 343 max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 }; 344 345 /* carryover low/fullspeed bandwidth that crosses uframe boundries */ 346 static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8]) 347 { 348 int i; 349 350 for (i = 0; i < 7; i++) { 351 if (max_tt_usecs[i] < tt_usecs[i]) { 352 tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i]; 353 tt_usecs[i] = max_tt_usecs[i]; 354 } 355 } 356 } 357 358 /* 359 * Return true if the device's tt's downstream bus is available for a 360 * periodic transfer of the specified length (usecs), starting at the 361 * specified frame/uframe. Note that (as summarized in section 11.19 362 * of the usb 2.0 spec) TTs can buffer multiple transactions for each 363 * uframe. 364 * 365 * The uframe parameter is when the fullspeed/lowspeed transfer 366 * should be executed in "B-frame" terms, which is the same as the 367 * highspeed ssplit's uframe (which is in "H-frame" terms). For example 368 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0. 369 * See the EHCI spec sec 4.5 and fig 4.7. 370 * 371 * This checks if the full/lowspeed bus, at the specified starting uframe, 372 * has the specified bandwidth available, according to rules listed 373 * in USB 2.0 spec section 11.18.1 fig 11-60. 374 * 375 * This does not check if the transfer would exceed the max ssplit 376 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4, 377 * since proper scheduling limits ssplits to less than 16 per uframe. 378 */ 379 static int tt_available( 380 struct ehci_hcd *ehci, 381 struct ehci_per_sched *ps, 382 struct ehci_tt *tt, 383 unsigned frame, 384 unsigned uframe 385 ) 386 { 387 unsigned period = ps->bw_period; 388 unsigned usecs = ps->tt_usecs; 389 390 if ((period == 0) || (uframe >= 7)) /* error */ 391 return 0; 392 393 for (frame &= period - 1; frame < EHCI_BANDWIDTH_FRAMES; 394 frame += period) { 395 unsigned i, uf; 396 unsigned short tt_usecs[8]; 397 398 if (tt->bandwidth[frame] + usecs > 900) 399 return 0; 400 401 uf = frame << 3; 402 for (i = 0; i < 8; (++i, ++uf)) 403 tt_usecs[i] = ehci->tt_budget[uf]; 404 405 if (max_tt_usecs[uframe] <= tt_usecs[uframe]) 406 return 0; 407 408 /* special case for isoc transfers larger than 125us: 409 * the first and each subsequent fully used uframe 410 * must be empty, so as to not illegally delay 411 * already scheduled transactions 412 */ 413 if (usecs > 125) { 414 int ufs = (usecs / 125); 415 416 for (i = uframe; i < (uframe + ufs) && i < 8; i++) 417 if (tt_usecs[i] > 0) 418 return 0; 419 } 420 421 tt_usecs[uframe] += usecs; 422 423 carryover_tt_bandwidth(tt_usecs); 424 425 /* fail if the carryover pushed bw past the last uframe's limit */ 426 if (max_tt_usecs[7] < tt_usecs[7]) 427 return 0; 428 } 429 430 return 1; 431 } 432 433 #else 434 435 /* return true iff the device's transaction translator is available 436 * for a periodic transfer starting at the specified frame, using 437 * all the uframes in the mask. 438 */ 439 static int tt_no_collision( 440 struct ehci_hcd *ehci, 441 unsigned period, 442 struct usb_device *dev, 443 unsigned frame, 444 u32 uf_mask 445 ) 446 { 447 if (period == 0) /* error */ 448 return 0; 449 450 /* note bandwidth wastage: split never follows csplit 451 * (different dev or endpoint) until the next uframe. 452 * calling convention doesn't make that distinction. 453 */ 454 for (; frame < ehci->periodic_size; frame += period) { 455 union ehci_shadow here; 456 __hc32 type; 457 struct ehci_qh_hw *hw; 458 459 here = ehci->pshadow[frame]; 460 type = Q_NEXT_TYPE(ehci, ehci->periodic[frame]); 461 while (here.ptr) { 462 switch (hc32_to_cpu(ehci, type)) { 463 case Q_TYPE_ITD: 464 type = Q_NEXT_TYPE(ehci, here.itd->hw_next); 465 here = here.itd->itd_next; 466 continue; 467 case Q_TYPE_QH: 468 hw = here.qh->hw; 469 if (same_tt(dev, here.qh->ps.udev)) { 470 u32 mask; 471 472 mask = hc32_to_cpu(ehci, 473 hw->hw_info2); 474 /* "knows" no gap is needed */ 475 mask |= mask >> 8; 476 if (mask & uf_mask) 477 break; 478 } 479 type = Q_NEXT_TYPE(ehci, hw->hw_next); 480 here = here.qh->qh_next; 481 continue; 482 case Q_TYPE_SITD: 483 if (same_tt(dev, here.sitd->urb->dev)) { 484 u16 mask; 485 486 mask = hc32_to_cpu(ehci, here.sitd 487 ->hw_uframe); 488 /* FIXME assumes no gap for IN! */ 489 mask |= mask >> 8; 490 if (mask & uf_mask) 491 break; 492 } 493 type = Q_NEXT_TYPE(ehci, here.sitd->hw_next); 494 here = here.sitd->sitd_next; 495 continue; 496 /* case Q_TYPE_FSTN: */ 497 default: 498 ehci_dbg(ehci, 499 "periodic frame %d bogus type %d\n", 500 frame, type); 501 } 502 503 /* collision or error */ 504 return 0; 505 } 506 } 507 508 /* no collision */ 509 return 1; 510 } 511 512 #endif /* CONFIG_USB_EHCI_TT_NEWSCHED */ 513 514 /*-------------------------------------------------------------------------*/ 515 516 static void enable_periodic(struct ehci_hcd *ehci) 517 { 518 if (ehci->periodic_count++) 519 return; 520 521 /* Stop waiting to turn off the periodic schedule */ 522 ehci->enabled_hrtimer_events &= ~BIT(EHCI_HRTIMER_DISABLE_PERIODIC); 523 524 /* Don't start the schedule until PSS is 0 */ 525 ehci_poll_PSS(ehci); 526 turn_on_io_watchdog(ehci); 527 } 528 529 static void disable_periodic(struct ehci_hcd *ehci) 530 { 531 if (--ehci->periodic_count) 532 return; 533 534 /* Don't turn off the schedule until PSS is 1 */ 535 ehci_poll_PSS(ehci); 536 } 537 538 /*-------------------------------------------------------------------------*/ 539 540 /* periodic schedule slots have iso tds (normal or split) first, then a 541 * sparse tree for active interrupt transfers. 542 * 543 * this just links in a qh; caller guarantees uframe masks are set right. 544 * no FSTN support (yet; ehci 0.96+) 545 */ 546 static void qh_link_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh) 547 { 548 unsigned i; 549 unsigned period = qh->ps.period; 550 551 dev_dbg(&qh->ps.udev->dev, 552 "link qh%d-%04x/%p start %d [%d/%d us]\n", 553 period, hc32_to_cpup(ehci, &qh->hw->hw_info2) 554 & (QH_CMASK | QH_SMASK), 555 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs); 556 557 /* high bandwidth, or otherwise every microframe */ 558 if (period == 0) 559 period = 1; 560 561 for (i = qh->ps.phase; i < ehci->periodic_size; i += period) { 562 union ehci_shadow *prev = &ehci->pshadow[i]; 563 __hc32 *hw_p = &ehci->periodic[i]; 564 union ehci_shadow here = *prev; 565 __hc32 type = 0; 566 567 /* skip the iso nodes at list head */ 568 while (here.ptr) { 569 type = Q_NEXT_TYPE(ehci, *hw_p); 570 if (type == cpu_to_hc32(ehci, Q_TYPE_QH)) 571 break; 572 prev = periodic_next_shadow(ehci, prev, type); 573 hw_p = shadow_next_periodic(ehci, &here, type); 574 here = *prev; 575 } 576 577 /* sorting each branch by period (slow-->fast) 578 * enables sharing interior tree nodes 579 */ 580 while (here.ptr && qh != here.qh) { 581 if (qh->ps.period > here.qh->ps.period) 582 break; 583 prev = &here.qh->qh_next; 584 hw_p = &here.qh->hw->hw_next; 585 here = *prev; 586 } 587 /* link in this qh, unless some earlier pass did that */ 588 if (qh != here.qh) { 589 qh->qh_next = here; 590 if (here.qh) 591 qh->hw->hw_next = *hw_p; 592 wmb(); 593 prev->qh = qh; 594 *hw_p = QH_NEXT(ehci, qh->qh_dma); 595 } 596 } 597 qh->qh_state = QH_STATE_LINKED; 598 qh->xacterrs = 0; 599 qh->unlink_reason = 0; 600 601 /* update per-qh bandwidth for debugfs */ 602 ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->ps.bw_period 603 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period) 604 : (qh->ps.usecs * 8); 605 606 list_add(&qh->intr_node, &ehci->intr_qh_list); 607 608 /* maybe enable periodic schedule processing */ 609 ++ehci->intr_count; 610 enable_periodic(ehci); 611 } 612 613 static void qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh) 614 { 615 unsigned i; 616 unsigned period; 617 618 /* 619 * If qh is for a low/full-speed device, simply unlinking it 620 * could interfere with an ongoing split transaction. To unlink 621 * it safely would require setting the QH_INACTIVATE bit and 622 * waiting at least one frame, as described in EHCI 4.12.2.5. 623 * 624 * We won't bother with any of this. Instead, we assume that the 625 * only reason for unlinking an interrupt QH while the current URB 626 * is still active is to dequeue all the URBs (flush the whole 627 * endpoint queue). 628 * 629 * If rebalancing the periodic schedule is ever implemented, this 630 * approach will no longer be valid. 631 */ 632 633 /* high bandwidth, or otherwise part of every microframe */ 634 period = qh->ps.period ? : 1; 635 636 for (i = qh->ps.phase; i < ehci->periodic_size; i += period) 637 periodic_unlink(ehci, i, qh); 638 639 /* update per-qh bandwidth for debugfs */ 640 ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->ps.bw_period 641 ? ((qh->ps.usecs + qh->ps.c_usecs) / qh->ps.bw_period) 642 : (qh->ps.usecs * 8); 643 644 dev_dbg(&qh->ps.udev->dev, 645 "unlink qh%d-%04x/%p start %d [%d/%d us]\n", 646 qh->ps.period, 647 hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK), 648 qh, qh->ps.phase, qh->ps.usecs, qh->ps.c_usecs); 649 650 /* qh->qh_next still "live" to HC */ 651 qh->qh_state = QH_STATE_UNLINK; 652 qh->qh_next.ptr = NULL; 653 654 if (ehci->qh_scan_next == qh) 655 ehci->qh_scan_next = list_entry(qh->intr_node.next, 656 struct ehci_qh, intr_node); 657 list_del(&qh->intr_node); 658 } 659 660 static void cancel_unlink_wait_intr(struct ehci_hcd *ehci, struct ehci_qh *qh) 661 { 662 if (qh->qh_state != QH_STATE_LINKED || 663 list_empty(&qh->unlink_node)) 664 return; 665 666 list_del_init(&qh->unlink_node); 667 668 /* 669 * TODO: disable the event of EHCI_HRTIMER_START_UNLINK_INTR for 670 * avoiding unnecessary CPU wakeup 671 */ 672 } 673 674 static void start_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh) 675 { 676 /* If the QH isn't linked then there's nothing we can do. */ 677 if (qh->qh_state != QH_STATE_LINKED) 678 return; 679 680 /* if the qh is waiting for unlink, cancel it now */ 681 cancel_unlink_wait_intr(ehci, qh); 682 683 qh_unlink_periodic(ehci, qh); 684 685 /* Make sure the unlinks are visible before starting the timer */ 686 wmb(); 687 688 /* 689 * The EHCI spec doesn't say how long it takes the controller to 690 * stop accessing an unlinked interrupt QH. The timer delay is 691 * 9 uframes; presumably that will be long enough. 692 */ 693 qh->unlink_cycle = ehci->intr_unlink_cycle; 694 695 /* New entries go at the end of the intr_unlink list */ 696 list_add_tail(&qh->unlink_node, &ehci->intr_unlink); 697 698 if (ehci->intr_unlinking) 699 ; /* Avoid recursive calls */ 700 else if (ehci->rh_state < EHCI_RH_RUNNING) 701 ehci_handle_intr_unlinks(ehci); 702 else if (ehci->intr_unlink.next == &qh->unlink_node) { 703 ehci_enable_event(ehci, EHCI_HRTIMER_UNLINK_INTR, true); 704 ++ehci->intr_unlink_cycle; 705 } 706 } 707 708 /* 709 * It is common only one intr URB is scheduled on one qh, and 710 * given complete() is run in tasklet context, introduce a bit 711 * delay to avoid unlink qh too early. 712 */ 713 static void start_unlink_intr_wait(struct ehci_hcd *ehci, 714 struct ehci_qh *qh) 715 { 716 qh->unlink_cycle = ehci->intr_unlink_wait_cycle; 717 718 /* New entries go at the end of the intr_unlink_wait list */ 719 list_add_tail(&qh->unlink_node, &ehci->intr_unlink_wait); 720 721 if (ehci->rh_state < EHCI_RH_RUNNING) 722 ehci_handle_start_intr_unlinks(ehci); 723 else if (ehci->intr_unlink_wait.next == &qh->unlink_node) { 724 ehci_enable_event(ehci, EHCI_HRTIMER_START_UNLINK_INTR, true); 725 ++ehci->intr_unlink_wait_cycle; 726 } 727 } 728 729 static void end_unlink_intr(struct ehci_hcd *ehci, struct ehci_qh *qh) 730 { 731 struct ehci_qh_hw *hw = qh->hw; 732 int rc; 733 734 qh->qh_state = QH_STATE_IDLE; 735 hw->hw_next = EHCI_LIST_END(ehci); 736 737 if (!list_empty(&qh->qtd_list)) 738 qh_completions(ehci, qh); 739 740 /* reschedule QH iff another request is queued */ 741 if (!list_empty(&qh->qtd_list) && ehci->rh_state == EHCI_RH_RUNNING) { 742 rc = qh_schedule(ehci, qh); 743 if (rc == 0) { 744 qh_refresh(ehci, qh); 745 qh_link_periodic(ehci, qh); 746 } 747 748 /* An error here likely indicates handshake failure 749 * or no space left in the schedule. Neither fault 750 * should happen often ... 751 * 752 * FIXME kill the now-dysfunctional queued urbs 753 */ 754 else { 755 ehci_err(ehci, "can't reschedule qh %p, err %d\n", 756 qh, rc); 757 } 758 } 759 760 /* maybe turn off periodic schedule */ 761 --ehci->intr_count; 762 disable_periodic(ehci); 763 } 764 765 /*-------------------------------------------------------------------------*/ 766 767 static int check_period( 768 struct ehci_hcd *ehci, 769 unsigned frame, 770 unsigned uframe, 771 unsigned uperiod, 772 unsigned usecs 773 ) { 774 /* complete split running into next frame? 775 * given FSTN support, we could sometimes check... 776 */ 777 if (uframe >= 8) 778 return 0; 779 780 /* convert "usecs we need" to "max already claimed" */ 781 usecs = ehci->uframe_periodic_max - usecs; 782 783 for (uframe += frame << 3; uframe < EHCI_BANDWIDTH_SIZE; 784 uframe += uperiod) { 785 if (ehci->bandwidth[uframe] > usecs) 786 return 0; 787 } 788 789 /* success! */ 790 return 1; 791 } 792 793 static int check_intr_schedule( 794 struct ehci_hcd *ehci, 795 unsigned frame, 796 unsigned uframe, 797 struct ehci_qh *qh, 798 unsigned *c_maskp, 799 struct ehci_tt *tt 800 ) 801 { 802 int retval = -ENOSPC; 803 u8 mask = 0; 804 805 if (qh->ps.c_usecs && uframe >= 6) /* FSTN territory? */ 806 goto done; 807 808 if (!check_period(ehci, frame, uframe, qh->ps.bw_uperiod, qh->ps.usecs)) 809 goto done; 810 if (!qh->ps.c_usecs) { 811 retval = 0; 812 *c_maskp = 0; 813 goto done; 814 } 815 816 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED 817 if (tt_available(ehci, &qh->ps, tt, frame, uframe)) { 818 unsigned i; 819 820 /* TODO : this may need FSTN for SSPLIT in uframe 5. */ 821 for (i = uframe+2; i < 8 && i <= uframe+4; i++) 822 if (!check_period(ehci, frame, i, 823 qh->ps.bw_uperiod, qh->ps.c_usecs)) 824 goto done; 825 else 826 mask |= 1 << i; 827 828 retval = 0; 829 830 *c_maskp = mask; 831 } 832 #else 833 /* Make sure this tt's buffer is also available for CSPLITs. 834 * We pessimize a bit; probably the typical full speed case 835 * doesn't need the second CSPLIT. 836 * 837 * NOTE: both SPLIT and CSPLIT could be checked in just 838 * one smart pass... 839 */ 840 mask = 0x03 << (uframe + qh->gap_uf); 841 *c_maskp = mask; 842 843 mask |= 1 << uframe; 844 if (tt_no_collision(ehci, qh->ps.bw_period, qh->ps.udev, frame, mask)) { 845 if (!check_period(ehci, frame, uframe + qh->gap_uf + 1, 846 qh->ps.bw_uperiod, qh->ps.c_usecs)) 847 goto done; 848 if (!check_period(ehci, frame, uframe + qh->gap_uf, 849 qh->ps.bw_uperiod, qh->ps.c_usecs)) 850 goto done; 851 retval = 0; 852 } 853 #endif 854 done: 855 return retval; 856 } 857 858 /* "first fit" scheduling policy used the first time through, 859 * or when the previous schedule slot can't be re-used. 860 */ 861 static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh) 862 { 863 int status = 0; 864 unsigned uframe; 865 unsigned c_mask; 866 struct ehci_qh_hw *hw = qh->hw; 867 struct ehci_tt *tt; 868 869 hw->hw_next = EHCI_LIST_END(ehci); 870 871 /* reuse the previous schedule slots, if we can */ 872 if (qh->ps.phase != NO_FRAME) { 873 ehci_dbg(ehci, "reused qh %p schedule\n", qh); 874 return 0; 875 } 876 877 uframe = 0; 878 c_mask = 0; 879 tt = find_tt(qh->ps.udev); 880 if (IS_ERR(tt)) { 881 status = PTR_ERR(tt); 882 goto done; 883 } 884 compute_tt_budget(ehci->tt_budget, tt); 885 886 /* else scan the schedule to find a group of slots such that all 887 * uframes have enough periodic bandwidth available. 888 */ 889 /* "normal" case, uframing flexible except with splits */ 890 if (qh->ps.bw_period) { 891 int i; 892 unsigned frame; 893 894 for (i = qh->ps.bw_period; i > 0; --i) { 895 frame = ++ehci->random_frame & (qh->ps.bw_period - 1); 896 for (uframe = 0; uframe < 8; uframe++) { 897 status = check_intr_schedule(ehci, 898 frame, uframe, qh, &c_mask, tt); 899 if (status == 0) 900 goto got_it; 901 } 902 } 903 904 /* qh->ps.bw_period == 0 means every uframe */ 905 } else { 906 status = check_intr_schedule(ehci, 0, 0, qh, &c_mask, tt); 907 } 908 if (status) 909 goto done; 910 911 got_it: 912 qh->ps.phase = (qh->ps.period ? ehci->random_frame & 913 (qh->ps.period - 1) : 0); 914 qh->ps.bw_phase = qh->ps.phase & (qh->ps.bw_period - 1); 915 qh->ps.phase_uf = uframe; 916 qh->ps.cs_mask = qh->ps.period ? 917 (c_mask << 8) | (1 << uframe) : 918 QH_SMASK; 919 920 /* reset S-frame and (maybe) C-frame masks */ 921 hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK)); 922 hw->hw_info2 |= cpu_to_hc32(ehci, qh->ps.cs_mask); 923 reserve_release_intr_bandwidth(ehci, qh, 1); 924 925 done: 926 return status; 927 } 928 929 static int intr_submit( 930 struct ehci_hcd *ehci, 931 struct urb *urb, 932 struct list_head *qtd_list, 933 gfp_t mem_flags 934 ) { 935 unsigned epnum; 936 unsigned long flags; 937 struct ehci_qh *qh; 938 int status; 939 struct list_head empty; 940 941 /* get endpoint and transfer/schedule data */ 942 epnum = urb->ep->desc.bEndpointAddress; 943 944 spin_lock_irqsave(&ehci->lock, flags); 945 946 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 947 status = -ESHUTDOWN; 948 goto done_not_linked; 949 } 950 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 951 if (unlikely(status)) 952 goto done_not_linked; 953 954 /* get qh and force any scheduling errors */ 955 INIT_LIST_HEAD(&empty); 956 qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv); 957 if (qh == NULL) { 958 status = -ENOMEM; 959 goto done; 960 } 961 if (qh->qh_state == QH_STATE_IDLE) { 962 status = qh_schedule(ehci, qh); 963 if (status) 964 goto done; 965 } 966 967 /* then queue the urb's tds to the qh */ 968 qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv); 969 BUG_ON(qh == NULL); 970 971 /* stuff into the periodic schedule */ 972 if (qh->qh_state == QH_STATE_IDLE) { 973 qh_refresh(ehci, qh); 974 qh_link_periodic(ehci, qh); 975 } else { 976 /* cancel unlink wait for the qh */ 977 cancel_unlink_wait_intr(ehci, qh); 978 } 979 980 /* ... update usbfs periodic stats */ 981 ehci_to_hcd(ehci)->self.bandwidth_int_reqs++; 982 983 done: 984 if (unlikely(status)) 985 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 986 done_not_linked: 987 spin_unlock_irqrestore(&ehci->lock, flags); 988 if (status) 989 qtd_list_free(ehci, urb, qtd_list); 990 991 return status; 992 } 993 994 static void scan_intr(struct ehci_hcd *ehci) 995 { 996 struct ehci_qh *qh; 997 998 list_for_each_entry_safe(qh, ehci->qh_scan_next, &ehci->intr_qh_list, 999 intr_node) { 1000 1001 /* clean any finished work for this qh */ 1002 if (!list_empty(&qh->qtd_list)) { 1003 int temp; 1004 1005 /* 1006 * Unlinks could happen here; completion reporting 1007 * drops the lock. That's why ehci->qh_scan_next 1008 * always holds the next qh to scan; if the next qh 1009 * gets unlinked then ehci->qh_scan_next is adjusted 1010 * in qh_unlink_periodic(). 1011 */ 1012 temp = qh_completions(ehci, qh); 1013 if (unlikely(temp)) 1014 start_unlink_intr(ehci, qh); 1015 else if (unlikely(list_empty(&qh->qtd_list) && 1016 qh->qh_state == QH_STATE_LINKED)) 1017 start_unlink_intr_wait(ehci, qh); 1018 } 1019 } 1020 } 1021 1022 /*-------------------------------------------------------------------------*/ 1023 1024 /* ehci_iso_stream ops work with both ITD and SITD */ 1025 1026 static struct ehci_iso_stream * 1027 iso_stream_alloc(gfp_t mem_flags) 1028 { 1029 struct ehci_iso_stream *stream; 1030 1031 stream = kzalloc(sizeof(*stream), mem_flags); 1032 if (likely(stream != NULL)) { 1033 INIT_LIST_HEAD(&stream->td_list); 1034 INIT_LIST_HEAD(&stream->free_list); 1035 stream->next_uframe = NO_FRAME; 1036 stream->ps.phase = NO_FRAME; 1037 } 1038 return stream; 1039 } 1040 1041 static void 1042 iso_stream_init( 1043 struct ehci_hcd *ehci, 1044 struct ehci_iso_stream *stream, 1045 struct urb *urb 1046 ) 1047 { 1048 static const u8 smask_out[] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f }; 1049 1050 struct usb_device *dev = urb->dev; 1051 u32 buf1; 1052 unsigned epnum, maxp; 1053 int is_input; 1054 unsigned tmp; 1055 1056 /* 1057 * this might be a "high bandwidth" highspeed endpoint, 1058 * as encoded in the ep descriptor's wMaxPacket field 1059 */ 1060 epnum = usb_pipeendpoint(urb->pipe); 1061 is_input = usb_pipein(urb->pipe) ? USB_DIR_IN : 0; 1062 maxp = usb_endpoint_maxp(&urb->ep->desc); 1063 buf1 = is_input ? 1 << 11 : 0; 1064 1065 /* knows about ITD vs SITD */ 1066 if (dev->speed == USB_SPEED_HIGH) { 1067 unsigned multi = usb_endpoint_maxp_mult(&urb->ep->desc); 1068 1069 stream->highspeed = 1; 1070 1071 buf1 |= maxp; 1072 maxp *= multi; 1073 1074 stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum); 1075 stream->buf1 = cpu_to_hc32(ehci, buf1); 1076 stream->buf2 = cpu_to_hc32(ehci, multi); 1077 1078 /* usbfs wants to report the average usecs per frame tied up 1079 * when transfers on this endpoint are scheduled ... 1080 */ 1081 stream->ps.usecs = HS_USECS_ISO(maxp); 1082 1083 /* period for bandwidth allocation */ 1084 tmp = min_t(unsigned, EHCI_BANDWIDTH_SIZE, 1085 1 << (urb->ep->desc.bInterval - 1)); 1086 1087 /* Allow urb->interval to override */ 1088 stream->ps.bw_uperiod = min_t(unsigned, tmp, urb->interval); 1089 1090 stream->uperiod = urb->interval; 1091 stream->ps.period = urb->interval >> 3; 1092 stream->bandwidth = stream->ps.usecs * 8 / 1093 stream->ps.bw_uperiod; 1094 1095 } else { 1096 u32 addr; 1097 int think_time; 1098 int hs_transfers; 1099 1100 addr = dev->ttport << 24; 1101 if (!ehci_is_TDI(ehci) 1102 || (dev->tt->hub != 1103 ehci_to_hcd(ehci)->self.root_hub)) 1104 addr |= dev->tt->hub->devnum << 16; 1105 addr |= epnum << 8; 1106 addr |= dev->devnum; 1107 stream->ps.usecs = HS_USECS_ISO(maxp); 1108 think_time = dev->tt->think_time; 1109 stream->ps.tt_usecs = NS_TO_US(think_time + usb_calc_bus_time( 1110 dev->speed, is_input, 1, maxp)); 1111 hs_transfers = max(1u, (maxp + 187) / 188); 1112 if (is_input) { 1113 u32 tmp; 1114 1115 addr |= 1 << 31; 1116 stream->ps.c_usecs = stream->ps.usecs; 1117 stream->ps.usecs = HS_USECS_ISO(1); 1118 stream->ps.cs_mask = 1; 1119 1120 /* c-mask as specified in USB 2.0 11.18.4 3.c */ 1121 tmp = (1 << (hs_transfers + 2)) - 1; 1122 stream->ps.cs_mask |= tmp << (8 + 2); 1123 } else 1124 stream->ps.cs_mask = smask_out[hs_transfers - 1]; 1125 1126 /* period for bandwidth allocation */ 1127 tmp = min_t(unsigned, EHCI_BANDWIDTH_FRAMES, 1128 1 << (urb->ep->desc.bInterval - 1)); 1129 1130 /* Allow urb->interval to override */ 1131 stream->ps.bw_period = min_t(unsigned, tmp, urb->interval); 1132 stream->ps.bw_uperiod = stream->ps.bw_period << 3; 1133 1134 stream->ps.period = urb->interval; 1135 stream->uperiod = urb->interval << 3; 1136 stream->bandwidth = (stream->ps.usecs + stream->ps.c_usecs) / 1137 stream->ps.bw_period; 1138 1139 /* stream->splits gets created from cs_mask later */ 1140 stream->address = cpu_to_hc32(ehci, addr); 1141 } 1142 1143 stream->ps.udev = dev; 1144 stream->ps.ep = urb->ep; 1145 1146 stream->bEndpointAddress = is_input | epnum; 1147 stream->maxp = maxp; 1148 } 1149 1150 static struct ehci_iso_stream * 1151 iso_stream_find(struct ehci_hcd *ehci, struct urb *urb) 1152 { 1153 unsigned epnum; 1154 struct ehci_iso_stream *stream; 1155 struct usb_host_endpoint *ep; 1156 unsigned long flags; 1157 1158 epnum = usb_pipeendpoint (urb->pipe); 1159 if (usb_pipein(urb->pipe)) 1160 ep = urb->dev->ep_in[epnum]; 1161 else 1162 ep = urb->dev->ep_out[epnum]; 1163 1164 spin_lock_irqsave(&ehci->lock, flags); 1165 stream = ep->hcpriv; 1166 1167 if (unlikely(stream == NULL)) { 1168 stream = iso_stream_alloc(GFP_ATOMIC); 1169 if (likely(stream != NULL)) { 1170 ep->hcpriv = stream; 1171 iso_stream_init(ehci, stream, urb); 1172 } 1173 1174 /* if dev->ep [epnum] is a QH, hw is set */ 1175 } else if (unlikely(stream->hw != NULL)) { 1176 ehci_dbg(ehci, "dev %s ep%d%s, not iso??\n", 1177 urb->dev->devpath, epnum, 1178 usb_pipein(urb->pipe) ? "in" : "out"); 1179 stream = NULL; 1180 } 1181 1182 spin_unlock_irqrestore(&ehci->lock, flags); 1183 return stream; 1184 } 1185 1186 /*-------------------------------------------------------------------------*/ 1187 1188 /* ehci_iso_sched ops can be ITD-only or SITD-only */ 1189 1190 static struct ehci_iso_sched * 1191 iso_sched_alloc(unsigned packets, gfp_t mem_flags) 1192 { 1193 struct ehci_iso_sched *iso_sched; 1194 int size = sizeof(*iso_sched); 1195 1196 size += packets * sizeof(struct ehci_iso_packet); 1197 iso_sched = kzalloc(size, mem_flags); 1198 if (likely(iso_sched != NULL)) 1199 INIT_LIST_HEAD(&iso_sched->td_list); 1200 1201 return iso_sched; 1202 } 1203 1204 static inline void 1205 itd_sched_init( 1206 struct ehci_hcd *ehci, 1207 struct ehci_iso_sched *iso_sched, 1208 struct ehci_iso_stream *stream, 1209 struct urb *urb 1210 ) 1211 { 1212 unsigned i; 1213 dma_addr_t dma = urb->transfer_dma; 1214 1215 /* how many uframes are needed for these transfers */ 1216 iso_sched->span = urb->number_of_packets * stream->uperiod; 1217 1218 /* figure out per-uframe itd fields that we'll need later 1219 * when we fit new itds into the schedule. 1220 */ 1221 for (i = 0; i < urb->number_of_packets; i++) { 1222 struct ehci_iso_packet *uframe = &iso_sched->packet[i]; 1223 unsigned length; 1224 dma_addr_t buf; 1225 u32 trans; 1226 1227 length = urb->iso_frame_desc[i].length; 1228 buf = dma + urb->iso_frame_desc[i].offset; 1229 1230 trans = EHCI_ISOC_ACTIVE; 1231 trans |= buf & 0x0fff; 1232 if (unlikely(((i + 1) == urb->number_of_packets)) 1233 && !(urb->transfer_flags & URB_NO_INTERRUPT)) 1234 trans |= EHCI_ITD_IOC; 1235 trans |= length << 16; 1236 uframe->transaction = cpu_to_hc32(ehci, trans); 1237 1238 /* might need to cross a buffer page within a uframe */ 1239 uframe->bufp = (buf & ~(u64)0x0fff); 1240 buf += length; 1241 if (unlikely((uframe->bufp != (buf & ~(u64)0x0fff)))) 1242 uframe->cross = 1; 1243 } 1244 } 1245 1246 static void 1247 iso_sched_free( 1248 struct ehci_iso_stream *stream, 1249 struct ehci_iso_sched *iso_sched 1250 ) 1251 { 1252 if (!iso_sched) 1253 return; 1254 /* caller must hold ehci->lock! */ 1255 list_splice(&iso_sched->td_list, &stream->free_list); 1256 kfree(iso_sched); 1257 } 1258 1259 static int 1260 itd_urb_transaction( 1261 struct ehci_iso_stream *stream, 1262 struct ehci_hcd *ehci, 1263 struct urb *urb, 1264 gfp_t mem_flags 1265 ) 1266 { 1267 struct ehci_itd *itd; 1268 dma_addr_t itd_dma; 1269 int i; 1270 unsigned num_itds; 1271 struct ehci_iso_sched *sched; 1272 unsigned long flags; 1273 1274 sched = iso_sched_alloc(urb->number_of_packets, mem_flags); 1275 if (unlikely(sched == NULL)) 1276 return -ENOMEM; 1277 1278 itd_sched_init(ehci, sched, stream, urb); 1279 1280 if (urb->interval < 8) 1281 num_itds = 1 + (sched->span + 7) / 8; 1282 else 1283 num_itds = urb->number_of_packets; 1284 1285 /* allocate/init ITDs */ 1286 spin_lock_irqsave(&ehci->lock, flags); 1287 for (i = 0; i < num_itds; i++) { 1288 1289 /* 1290 * Use iTDs from the free list, but not iTDs that may 1291 * still be in use by the hardware. 1292 */ 1293 if (likely(!list_empty(&stream->free_list))) { 1294 itd = list_first_entry(&stream->free_list, 1295 struct ehci_itd, itd_list); 1296 if (itd->frame == ehci->now_frame) 1297 goto alloc_itd; 1298 list_del(&itd->itd_list); 1299 itd_dma = itd->itd_dma; 1300 } else { 1301 alloc_itd: 1302 spin_unlock_irqrestore(&ehci->lock, flags); 1303 itd = dma_pool_alloc(ehci->itd_pool, mem_flags, 1304 &itd_dma); 1305 spin_lock_irqsave(&ehci->lock, flags); 1306 if (!itd) { 1307 iso_sched_free(stream, sched); 1308 spin_unlock_irqrestore(&ehci->lock, flags); 1309 return -ENOMEM; 1310 } 1311 } 1312 1313 memset(itd, 0, sizeof(*itd)); 1314 itd->itd_dma = itd_dma; 1315 itd->frame = NO_FRAME; 1316 list_add(&itd->itd_list, &sched->td_list); 1317 } 1318 spin_unlock_irqrestore(&ehci->lock, flags); 1319 1320 /* temporarily store schedule info in hcpriv */ 1321 urb->hcpriv = sched; 1322 urb->error_count = 0; 1323 return 0; 1324 } 1325 1326 /*-------------------------------------------------------------------------*/ 1327 1328 static void reserve_release_iso_bandwidth(struct ehci_hcd *ehci, 1329 struct ehci_iso_stream *stream, int sign) 1330 { 1331 unsigned uframe; 1332 unsigned i, j; 1333 unsigned s_mask, c_mask, m; 1334 int usecs = stream->ps.usecs; 1335 int c_usecs = stream->ps.c_usecs; 1336 int tt_usecs = stream->ps.tt_usecs; 1337 struct ehci_tt *tt; 1338 1339 if (stream->ps.phase == NO_FRAME) /* Bandwidth wasn't reserved */ 1340 return; 1341 uframe = stream->ps.bw_phase << 3; 1342 1343 bandwidth_dbg(ehci, sign, "iso", &stream->ps); 1344 1345 if (sign < 0) { /* Release bandwidth */ 1346 usecs = -usecs; 1347 c_usecs = -c_usecs; 1348 tt_usecs = -tt_usecs; 1349 } 1350 1351 if (!stream->splits) { /* High speed */ 1352 for (i = uframe + stream->ps.phase_uf; i < EHCI_BANDWIDTH_SIZE; 1353 i += stream->ps.bw_uperiod) 1354 ehci->bandwidth[i] += usecs; 1355 1356 } else { /* Full speed */ 1357 s_mask = stream->ps.cs_mask; 1358 c_mask = s_mask >> 8; 1359 1360 /* NOTE: adjustment needed for frame overflow */ 1361 for (i = uframe; i < EHCI_BANDWIDTH_SIZE; 1362 i += stream->ps.bw_uperiod) { 1363 for ((j = stream->ps.phase_uf, m = 1 << j); j < 8; 1364 (++j, m <<= 1)) { 1365 if (s_mask & m) 1366 ehci->bandwidth[i+j] += usecs; 1367 else if (c_mask & m) 1368 ehci->bandwidth[i+j] += c_usecs; 1369 } 1370 } 1371 1372 tt = find_tt(stream->ps.udev); 1373 if (sign > 0) 1374 list_add_tail(&stream->ps.ps_list, &tt->ps_list); 1375 else 1376 list_del(&stream->ps.ps_list); 1377 1378 for (i = uframe >> 3; i < EHCI_BANDWIDTH_FRAMES; 1379 i += stream->ps.bw_period) 1380 tt->bandwidth[i] += tt_usecs; 1381 } 1382 } 1383 1384 static inline int 1385 itd_slot_ok( 1386 struct ehci_hcd *ehci, 1387 struct ehci_iso_stream *stream, 1388 unsigned uframe 1389 ) 1390 { 1391 unsigned usecs; 1392 1393 /* convert "usecs we need" to "max already claimed" */ 1394 usecs = ehci->uframe_periodic_max - stream->ps.usecs; 1395 1396 for (uframe &= stream->ps.bw_uperiod - 1; uframe < EHCI_BANDWIDTH_SIZE; 1397 uframe += stream->ps.bw_uperiod) { 1398 if (ehci->bandwidth[uframe] > usecs) 1399 return 0; 1400 } 1401 return 1; 1402 } 1403 1404 static inline int 1405 sitd_slot_ok( 1406 struct ehci_hcd *ehci, 1407 struct ehci_iso_stream *stream, 1408 unsigned uframe, 1409 struct ehci_iso_sched *sched, 1410 struct ehci_tt *tt 1411 ) 1412 { 1413 unsigned mask, tmp; 1414 unsigned frame, uf; 1415 1416 mask = stream->ps.cs_mask << (uframe & 7); 1417 1418 /* for OUT, don't wrap SSPLIT into H-microframe 7 */ 1419 if (((stream->ps.cs_mask & 0xff) << (uframe & 7)) >= (1 << 7)) 1420 return 0; 1421 1422 /* for IN, don't wrap CSPLIT into the next frame */ 1423 if (mask & ~0xffff) 1424 return 0; 1425 1426 /* check bandwidth */ 1427 uframe &= stream->ps.bw_uperiod - 1; 1428 frame = uframe >> 3; 1429 1430 #ifdef CONFIG_USB_EHCI_TT_NEWSCHED 1431 /* The tt's fullspeed bus bandwidth must be available. 1432 * tt_available scheduling guarantees 10+% for control/bulk. 1433 */ 1434 uf = uframe & 7; 1435 if (!tt_available(ehci, &stream->ps, tt, frame, uf)) 1436 return 0; 1437 #else 1438 /* tt must be idle for start(s), any gap, and csplit. 1439 * assume scheduling slop leaves 10+% for control/bulk. 1440 */ 1441 if (!tt_no_collision(ehci, stream->ps.bw_period, 1442 stream->ps.udev, frame, mask)) 1443 return 0; 1444 #endif 1445 1446 do { 1447 unsigned max_used; 1448 unsigned i; 1449 1450 /* check starts (OUT uses more than one) */ 1451 uf = uframe; 1452 max_used = ehci->uframe_periodic_max - stream->ps.usecs; 1453 for (tmp = stream->ps.cs_mask & 0xff; tmp; tmp >>= 1, uf++) { 1454 if (ehci->bandwidth[uf] > max_used) 1455 return 0; 1456 } 1457 1458 /* for IN, check CSPLIT */ 1459 if (stream->ps.c_usecs) { 1460 max_used = ehci->uframe_periodic_max - 1461 stream->ps.c_usecs; 1462 uf = uframe & ~7; 1463 tmp = 1 << (2+8); 1464 for (i = (uframe & 7) + 2; i < 8; (++i, tmp <<= 1)) { 1465 if ((stream->ps.cs_mask & tmp) == 0) 1466 continue; 1467 if (ehci->bandwidth[uf+i] > max_used) 1468 return 0; 1469 } 1470 } 1471 1472 uframe += stream->ps.bw_uperiod; 1473 } while (uframe < EHCI_BANDWIDTH_SIZE); 1474 1475 stream->ps.cs_mask <<= uframe & 7; 1476 stream->splits = cpu_to_hc32(ehci, stream->ps.cs_mask); 1477 return 1; 1478 } 1479 1480 /* 1481 * This scheduler plans almost as far into the future as it has actual 1482 * periodic schedule slots. (Affected by TUNE_FLS, which defaults to 1483 * "as small as possible" to be cache-friendlier.) That limits the size 1484 * transfers you can stream reliably; avoid more than 64 msec per urb. 1485 * Also avoid queue depths of less than ehci's worst irq latency (affected 1486 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter, 1487 * and other factors); or more than about 230 msec total (for portability, 1488 * given EHCI_TUNE_FLS and the slop). Or, write a smarter scheduler! 1489 */ 1490 1491 static int 1492 iso_stream_schedule( 1493 struct ehci_hcd *ehci, 1494 struct urb *urb, 1495 struct ehci_iso_stream *stream 1496 ) 1497 { 1498 u32 now, base, next, start, period, span, now2; 1499 u32 wrap = 0, skip = 0; 1500 int status = 0; 1501 unsigned mod = ehci->periodic_size << 3; 1502 struct ehci_iso_sched *sched = urb->hcpriv; 1503 bool empty = list_empty(&stream->td_list); 1504 bool new_stream = false; 1505 1506 period = stream->uperiod; 1507 span = sched->span; 1508 if (!stream->highspeed) 1509 span <<= 3; 1510 1511 /* Start a new isochronous stream? */ 1512 if (unlikely(empty && !hcd_periodic_completion_in_progress( 1513 ehci_to_hcd(ehci), urb->ep))) { 1514 1515 /* Schedule the endpoint */ 1516 if (stream->ps.phase == NO_FRAME) { 1517 int done = 0; 1518 struct ehci_tt *tt = find_tt(stream->ps.udev); 1519 1520 if (IS_ERR(tt)) { 1521 status = PTR_ERR(tt); 1522 goto fail; 1523 } 1524 compute_tt_budget(ehci->tt_budget, tt); 1525 1526 start = ((-(++ehci->random_frame)) << 3) & (period - 1); 1527 1528 /* find a uframe slot with enough bandwidth. 1529 * Early uframes are more precious because full-speed 1530 * iso IN transfers can't use late uframes, 1531 * and therefore they should be allocated last. 1532 */ 1533 next = start; 1534 start += period; 1535 do { 1536 start--; 1537 /* check schedule: enough space? */ 1538 if (stream->highspeed) { 1539 if (itd_slot_ok(ehci, stream, start)) 1540 done = 1; 1541 } else { 1542 if ((start % 8) >= 6) 1543 continue; 1544 if (sitd_slot_ok(ehci, stream, start, 1545 sched, tt)) 1546 done = 1; 1547 } 1548 } while (start > next && !done); 1549 1550 /* no room in the schedule */ 1551 if (!done) { 1552 ehci_dbg(ehci, "iso sched full %p", urb); 1553 status = -ENOSPC; 1554 goto fail; 1555 } 1556 stream->ps.phase = (start >> 3) & 1557 (stream->ps.period - 1); 1558 stream->ps.bw_phase = stream->ps.phase & 1559 (stream->ps.bw_period - 1); 1560 stream->ps.phase_uf = start & 7; 1561 reserve_release_iso_bandwidth(ehci, stream, 1); 1562 } 1563 1564 /* New stream is already scheduled; use the upcoming slot */ 1565 else { 1566 start = (stream->ps.phase << 3) + stream->ps.phase_uf; 1567 } 1568 1569 stream->next_uframe = start; 1570 new_stream = true; 1571 } 1572 1573 now = ehci_read_frame_index(ehci) & (mod - 1); 1574 1575 /* Take the isochronous scheduling threshold into account */ 1576 if (ehci->i_thresh) 1577 next = now + ehci->i_thresh; /* uframe cache */ 1578 else 1579 next = (now + 2 + 7) & ~0x07; /* full frame cache */ 1580 1581 /* If needed, initialize last_iso_frame so that this URB will be seen */ 1582 if (ehci->isoc_count == 0) 1583 ehci->last_iso_frame = now >> 3; 1584 1585 /* 1586 * Use ehci->last_iso_frame as the base. There can't be any 1587 * TDs scheduled for earlier than that. 1588 */ 1589 base = ehci->last_iso_frame << 3; 1590 next = (next - base) & (mod - 1); 1591 start = (stream->next_uframe - base) & (mod - 1); 1592 1593 if (unlikely(new_stream)) 1594 goto do_ASAP; 1595 1596 /* 1597 * Typical case: reuse current schedule, stream may still be active. 1598 * Hopefully there are no gaps from the host falling behind 1599 * (irq delays etc). If there are, the behavior depends on 1600 * whether URB_ISO_ASAP is set. 1601 */ 1602 now2 = (now - base) & (mod - 1); 1603 1604 /* Is the schedule about to wrap around? */ 1605 if (unlikely(!empty && start < period)) { 1606 ehci_dbg(ehci, "request %p would overflow (%u-%u < %u mod %u)\n", 1607 urb, stream->next_uframe, base, period, mod); 1608 status = -EFBIG; 1609 goto fail; 1610 } 1611 1612 /* Is the next packet scheduled after the base time? */ 1613 if (likely(!empty || start <= now2 + period)) { 1614 1615 /* URB_ISO_ASAP: make sure that start >= next */ 1616 if (unlikely(start < next && 1617 (urb->transfer_flags & URB_ISO_ASAP))) 1618 goto do_ASAP; 1619 1620 /* Otherwise use start, if it's not in the past */ 1621 if (likely(start >= now2)) 1622 goto use_start; 1623 1624 /* Otherwise we got an underrun while the queue was empty */ 1625 } else { 1626 if (urb->transfer_flags & URB_ISO_ASAP) 1627 goto do_ASAP; 1628 wrap = mod; 1629 now2 += mod; 1630 } 1631 1632 /* How many uframes and packets do we need to skip? */ 1633 skip = (now2 - start + period - 1) & -period; 1634 if (skip >= span) { /* Entirely in the past? */ 1635 ehci_dbg(ehci, "iso underrun %p (%u+%u < %u) [%u]\n", 1636 urb, start + base, span - period, now2 + base, 1637 base); 1638 1639 /* Try to keep the last TD intact for scanning later */ 1640 skip = span - period; 1641 1642 /* Will it come before the current scan position? */ 1643 if (empty) { 1644 skip = span; /* Skip the entire URB */ 1645 status = 1; /* and give it back immediately */ 1646 iso_sched_free(stream, sched); 1647 sched = NULL; 1648 } 1649 } 1650 urb->error_count = skip / period; 1651 if (sched) 1652 sched->first_packet = urb->error_count; 1653 goto use_start; 1654 1655 do_ASAP: 1656 /* Use the first slot after "next" */ 1657 start = next + ((start - next) & (period - 1)); 1658 1659 use_start: 1660 /* Tried to schedule too far into the future? */ 1661 if (unlikely(start + span - period >= mod + wrap)) { 1662 ehci_dbg(ehci, "request %p would overflow (%u+%u >= %u)\n", 1663 urb, start, span - period, mod + wrap); 1664 status = -EFBIG; 1665 goto fail; 1666 } 1667 1668 start += base; 1669 stream->next_uframe = (start + skip) & (mod - 1); 1670 1671 /* report high speed start in uframes; full speed, in frames */ 1672 urb->start_frame = start & (mod - 1); 1673 if (!stream->highspeed) 1674 urb->start_frame >>= 3; 1675 return status; 1676 1677 fail: 1678 iso_sched_free(stream, sched); 1679 urb->hcpriv = NULL; 1680 return status; 1681 } 1682 1683 /*-------------------------------------------------------------------------*/ 1684 1685 static inline void 1686 itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream, 1687 struct ehci_itd *itd) 1688 { 1689 int i; 1690 1691 /* it's been recently zeroed */ 1692 itd->hw_next = EHCI_LIST_END(ehci); 1693 itd->hw_bufp[0] = stream->buf0; 1694 itd->hw_bufp[1] = stream->buf1; 1695 itd->hw_bufp[2] = stream->buf2; 1696 1697 for (i = 0; i < 8; i++) 1698 itd->index[i] = -1; 1699 1700 /* All other fields are filled when scheduling */ 1701 } 1702 1703 static inline void 1704 itd_patch( 1705 struct ehci_hcd *ehci, 1706 struct ehci_itd *itd, 1707 struct ehci_iso_sched *iso_sched, 1708 unsigned index, 1709 u16 uframe 1710 ) 1711 { 1712 struct ehci_iso_packet *uf = &iso_sched->packet[index]; 1713 unsigned pg = itd->pg; 1714 1715 /* BUG_ON(pg == 6 && uf->cross); */ 1716 1717 uframe &= 0x07; 1718 itd->index[uframe] = index; 1719 1720 itd->hw_transaction[uframe] = uf->transaction; 1721 itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12); 1722 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0); 1723 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32)); 1724 1725 /* iso_frame_desc[].offset must be strictly increasing */ 1726 if (unlikely(uf->cross)) { 1727 u64 bufp = uf->bufp + 4096; 1728 1729 itd->pg = ++pg; 1730 itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0); 1731 itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32)); 1732 } 1733 } 1734 1735 static inline void 1736 itd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd) 1737 { 1738 union ehci_shadow *prev = &ehci->pshadow[frame]; 1739 __hc32 *hw_p = &ehci->periodic[frame]; 1740 union ehci_shadow here = *prev; 1741 __hc32 type = 0; 1742 1743 /* skip any iso nodes which might belong to previous microframes */ 1744 while (here.ptr) { 1745 type = Q_NEXT_TYPE(ehci, *hw_p); 1746 if (type == cpu_to_hc32(ehci, Q_TYPE_QH)) 1747 break; 1748 prev = periodic_next_shadow(ehci, prev, type); 1749 hw_p = shadow_next_periodic(ehci, &here, type); 1750 here = *prev; 1751 } 1752 1753 itd->itd_next = here; 1754 itd->hw_next = *hw_p; 1755 prev->itd = itd; 1756 itd->frame = frame; 1757 wmb(); 1758 *hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD); 1759 } 1760 1761 /* fit urb's itds into the selected schedule slot; activate as needed */ 1762 static void itd_link_urb( 1763 struct ehci_hcd *ehci, 1764 struct urb *urb, 1765 unsigned mod, 1766 struct ehci_iso_stream *stream 1767 ) 1768 { 1769 int packet; 1770 unsigned next_uframe, uframe, frame; 1771 struct ehci_iso_sched *iso_sched = urb->hcpriv; 1772 struct ehci_itd *itd; 1773 1774 next_uframe = stream->next_uframe & (mod - 1); 1775 1776 if (unlikely(list_empty(&stream->td_list))) 1777 ehci_to_hcd(ehci)->self.bandwidth_allocated 1778 += stream->bandwidth; 1779 1780 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 1781 if (ehci->amd_pll_fix == 1) 1782 usb_amd_quirk_pll_disable(); 1783 } 1784 1785 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; 1786 1787 /* fill iTDs uframe by uframe */ 1788 for (packet = iso_sched->first_packet, itd = NULL; 1789 packet < urb->number_of_packets;) { 1790 if (itd == NULL) { 1791 /* ASSERT: we have all necessary itds */ 1792 /* BUG_ON(list_empty(&iso_sched->td_list)); */ 1793 1794 /* ASSERT: no itds for this endpoint in this uframe */ 1795 1796 itd = list_entry(iso_sched->td_list.next, 1797 struct ehci_itd, itd_list); 1798 list_move_tail(&itd->itd_list, &stream->td_list); 1799 itd->stream = stream; 1800 itd->urb = urb; 1801 itd_init(ehci, stream, itd); 1802 } 1803 1804 uframe = next_uframe & 0x07; 1805 frame = next_uframe >> 3; 1806 1807 itd_patch(ehci, itd, iso_sched, packet, uframe); 1808 1809 next_uframe += stream->uperiod; 1810 next_uframe &= mod - 1; 1811 packet++; 1812 1813 /* link completed itds into the schedule */ 1814 if (((next_uframe >> 3) != frame) 1815 || packet == urb->number_of_packets) { 1816 itd_link(ehci, frame & (ehci->periodic_size - 1), itd); 1817 itd = NULL; 1818 } 1819 } 1820 stream->next_uframe = next_uframe; 1821 1822 /* don't need that schedule data any more */ 1823 iso_sched_free(stream, iso_sched); 1824 urb->hcpriv = stream; 1825 1826 ++ehci->isoc_count; 1827 enable_periodic(ehci); 1828 } 1829 1830 #define ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR) 1831 1832 /* Process and recycle a completed ITD. Return true iff its urb completed, 1833 * and hence its completion callback probably added things to the hardware 1834 * schedule. 1835 * 1836 * Note that we carefully avoid recycling this descriptor until after any 1837 * completion callback runs, so that it won't be reused quickly. That is, 1838 * assuming (a) no more than two urbs per frame on this endpoint, and also 1839 * (b) only this endpoint's completions submit URBs. It seems some silicon 1840 * corrupts things if you reuse completed descriptors very quickly... 1841 */ 1842 static bool itd_complete(struct ehci_hcd *ehci, struct ehci_itd *itd) 1843 { 1844 struct urb *urb = itd->urb; 1845 struct usb_iso_packet_descriptor *desc; 1846 u32 t; 1847 unsigned uframe; 1848 int urb_index = -1; 1849 struct ehci_iso_stream *stream = itd->stream; 1850 struct usb_device *dev; 1851 bool retval = false; 1852 1853 /* for each uframe with a packet */ 1854 for (uframe = 0; uframe < 8; uframe++) { 1855 if (likely(itd->index[uframe] == -1)) 1856 continue; 1857 urb_index = itd->index[uframe]; 1858 desc = &urb->iso_frame_desc[urb_index]; 1859 1860 t = hc32_to_cpup(ehci, &itd->hw_transaction[uframe]); 1861 itd->hw_transaction[uframe] = 0; 1862 1863 /* report transfer status */ 1864 if (unlikely(t & ISO_ERRS)) { 1865 urb->error_count++; 1866 if (t & EHCI_ISOC_BUF_ERR) 1867 desc->status = usb_pipein(urb->pipe) 1868 ? -ENOSR /* hc couldn't read */ 1869 : -ECOMM; /* hc couldn't write */ 1870 else if (t & EHCI_ISOC_BABBLE) 1871 desc->status = -EOVERFLOW; 1872 else /* (t & EHCI_ISOC_XACTERR) */ 1873 desc->status = -EPROTO; 1874 1875 /* HC need not update length with this error */ 1876 if (!(t & EHCI_ISOC_BABBLE)) { 1877 desc->actual_length = EHCI_ITD_LENGTH(t); 1878 urb->actual_length += desc->actual_length; 1879 } 1880 } else if (likely((t & EHCI_ISOC_ACTIVE) == 0)) { 1881 desc->status = 0; 1882 desc->actual_length = EHCI_ITD_LENGTH(t); 1883 urb->actual_length += desc->actual_length; 1884 } else { 1885 /* URB was too late */ 1886 urb->error_count++; 1887 } 1888 } 1889 1890 /* handle completion now? */ 1891 if (likely((urb_index + 1) != urb->number_of_packets)) 1892 goto done; 1893 1894 /* 1895 * ASSERT: it's really the last itd for this urb 1896 * list_for_each_entry (itd, &stream->td_list, itd_list) 1897 * BUG_ON(itd->urb == urb); 1898 */ 1899 1900 /* give urb back to the driver; completion often (re)submits */ 1901 dev = urb->dev; 1902 ehci_urb_done(ehci, urb, 0); 1903 retval = true; 1904 urb = NULL; 1905 1906 --ehci->isoc_count; 1907 disable_periodic(ehci); 1908 1909 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; 1910 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 1911 if (ehci->amd_pll_fix == 1) 1912 usb_amd_quirk_pll_enable(); 1913 } 1914 1915 if (unlikely(list_is_singular(&stream->td_list))) 1916 ehci_to_hcd(ehci)->self.bandwidth_allocated 1917 -= stream->bandwidth; 1918 1919 done: 1920 itd->urb = NULL; 1921 1922 /* Add to the end of the free list for later reuse */ 1923 list_move_tail(&itd->itd_list, &stream->free_list); 1924 1925 /* Recycle the iTDs when the pipeline is empty (ep no longer in use) */ 1926 if (list_empty(&stream->td_list)) { 1927 list_splice_tail_init(&stream->free_list, 1928 &ehci->cached_itd_list); 1929 start_free_itds(ehci); 1930 } 1931 1932 return retval; 1933 } 1934 1935 /*-------------------------------------------------------------------------*/ 1936 1937 static int itd_submit(struct ehci_hcd *ehci, struct urb *urb, 1938 gfp_t mem_flags) 1939 { 1940 int status = -EINVAL; 1941 unsigned long flags; 1942 struct ehci_iso_stream *stream; 1943 1944 /* Get iso_stream head */ 1945 stream = iso_stream_find(ehci, urb); 1946 if (unlikely(stream == NULL)) { 1947 ehci_dbg(ehci, "can't get iso stream\n"); 1948 return -ENOMEM; 1949 } 1950 if (unlikely(urb->interval != stream->uperiod)) { 1951 ehci_dbg(ehci, "can't change iso interval %d --> %d\n", 1952 stream->uperiod, urb->interval); 1953 goto done; 1954 } 1955 1956 #ifdef EHCI_URB_TRACE 1957 ehci_dbg(ehci, 1958 "%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n", 1959 __func__, urb->dev->devpath, urb, 1960 usb_pipeendpoint(urb->pipe), 1961 usb_pipein(urb->pipe) ? "in" : "out", 1962 urb->transfer_buffer_length, 1963 urb->number_of_packets, urb->interval, 1964 stream); 1965 #endif 1966 1967 /* allocate ITDs w/o locking anything */ 1968 status = itd_urb_transaction(stream, ehci, urb, mem_flags); 1969 if (unlikely(status < 0)) { 1970 ehci_dbg(ehci, "can't init itds\n"); 1971 goto done; 1972 } 1973 1974 /* schedule ... need to lock */ 1975 spin_lock_irqsave(&ehci->lock, flags); 1976 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 1977 status = -ESHUTDOWN; 1978 goto done_not_linked; 1979 } 1980 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 1981 if (unlikely(status)) 1982 goto done_not_linked; 1983 status = iso_stream_schedule(ehci, urb, stream); 1984 if (likely(status == 0)) { 1985 itd_link_urb(ehci, urb, ehci->periodic_size << 3, stream); 1986 } else if (status > 0) { 1987 status = 0; 1988 ehci_urb_done(ehci, urb, 0); 1989 } else { 1990 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 1991 } 1992 done_not_linked: 1993 spin_unlock_irqrestore(&ehci->lock, flags); 1994 done: 1995 return status; 1996 } 1997 1998 /*-------------------------------------------------------------------------*/ 1999 2000 /* 2001 * "Split ISO TDs" ... used for USB 1.1 devices going through the 2002 * TTs in USB 2.0 hubs. These need microframe scheduling. 2003 */ 2004 2005 static inline void 2006 sitd_sched_init( 2007 struct ehci_hcd *ehci, 2008 struct ehci_iso_sched *iso_sched, 2009 struct ehci_iso_stream *stream, 2010 struct urb *urb 2011 ) 2012 { 2013 unsigned i; 2014 dma_addr_t dma = urb->transfer_dma; 2015 2016 /* how many frames are needed for these transfers */ 2017 iso_sched->span = urb->number_of_packets * stream->ps.period; 2018 2019 /* figure out per-frame sitd fields that we'll need later 2020 * when we fit new sitds into the schedule. 2021 */ 2022 for (i = 0; i < urb->number_of_packets; i++) { 2023 struct ehci_iso_packet *packet = &iso_sched->packet[i]; 2024 unsigned length; 2025 dma_addr_t buf; 2026 u32 trans; 2027 2028 length = urb->iso_frame_desc[i].length & 0x03ff; 2029 buf = dma + urb->iso_frame_desc[i].offset; 2030 2031 trans = SITD_STS_ACTIVE; 2032 if (((i + 1) == urb->number_of_packets) 2033 && !(urb->transfer_flags & URB_NO_INTERRUPT)) 2034 trans |= SITD_IOC; 2035 trans |= length << 16; 2036 packet->transaction = cpu_to_hc32(ehci, trans); 2037 2038 /* might need to cross a buffer page within a td */ 2039 packet->bufp = buf; 2040 packet->buf1 = (buf + length) & ~0x0fff; 2041 if (packet->buf1 != (buf & ~(u64)0x0fff)) 2042 packet->cross = 1; 2043 2044 /* OUT uses multiple start-splits */ 2045 if (stream->bEndpointAddress & USB_DIR_IN) 2046 continue; 2047 length = (length + 187) / 188; 2048 if (length > 1) /* BEGIN vs ALL */ 2049 length |= 1 << 3; 2050 packet->buf1 |= length; 2051 } 2052 } 2053 2054 static int 2055 sitd_urb_transaction( 2056 struct ehci_iso_stream *stream, 2057 struct ehci_hcd *ehci, 2058 struct urb *urb, 2059 gfp_t mem_flags 2060 ) 2061 { 2062 struct ehci_sitd *sitd; 2063 dma_addr_t sitd_dma; 2064 int i; 2065 struct ehci_iso_sched *iso_sched; 2066 unsigned long flags; 2067 2068 iso_sched = iso_sched_alloc(urb->number_of_packets, mem_flags); 2069 if (iso_sched == NULL) 2070 return -ENOMEM; 2071 2072 sitd_sched_init(ehci, iso_sched, stream, urb); 2073 2074 /* allocate/init sITDs */ 2075 spin_lock_irqsave(&ehci->lock, flags); 2076 for (i = 0; i < urb->number_of_packets; i++) { 2077 2078 /* NOTE: for now, we don't try to handle wraparound cases 2079 * for IN (using sitd->hw_backpointer, like a FSTN), which 2080 * means we never need two sitds for full speed packets. 2081 */ 2082 2083 /* 2084 * Use siTDs from the free list, but not siTDs that may 2085 * still be in use by the hardware. 2086 */ 2087 if (likely(!list_empty(&stream->free_list))) { 2088 sitd = list_first_entry(&stream->free_list, 2089 struct ehci_sitd, sitd_list); 2090 if (sitd->frame == ehci->now_frame) 2091 goto alloc_sitd; 2092 list_del(&sitd->sitd_list); 2093 sitd_dma = sitd->sitd_dma; 2094 } else { 2095 alloc_sitd: 2096 spin_unlock_irqrestore(&ehci->lock, flags); 2097 sitd = dma_pool_alloc(ehci->sitd_pool, mem_flags, 2098 &sitd_dma); 2099 spin_lock_irqsave(&ehci->lock, flags); 2100 if (!sitd) { 2101 iso_sched_free(stream, iso_sched); 2102 spin_unlock_irqrestore(&ehci->lock, flags); 2103 return -ENOMEM; 2104 } 2105 } 2106 2107 memset(sitd, 0, sizeof(*sitd)); 2108 sitd->sitd_dma = sitd_dma; 2109 sitd->frame = NO_FRAME; 2110 list_add(&sitd->sitd_list, &iso_sched->td_list); 2111 } 2112 2113 /* temporarily store schedule info in hcpriv */ 2114 urb->hcpriv = iso_sched; 2115 urb->error_count = 0; 2116 2117 spin_unlock_irqrestore(&ehci->lock, flags); 2118 return 0; 2119 } 2120 2121 /*-------------------------------------------------------------------------*/ 2122 2123 static inline void 2124 sitd_patch( 2125 struct ehci_hcd *ehci, 2126 struct ehci_iso_stream *stream, 2127 struct ehci_sitd *sitd, 2128 struct ehci_iso_sched *iso_sched, 2129 unsigned index 2130 ) 2131 { 2132 struct ehci_iso_packet *uf = &iso_sched->packet[index]; 2133 u64 bufp; 2134 2135 sitd->hw_next = EHCI_LIST_END(ehci); 2136 sitd->hw_fullspeed_ep = stream->address; 2137 sitd->hw_uframe = stream->splits; 2138 sitd->hw_results = uf->transaction; 2139 sitd->hw_backpointer = EHCI_LIST_END(ehci); 2140 2141 bufp = uf->bufp; 2142 sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp); 2143 sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32); 2144 2145 sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1); 2146 if (uf->cross) 2147 bufp += 4096; 2148 sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32); 2149 sitd->index = index; 2150 } 2151 2152 static inline void 2153 sitd_link(struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd) 2154 { 2155 /* note: sitd ordering could matter (CSPLIT then SSPLIT) */ 2156 sitd->sitd_next = ehci->pshadow[frame]; 2157 sitd->hw_next = ehci->periodic[frame]; 2158 ehci->pshadow[frame].sitd = sitd; 2159 sitd->frame = frame; 2160 wmb(); 2161 ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD); 2162 } 2163 2164 /* fit urb's sitds into the selected schedule slot; activate as needed */ 2165 static void sitd_link_urb( 2166 struct ehci_hcd *ehci, 2167 struct urb *urb, 2168 unsigned mod, 2169 struct ehci_iso_stream *stream 2170 ) 2171 { 2172 int packet; 2173 unsigned next_uframe; 2174 struct ehci_iso_sched *sched = urb->hcpriv; 2175 struct ehci_sitd *sitd; 2176 2177 next_uframe = stream->next_uframe; 2178 2179 if (list_empty(&stream->td_list)) 2180 /* usbfs ignores TT bandwidth */ 2181 ehci_to_hcd(ehci)->self.bandwidth_allocated 2182 += stream->bandwidth; 2183 2184 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 2185 if (ehci->amd_pll_fix == 1) 2186 usb_amd_quirk_pll_disable(); 2187 } 2188 2189 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++; 2190 2191 /* fill sITDs frame by frame */ 2192 for (packet = sched->first_packet, sitd = NULL; 2193 packet < urb->number_of_packets; 2194 packet++) { 2195 2196 /* ASSERT: we have all necessary sitds */ 2197 BUG_ON(list_empty(&sched->td_list)); 2198 2199 /* ASSERT: no itds for this endpoint in this frame */ 2200 2201 sitd = list_entry(sched->td_list.next, 2202 struct ehci_sitd, sitd_list); 2203 list_move_tail(&sitd->sitd_list, &stream->td_list); 2204 sitd->stream = stream; 2205 sitd->urb = urb; 2206 2207 sitd_patch(ehci, stream, sitd, sched, packet); 2208 sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1), 2209 sitd); 2210 2211 next_uframe += stream->uperiod; 2212 } 2213 stream->next_uframe = next_uframe & (mod - 1); 2214 2215 /* don't need that schedule data any more */ 2216 iso_sched_free(stream, sched); 2217 urb->hcpriv = stream; 2218 2219 ++ehci->isoc_count; 2220 enable_periodic(ehci); 2221 } 2222 2223 /*-------------------------------------------------------------------------*/ 2224 2225 #define SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \ 2226 | SITD_STS_XACT | SITD_STS_MMF) 2227 2228 /* Process and recycle a completed SITD. Return true iff its urb completed, 2229 * and hence its completion callback probably added things to the hardware 2230 * schedule. 2231 * 2232 * Note that we carefully avoid recycling this descriptor until after any 2233 * completion callback runs, so that it won't be reused quickly. That is, 2234 * assuming (a) no more than two urbs per frame on this endpoint, and also 2235 * (b) only this endpoint's completions submit URBs. It seems some silicon 2236 * corrupts things if you reuse completed descriptors very quickly... 2237 */ 2238 static bool sitd_complete(struct ehci_hcd *ehci, struct ehci_sitd *sitd) 2239 { 2240 struct urb *urb = sitd->urb; 2241 struct usb_iso_packet_descriptor *desc; 2242 u32 t; 2243 int urb_index; 2244 struct ehci_iso_stream *stream = sitd->stream; 2245 struct usb_device *dev; 2246 bool retval = false; 2247 2248 urb_index = sitd->index; 2249 desc = &urb->iso_frame_desc[urb_index]; 2250 t = hc32_to_cpup(ehci, &sitd->hw_results); 2251 2252 /* report transfer status */ 2253 if (unlikely(t & SITD_ERRS)) { 2254 urb->error_count++; 2255 if (t & SITD_STS_DBE) 2256 desc->status = usb_pipein(urb->pipe) 2257 ? -ENOSR /* hc couldn't read */ 2258 : -ECOMM; /* hc couldn't write */ 2259 else if (t & SITD_STS_BABBLE) 2260 desc->status = -EOVERFLOW; 2261 else /* XACT, MMF, etc */ 2262 desc->status = -EPROTO; 2263 } else if (unlikely(t & SITD_STS_ACTIVE)) { 2264 /* URB was too late */ 2265 urb->error_count++; 2266 } else { 2267 desc->status = 0; 2268 desc->actual_length = desc->length - SITD_LENGTH(t); 2269 urb->actual_length += desc->actual_length; 2270 } 2271 2272 /* handle completion now? */ 2273 if ((urb_index + 1) != urb->number_of_packets) 2274 goto done; 2275 2276 /* 2277 * ASSERT: it's really the last sitd for this urb 2278 * list_for_each_entry (sitd, &stream->td_list, sitd_list) 2279 * BUG_ON(sitd->urb == urb); 2280 */ 2281 2282 /* give urb back to the driver; completion often (re)submits */ 2283 dev = urb->dev; 2284 ehci_urb_done(ehci, urb, 0); 2285 retval = true; 2286 urb = NULL; 2287 2288 --ehci->isoc_count; 2289 disable_periodic(ehci); 2290 2291 ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--; 2292 if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) { 2293 if (ehci->amd_pll_fix == 1) 2294 usb_amd_quirk_pll_enable(); 2295 } 2296 2297 if (list_is_singular(&stream->td_list)) 2298 ehci_to_hcd(ehci)->self.bandwidth_allocated 2299 -= stream->bandwidth; 2300 2301 done: 2302 sitd->urb = NULL; 2303 2304 /* Add to the end of the free list for later reuse */ 2305 list_move_tail(&sitd->sitd_list, &stream->free_list); 2306 2307 /* Recycle the siTDs when the pipeline is empty (ep no longer in use) */ 2308 if (list_empty(&stream->td_list)) { 2309 list_splice_tail_init(&stream->free_list, 2310 &ehci->cached_sitd_list); 2311 start_free_itds(ehci); 2312 } 2313 2314 return retval; 2315 } 2316 2317 2318 static int sitd_submit(struct ehci_hcd *ehci, struct urb *urb, 2319 gfp_t mem_flags) 2320 { 2321 int status = -EINVAL; 2322 unsigned long flags; 2323 struct ehci_iso_stream *stream; 2324 2325 /* Get iso_stream head */ 2326 stream = iso_stream_find(ehci, urb); 2327 if (stream == NULL) { 2328 ehci_dbg(ehci, "can't get iso stream\n"); 2329 return -ENOMEM; 2330 } 2331 if (urb->interval != stream->ps.period) { 2332 ehci_dbg(ehci, "can't change iso interval %d --> %d\n", 2333 stream->ps.period, urb->interval); 2334 goto done; 2335 } 2336 2337 #ifdef EHCI_URB_TRACE 2338 ehci_dbg(ehci, 2339 "submit %p dev%s ep%d%s-iso len %d\n", 2340 urb, urb->dev->devpath, 2341 usb_pipeendpoint(urb->pipe), 2342 usb_pipein(urb->pipe) ? "in" : "out", 2343 urb->transfer_buffer_length); 2344 #endif 2345 2346 /* allocate SITDs */ 2347 status = sitd_urb_transaction(stream, ehci, urb, mem_flags); 2348 if (status < 0) { 2349 ehci_dbg(ehci, "can't init sitds\n"); 2350 goto done; 2351 } 2352 2353 /* schedule ... need to lock */ 2354 spin_lock_irqsave(&ehci->lock, flags); 2355 if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) { 2356 status = -ESHUTDOWN; 2357 goto done_not_linked; 2358 } 2359 status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb); 2360 if (unlikely(status)) 2361 goto done_not_linked; 2362 status = iso_stream_schedule(ehci, urb, stream); 2363 if (likely(status == 0)) { 2364 sitd_link_urb(ehci, urb, ehci->periodic_size << 3, stream); 2365 } else if (status > 0) { 2366 status = 0; 2367 ehci_urb_done(ehci, urb, 0); 2368 } else { 2369 usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb); 2370 } 2371 done_not_linked: 2372 spin_unlock_irqrestore(&ehci->lock, flags); 2373 done: 2374 return status; 2375 } 2376 2377 /*-------------------------------------------------------------------------*/ 2378 2379 static void scan_isoc(struct ehci_hcd *ehci) 2380 { 2381 unsigned uf, now_frame, frame; 2382 unsigned fmask = ehci->periodic_size - 1; 2383 bool modified, live; 2384 union ehci_shadow q, *q_p; 2385 __hc32 type, *hw_p; 2386 2387 /* 2388 * When running, scan from last scan point up to "now" 2389 * else clean up by scanning everything that's left. 2390 * Touches as few pages as possible: cache-friendly. 2391 */ 2392 if (ehci->rh_state >= EHCI_RH_RUNNING) { 2393 uf = ehci_read_frame_index(ehci); 2394 now_frame = (uf >> 3) & fmask; 2395 live = true; 2396 } else { 2397 now_frame = (ehci->last_iso_frame - 1) & fmask; 2398 live = false; 2399 } 2400 ehci->now_frame = now_frame; 2401 2402 frame = ehci->last_iso_frame; 2403 2404 restart: 2405 /* Scan each element in frame's queue for completions */ 2406 q_p = &ehci->pshadow[frame]; 2407 hw_p = &ehci->periodic[frame]; 2408 q.ptr = q_p->ptr; 2409 type = Q_NEXT_TYPE(ehci, *hw_p); 2410 modified = false; 2411 2412 while (q.ptr != NULL) { 2413 switch (hc32_to_cpu(ehci, type)) { 2414 case Q_TYPE_ITD: 2415 /* 2416 * If this ITD is still active, leave it for 2417 * later processing ... check the next entry. 2418 * No need to check for activity unless the 2419 * frame is current. 2420 */ 2421 if (frame == now_frame && live) { 2422 rmb(); 2423 for (uf = 0; uf < 8; uf++) { 2424 if (q.itd->hw_transaction[uf] & 2425 ITD_ACTIVE(ehci)) 2426 break; 2427 } 2428 if (uf < 8) { 2429 q_p = &q.itd->itd_next; 2430 hw_p = &q.itd->hw_next; 2431 type = Q_NEXT_TYPE(ehci, 2432 q.itd->hw_next); 2433 q = *q_p; 2434 break; 2435 } 2436 } 2437 2438 /* 2439 * Take finished ITDs out of the schedule 2440 * and process them: recycle, maybe report 2441 * URB completion. HC won't cache the 2442 * pointer for much longer, if at all. 2443 */ 2444 *q_p = q.itd->itd_next; 2445 if (!ehci->use_dummy_qh || 2446 q.itd->hw_next != EHCI_LIST_END(ehci)) 2447 *hw_p = q.itd->hw_next; 2448 else 2449 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma); 2450 type = Q_NEXT_TYPE(ehci, q.itd->hw_next); 2451 wmb(); 2452 modified = itd_complete(ehci, q.itd); 2453 q = *q_p; 2454 break; 2455 case Q_TYPE_SITD: 2456 /* 2457 * If this SITD is still active, leave it for 2458 * later processing ... check the next entry. 2459 * No need to check for activity unless the 2460 * frame is current. 2461 */ 2462 if (((frame == now_frame) || 2463 (((frame + 1) & fmask) == now_frame)) 2464 && live 2465 && (q.sitd->hw_results & SITD_ACTIVE(ehci))) { 2466 2467 q_p = &q.sitd->sitd_next; 2468 hw_p = &q.sitd->hw_next; 2469 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next); 2470 q = *q_p; 2471 break; 2472 } 2473 2474 /* 2475 * Take finished SITDs out of the schedule 2476 * and process them: recycle, maybe report 2477 * URB completion. 2478 */ 2479 *q_p = q.sitd->sitd_next; 2480 if (!ehci->use_dummy_qh || 2481 q.sitd->hw_next != EHCI_LIST_END(ehci)) 2482 *hw_p = q.sitd->hw_next; 2483 else 2484 *hw_p = cpu_to_hc32(ehci, ehci->dummy->qh_dma); 2485 type = Q_NEXT_TYPE(ehci, q.sitd->hw_next); 2486 wmb(); 2487 modified = sitd_complete(ehci, q.sitd); 2488 q = *q_p; 2489 break; 2490 default: 2491 ehci_dbg(ehci, "corrupt type %d frame %d shadow %p\n", 2492 type, frame, q.ptr); 2493 /* BUG(); */ 2494 /* FALL THROUGH */ 2495 case Q_TYPE_QH: 2496 case Q_TYPE_FSTN: 2497 /* End of the iTDs and siTDs */ 2498 q.ptr = NULL; 2499 break; 2500 } 2501 2502 /* Assume completion callbacks modify the queue */ 2503 if (unlikely(modified && ehci->isoc_count > 0)) 2504 goto restart; 2505 } 2506 2507 /* Stop when we have reached the current frame */ 2508 if (frame == now_frame) 2509 return; 2510 2511 /* The last frame may still have active siTDs */ 2512 ehci->last_iso_frame = frame; 2513 frame = (frame + 1) & fmask; 2514 2515 goto restart; 2516 } 2517