1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * xHCI host controller driver 4 * 5 * Copyright (C) 2008 Intel Corp. 6 * 7 * Author: Sarah Sharp 8 * Some code borrowed from the Linux EHCI driver. 9 */ 10 11 /* 12 * Ring initialization rules: 13 * 1. Each segment is initialized to zero, except for link TRBs. 14 * 2. Ring cycle state = 0. This represents Producer Cycle State (PCS) or 15 * Consumer Cycle State (CCS), depending on ring function. 16 * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment. 17 * 18 * Ring behavior rules: 19 * 1. A ring is empty if enqueue == dequeue. This means there will always be at 20 * least one free TRB in the ring. This is useful if you want to turn that 21 * into a link TRB and expand the ring. 22 * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a 23 * link TRB, then load the pointer with the address in the link TRB. If the 24 * link TRB had its toggle bit set, you may need to update the ring cycle 25 * state (see cycle bit rules). You may have to do this multiple times 26 * until you reach a non-link TRB. 27 * 3. A ring is full if enqueue++ (for the definition of increment above) 28 * equals the dequeue pointer. 29 * 30 * Cycle bit rules: 31 * 1. When a consumer increments a dequeue pointer and encounters a toggle bit 32 * in a link TRB, it must toggle the ring cycle state. 33 * 2. When a producer increments an enqueue pointer and encounters a toggle bit 34 * in a link TRB, it must toggle the ring cycle state. 35 * 36 * Producer rules: 37 * 1. Check if ring is full before you enqueue. 38 * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing. 39 * Update enqueue pointer between each write (which may update the ring 40 * cycle state). 41 * 3. Notify consumer. If SW is producer, it rings the doorbell for command 42 * and endpoint rings. If HC is the producer for the event ring, 43 * and it generates an interrupt according to interrupt modulation rules. 44 * 45 * Consumer rules: 46 * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state, 47 * the TRB is owned by the consumer. 48 * 2. Update dequeue pointer (which may update the ring cycle state) and 49 * continue processing TRBs until you reach a TRB which is not owned by you. 50 * 3. Notify the producer. SW is the consumer for the event ring, and it 51 * updates event ring dequeue pointer. HC is the consumer for the command and 52 * endpoint rings; it generates events on the event ring for these. 53 */ 54 55 #include <linux/jiffies.h> 56 #include <linux/scatterlist.h> 57 #include <linux/slab.h> 58 #include <linux/dma-mapping.h> 59 #include "xhci.h" 60 #include "xhci-trace.h" 61 62 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, 63 u32 field1, u32 field2, 64 u32 field3, u32 field4, bool command_must_succeed); 65 66 /* 67 * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA 68 * address of the TRB. 69 */ 70 dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg, 71 union xhci_trb *trb) 72 { 73 unsigned long segment_offset; 74 75 if (!seg || !trb || trb < seg->trbs) 76 return 0; 77 /* offset in TRBs */ 78 segment_offset = trb - seg->trbs; 79 if (segment_offset >= TRBS_PER_SEGMENT) 80 return 0; 81 return seg->dma + (segment_offset * sizeof(*trb)); 82 } 83 84 static bool trb_is_noop(union xhci_trb *trb) 85 { 86 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]); 87 } 88 89 static bool trb_is_link(union xhci_trb *trb) 90 { 91 return TRB_TYPE_LINK_LE32(trb->link.control); 92 } 93 94 static bool last_trb_on_seg(struct xhci_segment *seg, union xhci_trb *trb) 95 { 96 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1]; 97 } 98 99 static bool last_trb_on_ring(struct xhci_ring *ring, 100 struct xhci_segment *seg, union xhci_trb *trb) 101 { 102 return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg); 103 } 104 105 static bool link_trb_toggles_cycle(union xhci_trb *trb) 106 { 107 return le32_to_cpu(trb->link.control) & LINK_TOGGLE; 108 } 109 110 static bool last_td_in_urb(struct xhci_td *td) 111 { 112 struct urb_priv *urb_priv = td->urb->hcpriv; 113 114 return urb_priv->num_tds_done == urb_priv->num_tds; 115 } 116 117 static void inc_td_cnt(struct urb *urb) 118 { 119 struct urb_priv *urb_priv = urb->hcpriv; 120 121 urb_priv->num_tds_done++; 122 } 123 124 static void trb_to_noop(union xhci_trb *trb, u32 noop_type) 125 { 126 if (trb_is_link(trb)) { 127 /* unchain chained link TRBs */ 128 trb->link.control &= cpu_to_le32(~TRB_CHAIN); 129 } else { 130 trb->generic.field[0] = 0; 131 trb->generic.field[1] = 0; 132 trb->generic.field[2] = 0; 133 /* Preserve only the cycle bit of this TRB */ 134 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE); 135 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type)); 136 } 137 } 138 139 /* Updates trb to point to the next TRB in the ring, and updates seg if the next 140 * TRB is in a new segment. This does not skip over link TRBs, and it does not 141 * effect the ring dequeue or enqueue pointers. 142 */ 143 static void next_trb(struct xhci_hcd *xhci, 144 struct xhci_ring *ring, 145 struct xhci_segment **seg, 146 union xhci_trb **trb) 147 { 148 if (trb_is_link(*trb)) { 149 *seg = (*seg)->next; 150 *trb = ((*seg)->trbs); 151 } else { 152 (*trb)++; 153 } 154 } 155 156 /* 157 * See Cycle bit rules. SW is the consumer for the event ring only. 158 */ 159 void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring) 160 { 161 unsigned int link_trb_count = 0; 162 163 /* event ring doesn't have link trbs, check for last trb */ 164 if (ring->type == TYPE_EVENT) { 165 if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) { 166 ring->dequeue++; 167 goto out; 168 } 169 if (last_trb_on_ring(ring, ring->deq_seg, ring->dequeue)) 170 ring->cycle_state ^= 1; 171 ring->deq_seg = ring->deq_seg->next; 172 ring->dequeue = ring->deq_seg->trbs; 173 goto out; 174 } 175 176 /* All other rings have link trbs */ 177 if (!trb_is_link(ring->dequeue)) { 178 if (last_trb_on_seg(ring->deq_seg, ring->dequeue)) 179 xhci_warn(xhci, "Missing link TRB at end of segment\n"); 180 else 181 ring->dequeue++; 182 } 183 184 while (trb_is_link(ring->dequeue)) { 185 ring->deq_seg = ring->deq_seg->next; 186 ring->dequeue = ring->deq_seg->trbs; 187 188 if (link_trb_count++ > ring->num_segs) { 189 xhci_warn(xhci, "Ring is an endless link TRB loop\n"); 190 break; 191 } 192 } 193 out: 194 trace_xhci_inc_deq(ring); 195 196 return; 197 } 198 199 /* 200 * See Cycle bit rules. SW is the consumer for the event ring only. 201 * 202 * If we've just enqueued a TRB that is in the middle of a TD (meaning the 203 * chain bit is set), then set the chain bit in all the following link TRBs. 204 * If we've enqueued the last TRB in a TD, make sure the following link TRBs 205 * have their chain bit cleared (so that each Link TRB is a separate TD). 206 * 207 * Section 6.4.4.1 of the 0.95 spec says link TRBs cannot have the chain bit 208 * set, but other sections talk about dealing with the chain bit set. This was 209 * fixed in the 0.96 specification errata, but we have to assume that all 0.95 210 * xHCI hardware can't handle the chain bit being cleared on a link TRB. 211 * 212 * @more_trbs_coming: Will you enqueue more TRBs before calling 213 * prepare_transfer()? 214 */ 215 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring, 216 bool more_trbs_coming) 217 { 218 u32 chain; 219 union xhci_trb *next; 220 unsigned int link_trb_count = 0; 221 222 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN; 223 224 if (last_trb_on_seg(ring->enq_seg, ring->enqueue)) { 225 xhci_err(xhci, "Tried to move enqueue past ring segment\n"); 226 return; 227 } 228 229 next = ++(ring->enqueue); 230 231 /* Update the dequeue pointer further if that was a link TRB */ 232 while (trb_is_link(next)) { 233 234 /* 235 * If the caller doesn't plan on enqueueing more TDs before 236 * ringing the doorbell, then we don't want to give the link TRB 237 * to the hardware just yet. We'll give the link TRB back in 238 * prepare_ring() just before we enqueue the TD at the top of 239 * the ring. 240 */ 241 if (!chain && !more_trbs_coming) 242 break; 243 244 /* If we're not dealing with 0.95 hardware or isoc rings on 245 * AMD 0.96 host, carry over the chain bit of the previous TRB 246 * (which may mean the chain bit is cleared). 247 */ 248 if (!(ring->type == TYPE_ISOC && 249 (xhci->quirks & XHCI_AMD_0x96_HOST)) && 250 !xhci_link_trb_quirk(xhci)) { 251 next->link.control &= cpu_to_le32(~TRB_CHAIN); 252 next->link.control |= cpu_to_le32(chain); 253 } 254 /* Give this link TRB to the hardware */ 255 wmb(); 256 next->link.control ^= cpu_to_le32(TRB_CYCLE); 257 258 /* Toggle the cycle bit after the last ring segment. */ 259 if (link_trb_toggles_cycle(next)) 260 ring->cycle_state ^= 1; 261 262 ring->enq_seg = ring->enq_seg->next; 263 ring->enqueue = ring->enq_seg->trbs; 264 next = ring->enqueue; 265 266 if (link_trb_count++ > ring->num_segs) { 267 xhci_warn(xhci, "%s: Ring link TRB loop\n", __func__); 268 break; 269 } 270 } 271 272 trace_xhci_inc_enq(ring); 273 } 274 275 /* 276 * Return number of free normal TRBs from enqueue to dequeue pointer on ring. 277 * Not counting an assumed link TRB at end of each TRBS_PER_SEGMENT sized segment. 278 * Only for transfer and command rings where driver is the producer, not for 279 * event rings. 280 */ 281 static unsigned int xhci_num_trbs_free(struct xhci_hcd *xhci, struct xhci_ring *ring) 282 { 283 struct xhci_segment *enq_seg = ring->enq_seg; 284 union xhci_trb *enq = ring->enqueue; 285 union xhci_trb *last_on_seg; 286 unsigned int free = 0; 287 int i = 0; 288 289 /* Ring might be empty even if enq != deq if enq is left on a link trb */ 290 if (trb_is_link(enq)) { 291 enq_seg = enq_seg->next; 292 enq = enq_seg->trbs; 293 } 294 295 /* Empty ring, common case, don't walk the segments */ 296 if (enq == ring->dequeue) 297 return ring->num_segs * (TRBS_PER_SEGMENT - 1); 298 299 do { 300 if (ring->deq_seg == enq_seg && ring->dequeue >= enq) 301 return free + (ring->dequeue - enq); 302 last_on_seg = &enq_seg->trbs[TRBS_PER_SEGMENT - 1]; 303 free += last_on_seg - enq; 304 enq_seg = enq_seg->next; 305 enq = enq_seg->trbs; 306 } while (i++ <= ring->num_segs); 307 308 return free; 309 } 310 311 /* 312 * Check to see if there's room to enqueue num_trbs on the ring and make sure 313 * enqueue pointer will not advance into dequeue segment. See rules above. 314 * return number of new segments needed to ensure this. 315 */ 316 317 static unsigned int xhci_ring_expansion_needed(struct xhci_hcd *xhci, struct xhci_ring *ring, 318 unsigned int num_trbs) 319 { 320 struct xhci_segment *seg; 321 int trbs_past_seg; 322 int enq_used; 323 int new_segs; 324 325 enq_used = ring->enqueue - ring->enq_seg->trbs; 326 327 /* how many trbs will be queued past the enqueue segment? */ 328 trbs_past_seg = enq_used + num_trbs - (TRBS_PER_SEGMENT - 1); 329 330 /* 331 * Consider expanding the ring already if num_trbs fills the current 332 * segment (i.e. trbs_past_seg == 0), not only when num_trbs goes into 333 * the next segment. Avoids confusing full ring with special empty ring 334 * case below 335 */ 336 if (trbs_past_seg < 0) 337 return 0; 338 339 /* Empty ring special case, enqueue stuck on link trb while dequeue advanced */ 340 if (trb_is_link(ring->enqueue) && ring->enq_seg->next->trbs == ring->dequeue) 341 return 0; 342 343 new_segs = 1 + (trbs_past_seg / (TRBS_PER_SEGMENT - 1)); 344 seg = ring->enq_seg; 345 346 while (new_segs > 0) { 347 seg = seg->next; 348 if (seg == ring->deq_seg) { 349 xhci_dbg(xhci, "Ring expansion by %d segments needed\n", 350 new_segs); 351 xhci_dbg(xhci, "Adding %d trbs moves enq %d trbs into deq seg\n", 352 num_trbs, trbs_past_seg % TRBS_PER_SEGMENT); 353 return new_segs; 354 } 355 new_segs--; 356 } 357 358 return 0; 359 } 360 361 /* Ring the host controller doorbell after placing a command on the ring */ 362 void xhci_ring_cmd_db(struct xhci_hcd *xhci) 363 { 364 if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING)) 365 return; 366 367 xhci_dbg(xhci, "// Ding dong!\n"); 368 369 trace_xhci_ring_host_doorbell(0, DB_VALUE_HOST); 370 371 writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]); 372 /* Flush PCI posted writes */ 373 readl(&xhci->dba->doorbell[0]); 374 } 375 376 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci) 377 { 378 return mod_delayed_work(system_wq, &xhci->cmd_timer, 379 msecs_to_jiffies(xhci->current_cmd->timeout_ms)); 380 } 381 382 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci) 383 { 384 return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command, 385 cmd_list); 386 } 387 388 /* 389 * Turn all commands on command ring with status set to "aborted" to no-op trbs. 390 * If there are other commands waiting then restart the ring and kick the timer. 391 * This must be called with command ring stopped and xhci->lock held. 392 */ 393 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci, 394 struct xhci_command *cur_cmd) 395 { 396 struct xhci_command *i_cmd; 397 398 /* Turn all aborted commands in list to no-ops, then restart */ 399 list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) { 400 401 if (i_cmd->status != COMP_COMMAND_ABORTED) 402 continue; 403 404 i_cmd->status = COMP_COMMAND_RING_STOPPED; 405 406 xhci_dbg(xhci, "Turn aborted command %p to no-op\n", 407 i_cmd->command_trb); 408 409 trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP); 410 411 /* 412 * caller waiting for completion is called when command 413 * completion event is received for these no-op commands 414 */ 415 } 416 417 xhci->cmd_ring_state = CMD_RING_STATE_RUNNING; 418 419 /* ring command ring doorbell to restart the command ring */ 420 if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) && 421 !(xhci->xhc_state & XHCI_STATE_DYING)) { 422 xhci->current_cmd = cur_cmd; 423 xhci_mod_cmd_timer(xhci); 424 xhci_ring_cmd_db(xhci); 425 } 426 } 427 428 /* Must be called with xhci->lock held, releases and aquires lock back */ 429 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags) 430 { 431 struct xhci_segment *new_seg = xhci->cmd_ring->deq_seg; 432 union xhci_trb *new_deq = xhci->cmd_ring->dequeue; 433 u64 crcr; 434 int ret; 435 436 xhci_dbg(xhci, "Abort command ring\n"); 437 438 reinit_completion(&xhci->cmd_ring_stop_completion); 439 440 /* 441 * The control bits like command stop, abort are located in lower 442 * dword of the command ring control register. 443 * Some controllers require all 64 bits to be written to abort the ring. 444 * Make sure the upper dword is valid, pointing to the next command, 445 * avoiding corrupting the command ring pointer in case the command ring 446 * is stopped by the time the upper dword is written. 447 */ 448 next_trb(xhci, NULL, &new_seg, &new_deq); 449 if (trb_is_link(new_deq)) 450 next_trb(xhci, NULL, &new_seg, &new_deq); 451 452 crcr = xhci_trb_virt_to_dma(new_seg, new_deq); 453 xhci_write_64(xhci, crcr | CMD_RING_ABORT, &xhci->op_regs->cmd_ring); 454 455 /* Section 4.6.1.2 of xHCI 1.0 spec says software should also time the 456 * completion of the Command Abort operation. If CRR is not negated in 5 457 * seconds then driver handles it as if host died (-ENODEV). 458 * In the future we should distinguish between -ENODEV and -ETIMEDOUT 459 * and try to recover a -ETIMEDOUT with a host controller reset. 460 */ 461 ret = xhci_handshake(&xhci->op_regs->cmd_ring, 462 CMD_RING_RUNNING, 0, 5 * 1000 * 1000); 463 if (ret < 0) { 464 xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret); 465 xhci_halt(xhci); 466 xhci_hc_died(xhci); 467 return ret; 468 } 469 /* 470 * Writing the CMD_RING_ABORT bit should cause a cmd completion event, 471 * however on some host hw the CMD_RING_RUNNING bit is correctly cleared 472 * but the completion event in never sent. Wait 2 secs (arbitrary 473 * number) to handle those cases after negation of CMD_RING_RUNNING. 474 */ 475 spin_unlock_irqrestore(&xhci->lock, flags); 476 ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion, 477 msecs_to_jiffies(2000)); 478 spin_lock_irqsave(&xhci->lock, flags); 479 if (!ret) { 480 xhci_dbg(xhci, "No stop event for abort, ring start fail?\n"); 481 xhci_cleanup_command_queue(xhci); 482 } else { 483 xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci)); 484 } 485 return 0; 486 } 487 488 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci, 489 unsigned int slot_id, 490 unsigned int ep_index, 491 unsigned int stream_id) 492 { 493 __le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id]; 494 struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index]; 495 unsigned int ep_state = ep->ep_state; 496 497 /* Don't ring the doorbell for this endpoint if there are pending 498 * cancellations because we don't want to interrupt processing. 499 * We don't want to restart any stream rings if there's a set dequeue 500 * pointer command pending because the device can choose to start any 501 * stream once the endpoint is on the HW schedule. 502 */ 503 if ((ep_state & EP_STOP_CMD_PENDING) || (ep_state & SET_DEQ_PENDING) || 504 (ep_state & EP_HALTED) || (ep_state & EP_CLEARING_TT)) 505 return; 506 507 trace_xhci_ring_ep_doorbell(slot_id, DB_VALUE(ep_index, stream_id)); 508 509 writel(DB_VALUE(ep_index, stream_id), db_addr); 510 /* flush the write */ 511 readl(db_addr); 512 } 513 514 /* Ring the doorbell for any rings with pending URBs */ 515 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci, 516 unsigned int slot_id, 517 unsigned int ep_index) 518 { 519 unsigned int stream_id; 520 struct xhci_virt_ep *ep; 521 522 ep = &xhci->devs[slot_id]->eps[ep_index]; 523 524 /* A ring has pending URBs if its TD list is not empty */ 525 if (!(ep->ep_state & EP_HAS_STREAMS)) { 526 if (ep->ring && !(list_empty(&ep->ring->td_list))) 527 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0); 528 return; 529 } 530 531 for (stream_id = 1; stream_id < ep->stream_info->num_streams; 532 stream_id++) { 533 struct xhci_stream_info *stream_info = ep->stream_info; 534 if (!list_empty(&stream_info->stream_rings[stream_id]->td_list)) 535 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 536 stream_id); 537 } 538 } 539 540 void xhci_ring_doorbell_for_active_rings(struct xhci_hcd *xhci, 541 unsigned int slot_id, 542 unsigned int ep_index) 543 { 544 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); 545 } 546 547 static struct xhci_virt_ep *xhci_get_virt_ep(struct xhci_hcd *xhci, 548 unsigned int slot_id, 549 unsigned int ep_index) 550 { 551 if (slot_id == 0 || slot_id >= MAX_HC_SLOTS) { 552 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); 553 return NULL; 554 } 555 if (ep_index >= EP_CTX_PER_DEV) { 556 xhci_warn(xhci, "Invalid endpoint index %u\n", ep_index); 557 return NULL; 558 } 559 if (!xhci->devs[slot_id]) { 560 xhci_warn(xhci, "No xhci virt device for slot_id %u\n", slot_id); 561 return NULL; 562 } 563 564 return &xhci->devs[slot_id]->eps[ep_index]; 565 } 566 567 static struct xhci_ring *xhci_virt_ep_to_ring(struct xhci_hcd *xhci, 568 struct xhci_virt_ep *ep, 569 unsigned int stream_id) 570 { 571 /* common case, no streams */ 572 if (!(ep->ep_state & EP_HAS_STREAMS)) 573 return ep->ring; 574 575 if (!ep->stream_info) 576 return NULL; 577 578 if (stream_id == 0 || stream_id >= ep->stream_info->num_streams) { 579 xhci_warn(xhci, "Invalid stream_id %u request for slot_id %u ep_index %u\n", 580 stream_id, ep->vdev->slot_id, ep->ep_index); 581 return NULL; 582 } 583 584 return ep->stream_info->stream_rings[stream_id]; 585 } 586 587 /* Get the right ring for the given slot_id, ep_index and stream_id. 588 * If the endpoint supports streams, boundary check the URB's stream ID. 589 * If the endpoint doesn't support streams, return the singular endpoint ring. 590 */ 591 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci, 592 unsigned int slot_id, unsigned int ep_index, 593 unsigned int stream_id) 594 { 595 struct xhci_virt_ep *ep; 596 597 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); 598 if (!ep) 599 return NULL; 600 601 return xhci_virt_ep_to_ring(xhci, ep, stream_id); 602 } 603 604 605 /* 606 * Get the hw dequeue pointer xHC stopped on, either directly from the 607 * endpoint context, or if streams are in use from the stream context. 608 * The returned hw_dequeue contains the lowest four bits with cycle state 609 * and possbile stream context type. 610 */ 611 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev, 612 unsigned int ep_index, unsigned int stream_id) 613 { 614 struct xhci_ep_ctx *ep_ctx; 615 struct xhci_stream_ctx *st_ctx; 616 struct xhci_virt_ep *ep; 617 618 ep = &vdev->eps[ep_index]; 619 620 if (ep->ep_state & EP_HAS_STREAMS) { 621 st_ctx = &ep->stream_info->stream_ctx_array[stream_id]; 622 return le64_to_cpu(st_ctx->stream_ring); 623 } 624 ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index); 625 return le64_to_cpu(ep_ctx->deq); 626 } 627 628 static int xhci_move_dequeue_past_td(struct xhci_hcd *xhci, 629 unsigned int slot_id, unsigned int ep_index, 630 unsigned int stream_id, struct xhci_td *td) 631 { 632 struct xhci_virt_device *dev = xhci->devs[slot_id]; 633 struct xhci_virt_ep *ep = &dev->eps[ep_index]; 634 struct xhci_ring *ep_ring; 635 struct xhci_command *cmd; 636 struct xhci_segment *new_seg; 637 union xhci_trb *new_deq; 638 int new_cycle; 639 dma_addr_t addr; 640 u64 hw_dequeue; 641 bool cycle_found = false; 642 bool td_last_trb_found = false; 643 u32 trb_sct = 0; 644 int ret; 645 646 ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id, 647 ep_index, stream_id); 648 if (!ep_ring) { 649 xhci_warn(xhci, "WARN can't find new dequeue, invalid stream ID %u\n", 650 stream_id); 651 return -ENODEV; 652 } 653 /* 654 * A cancelled TD can complete with a stall if HW cached the trb. 655 * In this case driver can't find td, but if the ring is empty we 656 * can move the dequeue pointer to the current enqueue position. 657 * We shouldn't hit this anymore as cached cancelled TRBs are given back 658 * after clearing the cache, but be on the safe side and keep it anyway 659 */ 660 if (!td) { 661 if (list_empty(&ep_ring->td_list)) { 662 new_seg = ep_ring->enq_seg; 663 new_deq = ep_ring->enqueue; 664 new_cycle = ep_ring->cycle_state; 665 xhci_dbg(xhci, "ep ring empty, Set new dequeue = enqueue"); 666 goto deq_found; 667 } else { 668 xhci_warn(xhci, "Can't find new dequeue state, missing td\n"); 669 return -EINVAL; 670 } 671 } 672 673 hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id); 674 new_seg = ep_ring->deq_seg; 675 new_deq = ep_ring->dequeue; 676 new_cycle = hw_dequeue & 0x1; 677 678 /* 679 * We want to find the pointer, segment and cycle state of the new trb 680 * (the one after current TD's last_trb). We know the cycle state at 681 * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are 682 * found. 683 */ 684 do { 685 if (!cycle_found && xhci_trb_virt_to_dma(new_seg, new_deq) 686 == (dma_addr_t)(hw_dequeue & ~0xf)) { 687 cycle_found = true; 688 if (td_last_trb_found) 689 break; 690 } 691 if (new_deq == td->last_trb) 692 td_last_trb_found = true; 693 694 if (cycle_found && trb_is_link(new_deq) && 695 link_trb_toggles_cycle(new_deq)) 696 new_cycle ^= 0x1; 697 698 next_trb(xhci, ep_ring, &new_seg, &new_deq); 699 700 /* Search wrapped around, bail out */ 701 if (new_deq == ep->ring->dequeue) { 702 xhci_err(xhci, "Error: Failed finding new dequeue state\n"); 703 return -EINVAL; 704 } 705 706 } while (!cycle_found || !td_last_trb_found); 707 708 deq_found: 709 710 /* Don't update the ring cycle state for the producer (us). */ 711 addr = xhci_trb_virt_to_dma(new_seg, new_deq); 712 if (addr == 0) { 713 xhci_warn(xhci, "Can't find dma of new dequeue ptr\n"); 714 xhci_warn(xhci, "deq seg = %p, deq ptr = %p\n", new_seg, new_deq); 715 return -EINVAL; 716 } 717 718 if ((ep->ep_state & SET_DEQ_PENDING)) { 719 xhci_warn(xhci, "Set TR Deq already pending, don't submit for 0x%pad\n", 720 &addr); 721 return -EBUSY; 722 } 723 724 /* This function gets called from contexts where it cannot sleep */ 725 cmd = xhci_alloc_command(xhci, false, GFP_ATOMIC); 726 if (!cmd) { 727 xhci_warn(xhci, "Can't alloc Set TR Deq cmd 0x%pad\n", &addr); 728 return -ENOMEM; 729 } 730 731 if (stream_id) 732 trb_sct = SCT_FOR_TRB(SCT_PRI_TR); 733 ret = queue_command(xhci, cmd, 734 lower_32_bits(addr) | trb_sct | new_cycle, 735 upper_32_bits(addr), 736 STREAM_ID_FOR_TRB(stream_id), SLOT_ID_FOR_TRB(slot_id) | 737 EP_ID_FOR_TRB(ep_index) | TRB_TYPE(TRB_SET_DEQ), false); 738 if (ret < 0) { 739 xhci_free_command(xhci, cmd); 740 return ret; 741 } 742 ep->queued_deq_seg = new_seg; 743 ep->queued_deq_ptr = new_deq; 744 745 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, 746 "Set TR Deq ptr 0x%llx, cycle %u\n", addr, new_cycle); 747 748 /* Stop the TD queueing code from ringing the doorbell until 749 * this command completes. The HC won't set the dequeue pointer 750 * if the ring is running, and ringing the doorbell starts the 751 * ring running. 752 */ 753 ep->ep_state |= SET_DEQ_PENDING; 754 xhci_ring_cmd_db(xhci); 755 return 0; 756 } 757 758 /* flip_cycle means flip the cycle bit of all but the first and last TRB. 759 * (The last TRB actually points to the ring enqueue pointer, which is not part 760 * of this TD.) This is used to remove partially enqueued isoc TDs from a ring. 761 */ 762 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, 763 struct xhci_td *td, bool flip_cycle) 764 { 765 struct xhci_segment *seg = td->start_seg; 766 union xhci_trb *trb = td->first_trb; 767 768 while (1) { 769 trb_to_noop(trb, TRB_TR_NOOP); 770 771 /* flip cycle if asked to */ 772 if (flip_cycle && trb != td->first_trb && trb != td->last_trb) 773 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE); 774 775 if (trb == td->last_trb) 776 break; 777 778 next_trb(xhci, ep_ring, &seg, &trb); 779 } 780 } 781 782 /* 783 * Must be called with xhci->lock held in interrupt context, 784 * releases and re-acquires xhci->lock 785 */ 786 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci, 787 struct xhci_td *cur_td, int status) 788 { 789 struct urb *urb = cur_td->urb; 790 struct urb_priv *urb_priv = urb->hcpriv; 791 struct usb_hcd *hcd = bus_to_hcd(urb->dev->bus); 792 793 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) { 794 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--; 795 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { 796 if (xhci->quirks & XHCI_AMD_PLL_FIX) 797 usb_amd_quirk_pll_enable(); 798 } 799 } 800 xhci_urb_free_priv(urb_priv); 801 usb_hcd_unlink_urb_from_ep(hcd, urb); 802 trace_xhci_urb_giveback(urb); 803 usb_hcd_giveback_urb(hcd, urb, status); 804 } 805 806 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci, 807 struct xhci_ring *ring, struct xhci_td *td) 808 { 809 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 810 struct xhci_segment *seg = td->bounce_seg; 811 struct urb *urb = td->urb; 812 size_t len; 813 814 if (!ring || !seg || !urb) 815 return; 816 817 if (usb_urb_dir_out(urb)) { 818 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, 819 DMA_TO_DEVICE); 820 return; 821 } 822 823 dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len, 824 DMA_FROM_DEVICE); 825 /* for in tranfers we need to copy the data from bounce to sg */ 826 if (urb->num_sgs) { 827 len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs, seg->bounce_buf, 828 seg->bounce_len, seg->bounce_offs); 829 if (len != seg->bounce_len) 830 xhci_warn(xhci, "WARN Wrong bounce buffer read length: %zu != %d\n", 831 len, seg->bounce_len); 832 } else { 833 memcpy(urb->transfer_buffer + seg->bounce_offs, seg->bounce_buf, 834 seg->bounce_len); 835 } 836 seg->bounce_len = 0; 837 seg->bounce_offs = 0; 838 } 839 840 static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td, 841 struct xhci_ring *ep_ring, int status) 842 { 843 struct urb *urb = NULL; 844 845 /* Clean up the endpoint's TD list */ 846 urb = td->urb; 847 848 /* if a bounce buffer was used to align this td then unmap it */ 849 xhci_unmap_td_bounce_buffer(xhci, ep_ring, td); 850 851 /* Do one last check of the actual transfer length. 852 * If the host controller said we transferred more data than the buffer 853 * length, urb->actual_length will be a very big number (since it's 854 * unsigned). Play it safe and say we didn't transfer anything. 855 */ 856 if (urb->actual_length > urb->transfer_buffer_length) { 857 xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n", 858 urb->transfer_buffer_length, urb->actual_length); 859 urb->actual_length = 0; 860 status = 0; 861 } 862 /* TD might be removed from td_list if we are giving back a cancelled URB */ 863 if (!list_empty(&td->td_list)) 864 list_del_init(&td->td_list); 865 /* Giving back a cancelled URB, or if a slated TD completed anyway */ 866 if (!list_empty(&td->cancelled_td_list)) 867 list_del_init(&td->cancelled_td_list); 868 869 inc_td_cnt(urb); 870 /* Giveback the urb when all the tds are completed */ 871 if (last_td_in_urb(td)) { 872 if ((urb->actual_length != urb->transfer_buffer_length && 873 (urb->transfer_flags & URB_SHORT_NOT_OK)) || 874 (status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc))) 875 xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n", 876 urb, urb->actual_length, 877 urb->transfer_buffer_length, status); 878 879 /* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */ 880 if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) 881 status = 0; 882 xhci_giveback_urb_in_irq(xhci, td, status); 883 } 884 885 return 0; 886 } 887 888 889 /* Complete the cancelled URBs we unlinked from td_list. */ 890 static void xhci_giveback_invalidated_tds(struct xhci_virt_ep *ep) 891 { 892 struct xhci_ring *ring; 893 struct xhci_td *td, *tmp_td; 894 895 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, 896 cancelled_td_list) { 897 898 ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); 899 900 if (td->cancel_status == TD_CLEARED) { 901 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", 902 __func__, td->urb); 903 xhci_td_cleanup(ep->xhci, td, ring, td->status); 904 } else { 905 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", 906 __func__, td->urb, td->cancel_status); 907 } 908 if (ep->xhci->xhc_state & XHCI_STATE_DYING) 909 return; 910 } 911 } 912 913 static int xhci_reset_halted_ep(struct xhci_hcd *xhci, unsigned int slot_id, 914 unsigned int ep_index, enum xhci_ep_reset_type reset_type) 915 { 916 struct xhci_command *command; 917 int ret = 0; 918 919 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); 920 if (!command) { 921 ret = -ENOMEM; 922 goto done; 923 } 924 925 xhci_dbg(xhci, "%s-reset ep %u, slot %u\n", 926 (reset_type == EP_HARD_RESET) ? "Hard" : "Soft", 927 ep_index, slot_id); 928 929 ret = xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type); 930 done: 931 if (ret) 932 xhci_err(xhci, "ERROR queuing reset endpoint for slot %d ep_index %d, %d\n", 933 slot_id, ep_index, ret); 934 return ret; 935 } 936 937 static int xhci_handle_halted_endpoint(struct xhci_hcd *xhci, 938 struct xhci_virt_ep *ep, 939 struct xhci_td *td, 940 enum xhci_ep_reset_type reset_type) 941 { 942 unsigned int slot_id = ep->vdev->slot_id; 943 int err; 944 945 /* 946 * Avoid resetting endpoint if link is inactive. Can cause host hang. 947 * Device will be reset soon to recover the link so don't do anything 948 */ 949 if (ep->vdev->flags & VDEV_PORT_ERROR) 950 return -ENODEV; 951 952 /* add td to cancelled list and let reset ep handler take care of it */ 953 if (reset_type == EP_HARD_RESET) { 954 ep->ep_state |= EP_HARD_CLEAR_TOGGLE; 955 if (td && list_empty(&td->cancelled_td_list)) { 956 list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list); 957 td->cancel_status = TD_HALTED; 958 } 959 } 960 961 if (ep->ep_state & EP_HALTED) { 962 xhci_dbg(xhci, "Reset ep command for ep_index %d already pending\n", 963 ep->ep_index); 964 return 0; 965 } 966 967 err = xhci_reset_halted_ep(xhci, slot_id, ep->ep_index, reset_type); 968 if (err) 969 return err; 970 971 ep->ep_state |= EP_HALTED; 972 973 xhci_ring_cmd_db(xhci); 974 975 return 0; 976 } 977 978 /* 979 * Fix up the ep ring first, so HW stops executing cancelled TDs. 980 * We have the xHCI lock, so nothing can modify this list until we drop it. 981 * We're also in the event handler, so we can't get re-interrupted if another 982 * Stop Endpoint command completes. 983 * 984 * only call this when ring is not in a running state 985 */ 986 987 static int xhci_invalidate_cancelled_tds(struct xhci_virt_ep *ep) 988 { 989 struct xhci_hcd *xhci; 990 struct xhci_td *td = NULL; 991 struct xhci_td *tmp_td = NULL; 992 struct xhci_td *cached_td = NULL; 993 struct xhci_ring *ring; 994 u64 hw_deq; 995 unsigned int slot_id = ep->vdev->slot_id; 996 int err; 997 998 /* 999 * This is not going to work if the hardware is changing its dequeue 1000 * pointers as we look at them. Completion handler will call us later. 1001 */ 1002 if (ep->ep_state & SET_DEQ_PENDING) 1003 return 0; 1004 1005 xhci = ep->xhci; 1006 1007 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) { 1008 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, 1009 "Removing canceled TD starting at 0x%llx (dma) in stream %u URB %p", 1010 (unsigned long long)xhci_trb_virt_to_dma( 1011 td->start_seg, td->first_trb), 1012 td->urb->stream_id, td->urb); 1013 list_del_init(&td->td_list); 1014 ring = xhci_urb_to_transfer_ring(xhci, td->urb); 1015 if (!ring) { 1016 xhci_warn(xhci, "WARN Cancelled URB %p has invalid stream ID %u.\n", 1017 td->urb, td->urb->stream_id); 1018 continue; 1019 } 1020 /* 1021 * If a ring stopped on the TD we need to cancel then we have to 1022 * move the xHC endpoint ring dequeue pointer past this TD. 1023 * Rings halted due to STALL may show hw_deq is past the stalled 1024 * TD, but still require a set TR Deq command to flush xHC cache. 1025 */ 1026 hw_deq = xhci_get_hw_deq(xhci, ep->vdev, ep->ep_index, 1027 td->urb->stream_id); 1028 hw_deq &= ~0xf; 1029 1030 if (td->cancel_status == TD_HALTED || 1031 trb_in_td(xhci, td->start_seg, td->first_trb, td->last_trb, hw_deq, false)) { 1032 switch (td->cancel_status) { 1033 case TD_CLEARED: /* TD is already no-op */ 1034 case TD_CLEARING_CACHE: /* set TR deq command already queued */ 1035 break; 1036 case TD_DIRTY: /* TD is cached, clear it */ 1037 case TD_HALTED: 1038 case TD_CLEARING_CACHE_DEFERRED: 1039 if (cached_td) { 1040 if (cached_td->urb->stream_id != td->urb->stream_id) { 1041 /* Multiple streams case, defer move dq */ 1042 xhci_dbg(xhci, 1043 "Move dq deferred: stream %u URB %p\n", 1044 td->urb->stream_id, td->urb); 1045 td->cancel_status = TD_CLEARING_CACHE_DEFERRED; 1046 break; 1047 } 1048 1049 /* Should never happen, but clear the TD if it does */ 1050 xhci_warn(xhci, 1051 "Found multiple active URBs %p and %p in stream %u?\n", 1052 td->urb, cached_td->urb, 1053 td->urb->stream_id); 1054 td_to_noop(xhci, ring, cached_td, false); 1055 cached_td->cancel_status = TD_CLEARED; 1056 } 1057 td_to_noop(xhci, ring, td, false); 1058 td->cancel_status = TD_CLEARING_CACHE; 1059 cached_td = td; 1060 break; 1061 } 1062 } else { 1063 td_to_noop(xhci, ring, td, false); 1064 td->cancel_status = TD_CLEARED; 1065 } 1066 } 1067 1068 /* If there's no need to move the dequeue pointer then we're done */ 1069 if (!cached_td) 1070 return 0; 1071 1072 err = xhci_move_dequeue_past_td(xhci, slot_id, ep->ep_index, 1073 cached_td->urb->stream_id, 1074 cached_td); 1075 if (err) { 1076 /* Failed to move past cached td, just set cached TDs to no-op */ 1077 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, cancelled_td_list) { 1078 /* 1079 * Deferred TDs need to have the deq pointer set after the above command 1080 * completes, so if that failed we just give up on all of them (and 1081 * complain loudly since this could cause issues due to caching). 1082 */ 1083 if (td->cancel_status != TD_CLEARING_CACHE && 1084 td->cancel_status != TD_CLEARING_CACHE_DEFERRED) 1085 continue; 1086 xhci_warn(xhci, "Failed to clear cancelled cached URB %p, mark clear anyway\n", 1087 td->urb); 1088 td_to_noop(xhci, ring, td, false); 1089 td->cancel_status = TD_CLEARED; 1090 } 1091 } 1092 return 0; 1093 } 1094 1095 /* 1096 * Erase queued TDs from transfer ring(s) and give back those the xHC didn't 1097 * stop on. If necessary, queue commands to move the xHC off cancelled TDs it 1098 * stopped on. Those will be given back later when the commands complete. 1099 * 1100 * Call under xhci->lock on a stopped endpoint. 1101 */ 1102 void xhci_process_cancelled_tds(struct xhci_virt_ep *ep) 1103 { 1104 xhci_invalidate_cancelled_tds(ep); 1105 xhci_giveback_invalidated_tds(ep); 1106 } 1107 1108 /* 1109 * Returns the TD the endpoint ring halted on. 1110 * Only call for non-running rings without streams. 1111 */ 1112 static struct xhci_td *find_halted_td(struct xhci_virt_ep *ep) 1113 { 1114 struct xhci_td *td; 1115 u64 hw_deq; 1116 1117 if (!list_empty(&ep->ring->td_list)) { /* Not streams compatible */ 1118 hw_deq = xhci_get_hw_deq(ep->xhci, ep->vdev, ep->ep_index, 0); 1119 hw_deq &= ~0xf; 1120 td = list_first_entry(&ep->ring->td_list, struct xhci_td, td_list); 1121 if (trb_in_td(ep->xhci, td->start_seg, td->first_trb, 1122 td->last_trb, hw_deq, false)) 1123 return td; 1124 } 1125 return NULL; 1126 } 1127 1128 /* 1129 * When we get a command completion for a Stop Endpoint Command, we need to 1130 * unlink any cancelled TDs from the ring. There are two ways to do that: 1131 * 1132 * 1. If the HW was in the middle of processing the TD that needs to be 1133 * cancelled, then we must move the ring's dequeue pointer past the last TRB 1134 * in the TD with a Set Dequeue Pointer Command. 1135 * 2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain 1136 * bit cleared) so that the HW will skip over them. 1137 */ 1138 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id, 1139 union xhci_trb *trb, u32 comp_code) 1140 { 1141 unsigned int ep_index; 1142 struct xhci_virt_ep *ep; 1143 struct xhci_ep_ctx *ep_ctx; 1144 struct xhci_td *td = NULL; 1145 enum xhci_ep_reset_type reset_type; 1146 struct xhci_command *command; 1147 int err; 1148 1149 if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) { 1150 if (!xhci->devs[slot_id]) 1151 xhci_warn(xhci, "Stop endpoint command completion for disabled slot %u\n", 1152 slot_id); 1153 return; 1154 } 1155 1156 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); 1157 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); 1158 if (!ep) 1159 return; 1160 1161 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); 1162 1163 trace_xhci_handle_cmd_stop_ep(ep_ctx); 1164 1165 if (comp_code == COMP_CONTEXT_STATE_ERROR) { 1166 /* 1167 * If stop endpoint command raced with a halting endpoint we need to 1168 * reset the host side endpoint first. 1169 * If the TD we halted on isn't cancelled the TD should be given back 1170 * with a proper error code, and the ring dequeue moved past the TD. 1171 * If streams case we can't find hw_deq, or the TD we halted on so do a 1172 * soft reset. 1173 * 1174 * Proper error code is unknown here, it would be -EPIPE if device side 1175 * of enadpoit halted (aka STALL), and -EPROTO if not (transaction error) 1176 * We use -EPROTO, if device is stalled it should return a stall error on 1177 * next transfer, which then will return -EPIPE, and device side stall is 1178 * noted and cleared by class driver. 1179 */ 1180 switch (GET_EP_CTX_STATE(ep_ctx)) { 1181 case EP_STATE_HALTED: 1182 xhci_dbg(xhci, "Stop ep completion raced with stall, reset ep\n"); 1183 if (ep->ep_state & EP_HAS_STREAMS) { 1184 reset_type = EP_SOFT_RESET; 1185 } else { 1186 reset_type = EP_HARD_RESET; 1187 td = find_halted_td(ep); 1188 if (td) 1189 td->status = -EPROTO; 1190 } 1191 /* reset ep, reset handler cleans up cancelled tds */ 1192 err = xhci_handle_halted_endpoint(xhci, ep, td, reset_type); 1193 if (err) 1194 break; 1195 ep->ep_state &= ~EP_STOP_CMD_PENDING; 1196 return; 1197 case EP_STATE_STOPPED: 1198 /* 1199 * Per xHCI 4.6.9, Stop Endpoint command on a Stopped 1200 * EP is a Context State Error, and EP stays Stopped. 1201 * 1202 * But maybe it failed on Halted, and somebody ran Reset 1203 * Endpoint later. EP state is now Stopped and EP_HALTED 1204 * still set because Reset EP handler will run after us. 1205 */ 1206 if (ep->ep_state & EP_HALTED) 1207 break; 1208 /* 1209 * On some HCs EP state remains Stopped for some tens of 1210 * us to a few ms or more after a doorbell ring, and any 1211 * new Stop Endpoint fails without aborting the restart. 1212 * This handler may run quickly enough to still see this 1213 * Stopped state, but it will soon change to Running. 1214 * 1215 * Assume this bug on unexpected Stop Endpoint failures. 1216 * Keep retrying until the EP starts and stops again, on 1217 * chips where this is known to help. Wait for 100ms. 1218 */ 1219 if (time_is_before_jiffies(ep->stop_time + msecs_to_jiffies(100))) 1220 break; 1221 fallthrough; 1222 case EP_STATE_RUNNING: 1223 /* Race, HW handled stop ep cmd before ep was running */ 1224 xhci_dbg(xhci, "Stop ep completion ctx error, ctx_state %d\n", 1225 GET_EP_CTX_STATE(ep_ctx)); 1226 1227 command = xhci_alloc_command(xhci, false, GFP_ATOMIC); 1228 if (!command) { 1229 ep->ep_state &= ~EP_STOP_CMD_PENDING; 1230 return; 1231 } 1232 xhci_queue_stop_endpoint(xhci, command, slot_id, ep_index, 0); 1233 xhci_ring_cmd_db(xhci); 1234 1235 return; 1236 default: 1237 break; 1238 } 1239 } 1240 1241 /* will queue a set TR deq if stopped on a cancelled, uncleared TD */ 1242 xhci_invalidate_cancelled_tds(ep); 1243 ep->ep_state &= ~EP_STOP_CMD_PENDING; 1244 1245 /* Otherwise ring the doorbell(s) to restart queued transfers */ 1246 xhci_giveback_invalidated_tds(ep); 1247 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); 1248 } 1249 1250 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring) 1251 { 1252 struct xhci_td *cur_td; 1253 struct xhci_td *tmp; 1254 1255 list_for_each_entry_safe(cur_td, tmp, &ring->td_list, td_list) { 1256 list_del_init(&cur_td->td_list); 1257 1258 if (!list_empty(&cur_td->cancelled_td_list)) 1259 list_del_init(&cur_td->cancelled_td_list); 1260 1261 xhci_unmap_td_bounce_buffer(xhci, ring, cur_td); 1262 1263 inc_td_cnt(cur_td->urb); 1264 if (last_td_in_urb(cur_td)) 1265 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); 1266 } 1267 } 1268 1269 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci, 1270 int slot_id, int ep_index) 1271 { 1272 struct xhci_td *cur_td; 1273 struct xhci_td *tmp; 1274 struct xhci_virt_ep *ep; 1275 struct xhci_ring *ring; 1276 1277 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); 1278 if (!ep) 1279 return; 1280 1281 if ((ep->ep_state & EP_HAS_STREAMS) || 1282 (ep->ep_state & EP_GETTING_NO_STREAMS)) { 1283 int stream_id; 1284 1285 for (stream_id = 1; stream_id < ep->stream_info->num_streams; 1286 stream_id++) { 1287 ring = ep->stream_info->stream_rings[stream_id]; 1288 if (!ring) 1289 continue; 1290 1291 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, 1292 "Killing URBs for slot ID %u, ep index %u, stream %u", 1293 slot_id, ep_index, stream_id); 1294 xhci_kill_ring_urbs(xhci, ring); 1295 } 1296 } else { 1297 ring = ep->ring; 1298 if (!ring) 1299 return; 1300 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, 1301 "Killing URBs for slot ID %u, ep index %u", 1302 slot_id, ep_index); 1303 xhci_kill_ring_urbs(xhci, ring); 1304 } 1305 1306 list_for_each_entry_safe(cur_td, tmp, &ep->cancelled_td_list, 1307 cancelled_td_list) { 1308 list_del_init(&cur_td->cancelled_td_list); 1309 inc_td_cnt(cur_td->urb); 1310 1311 if (last_td_in_urb(cur_td)) 1312 xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN); 1313 } 1314 } 1315 1316 /* 1317 * host controller died, register read returns 0xffffffff 1318 * Complete pending commands, mark them ABORTED. 1319 * URBs need to be given back as usb core might be waiting with device locks 1320 * held for the URBs to finish during device disconnect, blocking host remove. 1321 * 1322 * Call with xhci->lock held. 1323 * lock is relased and re-acquired while giving back urb. 1324 */ 1325 void xhci_hc_died(struct xhci_hcd *xhci) 1326 { 1327 int i, j; 1328 1329 if (xhci->xhc_state & XHCI_STATE_DYING) 1330 return; 1331 1332 xhci_err(xhci, "xHCI host controller not responding, assume dead\n"); 1333 xhci->xhc_state |= XHCI_STATE_DYING; 1334 1335 xhci_cleanup_command_queue(xhci); 1336 1337 /* return any pending urbs, remove may be waiting for them */ 1338 for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) { 1339 if (!xhci->devs[i]) 1340 continue; 1341 for (j = 0; j < 31; j++) 1342 xhci_kill_endpoint_urbs(xhci, i, j); 1343 } 1344 1345 /* inform usb core hc died if PCI remove isn't already handling it */ 1346 if (!(xhci->xhc_state & XHCI_STATE_REMOVING)) 1347 usb_hc_died(xhci_to_hcd(xhci)); 1348 } 1349 1350 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci, 1351 struct xhci_virt_device *dev, 1352 struct xhci_ring *ep_ring, 1353 unsigned int ep_index) 1354 { 1355 union xhci_trb *dequeue_temp; 1356 1357 dequeue_temp = ep_ring->dequeue; 1358 1359 /* If we get two back-to-back stalls, and the first stalled transfer 1360 * ends just before a link TRB, the dequeue pointer will be left on 1361 * the link TRB by the code in the while loop. So we have to update 1362 * the dequeue pointer one segment further, or we'll jump off 1363 * the segment into la-la-land. 1364 */ 1365 if (trb_is_link(ep_ring->dequeue)) { 1366 ep_ring->deq_seg = ep_ring->deq_seg->next; 1367 ep_ring->dequeue = ep_ring->deq_seg->trbs; 1368 } 1369 1370 while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) { 1371 /* We have more usable TRBs */ 1372 ep_ring->dequeue++; 1373 if (trb_is_link(ep_ring->dequeue)) { 1374 if (ep_ring->dequeue == 1375 dev->eps[ep_index].queued_deq_ptr) 1376 break; 1377 ep_ring->deq_seg = ep_ring->deq_seg->next; 1378 ep_ring->dequeue = ep_ring->deq_seg->trbs; 1379 } 1380 if (ep_ring->dequeue == dequeue_temp) { 1381 xhci_dbg(xhci, "Unable to find new dequeue pointer\n"); 1382 break; 1383 } 1384 } 1385 } 1386 1387 /* 1388 * When we get a completion for a Set Transfer Ring Dequeue Pointer command, 1389 * we need to clear the set deq pending flag in the endpoint ring state, so that 1390 * the TD queueing code can ring the doorbell again. We also need to ring the 1391 * endpoint doorbell to restart the ring, but only if there aren't more 1392 * cancellations pending. 1393 */ 1394 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id, 1395 union xhci_trb *trb, u32 cmd_comp_code) 1396 { 1397 unsigned int ep_index; 1398 unsigned int stream_id; 1399 struct xhci_ring *ep_ring; 1400 struct xhci_virt_ep *ep; 1401 struct xhci_ep_ctx *ep_ctx; 1402 struct xhci_slot_ctx *slot_ctx; 1403 struct xhci_td *td, *tmp_td; 1404 1405 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); 1406 stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2])); 1407 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); 1408 if (!ep) 1409 return; 1410 1411 ep_ring = xhci_virt_ep_to_ring(xhci, ep, stream_id); 1412 if (!ep_ring) { 1413 xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n", 1414 stream_id); 1415 /* XXX: Harmless??? */ 1416 goto cleanup; 1417 } 1418 1419 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); 1420 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); 1421 trace_xhci_handle_cmd_set_deq(slot_ctx); 1422 trace_xhci_handle_cmd_set_deq_ep(ep_ctx); 1423 1424 if (cmd_comp_code != COMP_SUCCESS) { 1425 unsigned int ep_state; 1426 unsigned int slot_state; 1427 1428 switch (cmd_comp_code) { 1429 case COMP_TRB_ERROR: 1430 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n"); 1431 break; 1432 case COMP_CONTEXT_STATE_ERROR: 1433 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n"); 1434 ep_state = GET_EP_CTX_STATE(ep_ctx); 1435 slot_state = le32_to_cpu(slot_ctx->dev_state); 1436 slot_state = GET_SLOT_STATE(slot_state); 1437 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, 1438 "Slot state = %u, EP state = %u", 1439 slot_state, ep_state); 1440 break; 1441 case COMP_SLOT_NOT_ENABLED_ERROR: 1442 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n", 1443 slot_id); 1444 break; 1445 default: 1446 xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n", 1447 cmd_comp_code); 1448 break; 1449 } 1450 /* OK what do we do now? The endpoint state is hosed, and we 1451 * should never get to this point if the synchronization between 1452 * queueing, and endpoint state are correct. This might happen 1453 * if the device gets disconnected after we've finished 1454 * cancelling URBs, which might not be an error... 1455 */ 1456 } else { 1457 u64 deq; 1458 /* 4.6.10 deq ptr is written to the stream ctx for streams */ 1459 if (ep->ep_state & EP_HAS_STREAMS) { 1460 struct xhci_stream_ctx *ctx = 1461 &ep->stream_info->stream_ctx_array[stream_id]; 1462 deq = le64_to_cpu(ctx->stream_ring) & SCTX_DEQ_MASK; 1463 1464 /* 1465 * Cadence xHCI controllers store some endpoint state 1466 * information within Rsvd0 fields of Stream Endpoint 1467 * context. This field is not cleared during Set TR 1468 * Dequeue Pointer command which causes XDMA to skip 1469 * over transfer ring and leads to data loss on stream 1470 * pipe. 1471 * To fix this issue driver must clear Rsvd0 field. 1472 */ 1473 if (xhci->quirks & XHCI_CDNS_SCTX_QUIRK) { 1474 ctx->reserved[0] = 0; 1475 ctx->reserved[1] = 0; 1476 } 1477 } else { 1478 deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK; 1479 } 1480 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb, 1481 "Successful Set TR Deq Ptr cmd, deq = @%08llx", deq); 1482 if (xhci_trb_virt_to_dma(ep->queued_deq_seg, 1483 ep->queued_deq_ptr) == deq) { 1484 /* Update the ring's dequeue segment and dequeue pointer 1485 * to reflect the new position. 1486 */ 1487 update_ring_for_set_deq_completion(xhci, ep->vdev, 1488 ep_ring, ep_index); 1489 } else { 1490 xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n"); 1491 xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n", 1492 ep->queued_deq_seg, ep->queued_deq_ptr); 1493 } 1494 } 1495 /* HW cached TDs cleared from cache, give them back */ 1496 list_for_each_entry_safe(td, tmp_td, &ep->cancelled_td_list, 1497 cancelled_td_list) { 1498 ep_ring = xhci_urb_to_transfer_ring(ep->xhci, td->urb); 1499 if (td->cancel_status == TD_CLEARING_CACHE) { 1500 td->cancel_status = TD_CLEARED; 1501 xhci_dbg(ep->xhci, "%s: Giveback cancelled URB %p TD\n", 1502 __func__, td->urb); 1503 xhci_td_cleanup(ep->xhci, td, ep_ring, td->status); 1504 } else { 1505 xhci_dbg(ep->xhci, "%s: Keep cancelled URB %p TD as cancel_status is %d\n", 1506 __func__, td->urb, td->cancel_status); 1507 } 1508 } 1509 cleanup: 1510 ep->ep_state &= ~SET_DEQ_PENDING; 1511 ep->queued_deq_seg = NULL; 1512 ep->queued_deq_ptr = NULL; 1513 1514 /* Check for deferred or newly cancelled TDs */ 1515 if (!list_empty(&ep->cancelled_td_list)) { 1516 xhci_dbg(ep->xhci, "%s: Pending TDs to clear, continuing with invalidation\n", 1517 __func__); 1518 xhci_invalidate_cancelled_tds(ep); 1519 /* Try to restart the endpoint if all is done */ 1520 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); 1521 /* Start giving back any TDs invalidated above */ 1522 xhci_giveback_invalidated_tds(ep); 1523 } else { 1524 /* Restart any rings with pending URBs */ 1525 xhci_dbg(ep->xhci, "%s: All TDs cleared, ring doorbell\n", __func__); 1526 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); 1527 } 1528 } 1529 1530 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id, 1531 union xhci_trb *trb, u32 cmd_comp_code) 1532 { 1533 struct xhci_virt_ep *ep; 1534 struct xhci_ep_ctx *ep_ctx; 1535 unsigned int ep_index; 1536 1537 ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3])); 1538 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); 1539 if (!ep) 1540 return; 1541 1542 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); 1543 trace_xhci_handle_cmd_reset_ep(ep_ctx); 1544 1545 /* This command will only fail if the endpoint wasn't halted, 1546 * but we don't care. 1547 */ 1548 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep, 1549 "Ignoring reset ep completion code of %u", cmd_comp_code); 1550 1551 /* Cleanup cancelled TDs as ep is stopped. May queue a Set TR Deq cmd */ 1552 xhci_invalidate_cancelled_tds(ep); 1553 1554 /* Clear our internal halted state */ 1555 ep->ep_state &= ~EP_HALTED; 1556 1557 xhci_giveback_invalidated_tds(ep); 1558 1559 /* if this was a soft reset, then restart */ 1560 if ((le32_to_cpu(trb->generic.field[3])) & TRB_TSP) 1561 ring_doorbell_for_active_rings(xhci, slot_id, ep_index); 1562 } 1563 1564 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id, 1565 struct xhci_command *command, u32 cmd_comp_code) 1566 { 1567 if (cmd_comp_code == COMP_SUCCESS) 1568 command->slot_id = slot_id; 1569 else 1570 command->slot_id = 0; 1571 } 1572 1573 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id) 1574 { 1575 struct xhci_virt_device *virt_dev; 1576 struct xhci_slot_ctx *slot_ctx; 1577 1578 virt_dev = xhci->devs[slot_id]; 1579 if (!virt_dev) 1580 return; 1581 1582 slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx); 1583 trace_xhci_handle_cmd_disable_slot(slot_ctx); 1584 1585 if (xhci->quirks & XHCI_EP_LIMIT_QUIRK) 1586 /* Delete default control endpoint resources */ 1587 xhci_free_device_endpoint_resources(xhci, virt_dev, true); 1588 } 1589 1590 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id, 1591 u32 cmd_comp_code) 1592 { 1593 struct xhci_virt_device *virt_dev; 1594 struct xhci_input_control_ctx *ctrl_ctx; 1595 struct xhci_ep_ctx *ep_ctx; 1596 unsigned int ep_index; 1597 u32 add_flags; 1598 1599 /* 1600 * Configure endpoint commands can come from the USB core configuration 1601 * or alt setting changes, or when streams were being configured. 1602 */ 1603 1604 virt_dev = xhci->devs[slot_id]; 1605 if (!virt_dev) 1606 return; 1607 ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx); 1608 if (!ctrl_ctx) { 1609 xhci_warn(xhci, "Could not get input context, bad type.\n"); 1610 return; 1611 } 1612 1613 add_flags = le32_to_cpu(ctrl_ctx->add_flags); 1614 1615 /* Input ctx add_flags are the endpoint index plus one */ 1616 ep_index = xhci_last_valid_endpoint(add_flags) - 1; 1617 1618 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index); 1619 trace_xhci_handle_cmd_config_ep(ep_ctx); 1620 1621 return; 1622 } 1623 1624 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id) 1625 { 1626 struct xhci_virt_device *vdev; 1627 struct xhci_slot_ctx *slot_ctx; 1628 1629 vdev = xhci->devs[slot_id]; 1630 if (!vdev) 1631 return; 1632 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); 1633 trace_xhci_handle_cmd_addr_dev(slot_ctx); 1634 } 1635 1636 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id) 1637 { 1638 struct xhci_virt_device *vdev; 1639 struct xhci_slot_ctx *slot_ctx; 1640 1641 vdev = xhci->devs[slot_id]; 1642 if (!vdev) { 1643 xhci_warn(xhci, "Reset device command completion for disabled slot %u\n", 1644 slot_id); 1645 return; 1646 } 1647 slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx); 1648 trace_xhci_handle_cmd_reset_dev(slot_ctx); 1649 1650 xhci_dbg(xhci, "Completed reset device command.\n"); 1651 } 1652 1653 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci, 1654 struct xhci_event_cmd *event) 1655 { 1656 if (!(xhci->quirks & XHCI_NEC_HOST)) { 1657 xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n"); 1658 return; 1659 } 1660 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks, 1661 "NEC firmware version %2x.%02x", 1662 NEC_FW_MAJOR(le32_to_cpu(event->status)), 1663 NEC_FW_MINOR(le32_to_cpu(event->status))); 1664 } 1665 1666 static void xhci_complete_del_and_free_cmd(struct xhci_command *cmd, u32 status) 1667 { 1668 list_del(&cmd->cmd_list); 1669 1670 if (cmd->completion) { 1671 cmd->status = status; 1672 complete(cmd->completion); 1673 } else { 1674 kfree(cmd); 1675 } 1676 } 1677 1678 void xhci_cleanup_command_queue(struct xhci_hcd *xhci) 1679 { 1680 struct xhci_command *cur_cmd, *tmp_cmd; 1681 xhci->current_cmd = NULL; 1682 list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list) 1683 xhci_complete_del_and_free_cmd(cur_cmd, COMP_COMMAND_ABORTED); 1684 } 1685 1686 void xhci_handle_command_timeout(struct work_struct *work) 1687 { 1688 struct xhci_hcd *xhci; 1689 unsigned long flags; 1690 char str[XHCI_MSG_MAX]; 1691 u64 hw_ring_state; 1692 u32 cmd_field3; 1693 u32 usbsts; 1694 1695 xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer); 1696 1697 spin_lock_irqsave(&xhci->lock, flags); 1698 1699 /* 1700 * If timeout work is pending, or current_cmd is NULL, it means we 1701 * raced with command completion. Command is handled so just return. 1702 */ 1703 if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) { 1704 spin_unlock_irqrestore(&xhci->lock, flags); 1705 return; 1706 } 1707 1708 cmd_field3 = le32_to_cpu(xhci->current_cmd->command_trb->generic.field[3]); 1709 usbsts = readl(&xhci->op_regs->status); 1710 xhci_dbg(xhci, "Command timeout, USBSTS:%s\n", xhci_decode_usbsts(str, usbsts)); 1711 1712 /* Bail out and tear down xhci if a stop endpoint command failed */ 1713 if (TRB_FIELD_TO_TYPE(cmd_field3) == TRB_STOP_RING) { 1714 struct xhci_virt_ep *ep; 1715 1716 xhci_warn(xhci, "xHCI host not responding to stop endpoint command\n"); 1717 1718 ep = xhci_get_virt_ep(xhci, TRB_TO_SLOT_ID(cmd_field3), 1719 TRB_TO_EP_INDEX(cmd_field3)); 1720 if (ep) 1721 ep->ep_state &= ~EP_STOP_CMD_PENDING; 1722 1723 xhci_halt(xhci); 1724 xhci_hc_died(xhci); 1725 goto time_out_completed; 1726 } 1727 1728 /* mark this command to be cancelled */ 1729 xhci->current_cmd->status = COMP_COMMAND_ABORTED; 1730 1731 /* Make sure command ring is running before aborting it */ 1732 hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring); 1733 if (hw_ring_state == ~(u64)0) { 1734 xhci_hc_died(xhci); 1735 goto time_out_completed; 1736 } 1737 1738 if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) && 1739 (hw_ring_state & CMD_RING_RUNNING)) { 1740 /* Prevent new doorbell, and start command abort */ 1741 xhci->cmd_ring_state = CMD_RING_STATE_ABORTED; 1742 xhci_dbg(xhci, "Command timeout\n"); 1743 xhci_abort_cmd_ring(xhci, flags); 1744 goto time_out_completed; 1745 } 1746 1747 /* host removed. Bail out */ 1748 if (xhci->xhc_state & XHCI_STATE_REMOVING) { 1749 xhci_dbg(xhci, "host removed, ring start fail?\n"); 1750 xhci_cleanup_command_queue(xhci); 1751 1752 goto time_out_completed; 1753 } 1754 1755 /* command timeout on stopped ring, ring can't be aborted */ 1756 xhci_dbg(xhci, "Command timeout on stopped ring\n"); 1757 xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd); 1758 1759 time_out_completed: 1760 spin_unlock_irqrestore(&xhci->lock, flags); 1761 return; 1762 } 1763 1764 static void handle_cmd_completion(struct xhci_hcd *xhci, 1765 struct xhci_event_cmd *event) 1766 { 1767 unsigned int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); 1768 u64 cmd_dma; 1769 dma_addr_t cmd_dequeue_dma; 1770 u32 cmd_comp_code; 1771 union xhci_trb *cmd_trb; 1772 struct xhci_command *cmd; 1773 u32 cmd_type; 1774 1775 if (slot_id >= MAX_HC_SLOTS) { 1776 xhci_warn(xhci, "Invalid slot_id %u\n", slot_id); 1777 return; 1778 } 1779 1780 cmd_dma = le64_to_cpu(event->cmd_trb); 1781 cmd_trb = xhci->cmd_ring->dequeue; 1782 1783 trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic); 1784 1785 cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status)); 1786 1787 /* If CMD ring stopped we own the trbs between enqueue and dequeue */ 1788 if (cmd_comp_code == COMP_COMMAND_RING_STOPPED) { 1789 complete_all(&xhci->cmd_ring_stop_completion); 1790 return; 1791 } 1792 1793 cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg, 1794 cmd_trb); 1795 /* 1796 * Check whether the completion event is for our internal kept 1797 * command. 1798 */ 1799 if (!cmd_dequeue_dma || cmd_dma != (u64)cmd_dequeue_dma) { 1800 xhci_warn(xhci, 1801 "ERROR mismatched command completion event\n"); 1802 return; 1803 } 1804 1805 cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list); 1806 1807 cancel_delayed_work(&xhci->cmd_timer); 1808 1809 if (cmd->command_trb != xhci->cmd_ring->dequeue) { 1810 xhci_err(xhci, 1811 "Command completion event does not match command\n"); 1812 return; 1813 } 1814 1815 /* 1816 * Host aborted the command ring, check if the current command was 1817 * supposed to be aborted, otherwise continue normally. 1818 * The command ring is stopped now, but the xHC will issue a Command 1819 * Ring Stopped event which will cause us to restart it. 1820 */ 1821 if (cmd_comp_code == COMP_COMMAND_ABORTED) { 1822 xhci->cmd_ring_state = CMD_RING_STATE_STOPPED; 1823 if (cmd->status == COMP_COMMAND_ABORTED) { 1824 if (xhci->current_cmd == cmd) 1825 xhci->current_cmd = NULL; 1826 goto event_handled; 1827 } 1828 } 1829 1830 cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3])); 1831 switch (cmd_type) { 1832 case TRB_ENABLE_SLOT: 1833 xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code); 1834 break; 1835 case TRB_DISABLE_SLOT: 1836 xhci_handle_cmd_disable_slot(xhci, slot_id); 1837 break; 1838 case TRB_CONFIG_EP: 1839 if (!cmd->completion) 1840 xhci_handle_cmd_config_ep(xhci, slot_id, cmd_comp_code); 1841 break; 1842 case TRB_EVAL_CONTEXT: 1843 break; 1844 case TRB_ADDR_DEV: 1845 xhci_handle_cmd_addr_dev(xhci, slot_id); 1846 break; 1847 case TRB_STOP_RING: 1848 WARN_ON(slot_id != TRB_TO_SLOT_ID( 1849 le32_to_cpu(cmd_trb->generic.field[3]))); 1850 if (!cmd->completion) 1851 xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, 1852 cmd_comp_code); 1853 break; 1854 case TRB_SET_DEQ: 1855 WARN_ON(slot_id != TRB_TO_SLOT_ID( 1856 le32_to_cpu(cmd_trb->generic.field[3]))); 1857 xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code); 1858 break; 1859 case TRB_CMD_NOOP: 1860 /* Is this an aborted command turned to NO-OP? */ 1861 if (cmd->status == COMP_COMMAND_RING_STOPPED) 1862 cmd_comp_code = COMP_COMMAND_RING_STOPPED; 1863 break; 1864 case TRB_RESET_EP: 1865 WARN_ON(slot_id != TRB_TO_SLOT_ID( 1866 le32_to_cpu(cmd_trb->generic.field[3]))); 1867 xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code); 1868 break; 1869 case TRB_RESET_DEV: 1870 /* SLOT_ID field in reset device cmd completion event TRB is 0. 1871 * Use the SLOT_ID from the command TRB instead (xhci 4.6.11) 1872 */ 1873 slot_id = TRB_TO_SLOT_ID( 1874 le32_to_cpu(cmd_trb->generic.field[3])); 1875 xhci_handle_cmd_reset_dev(xhci, slot_id); 1876 break; 1877 case TRB_NEC_GET_FW: 1878 xhci_handle_cmd_nec_get_fw(xhci, event); 1879 break; 1880 default: 1881 /* Skip over unknown commands on the event ring */ 1882 xhci_info(xhci, "INFO unknown command type %d\n", cmd_type); 1883 break; 1884 } 1885 1886 /* restart timer if this wasn't the last command */ 1887 if (!list_is_singular(&xhci->cmd_list)) { 1888 xhci->current_cmd = list_first_entry(&cmd->cmd_list, 1889 struct xhci_command, cmd_list); 1890 xhci_mod_cmd_timer(xhci); 1891 } else if (xhci->current_cmd == cmd) { 1892 xhci->current_cmd = NULL; 1893 } 1894 1895 event_handled: 1896 xhci_complete_del_and_free_cmd(cmd, cmd_comp_code); 1897 1898 inc_deq(xhci, xhci->cmd_ring); 1899 } 1900 1901 static void handle_vendor_event(struct xhci_hcd *xhci, 1902 union xhci_trb *event, u32 trb_type) 1903 { 1904 xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type); 1905 if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST)) 1906 handle_cmd_completion(xhci, &event->event_cmd); 1907 } 1908 1909 static void handle_device_notification(struct xhci_hcd *xhci, 1910 union xhci_trb *event) 1911 { 1912 u32 slot_id; 1913 struct usb_device *udev; 1914 1915 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3])); 1916 if (!xhci->devs[slot_id]) { 1917 xhci_warn(xhci, "Device Notification event for " 1918 "unused slot %u\n", slot_id); 1919 return; 1920 } 1921 1922 xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n", 1923 slot_id); 1924 udev = xhci->devs[slot_id]->udev; 1925 if (udev && udev->parent) 1926 usb_wakeup_notification(udev->parent, udev->portnum); 1927 } 1928 1929 /* 1930 * Quirk hanlder for errata seen on Cavium ThunderX2 processor XHCI 1931 * Controller. 1932 * As per ThunderX2errata-129 USB 2 device may come up as USB 1 1933 * If a connection to a USB 1 device is followed by another connection 1934 * to a USB 2 device. 1935 * 1936 * Reset the PHY after the USB device is disconnected if device speed 1937 * is less than HCD_USB3. 1938 * Retry the reset sequence max of 4 times checking the PLL lock status. 1939 * 1940 */ 1941 static void xhci_cavium_reset_phy_quirk(struct xhci_hcd *xhci) 1942 { 1943 struct usb_hcd *hcd = xhci_to_hcd(xhci); 1944 u32 pll_lock_check; 1945 u32 retry_count = 4; 1946 1947 do { 1948 /* Assert PHY reset */ 1949 writel(0x6F, hcd->regs + 0x1048); 1950 udelay(10); 1951 /* De-assert the PHY reset */ 1952 writel(0x7F, hcd->regs + 0x1048); 1953 udelay(200); 1954 pll_lock_check = readl(hcd->regs + 0x1070); 1955 } while (!(pll_lock_check & 0x1) && --retry_count); 1956 } 1957 1958 static void handle_port_status(struct xhci_hcd *xhci, 1959 struct xhci_interrupter *ir, 1960 union xhci_trb *event) 1961 { 1962 struct usb_hcd *hcd; 1963 u32 port_id; 1964 u32 portsc, cmd_reg; 1965 int max_ports; 1966 int slot_id; 1967 unsigned int hcd_portnum; 1968 struct xhci_bus_state *bus_state; 1969 bool bogus_port_status = false; 1970 struct xhci_port *port; 1971 1972 /* Port status change events always have a successful completion code */ 1973 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) 1974 xhci_warn(xhci, 1975 "WARN: xHC returned failed port status event\n"); 1976 1977 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); 1978 max_ports = HCS_MAX_PORTS(xhci->hcs_params1); 1979 1980 if ((port_id <= 0) || (port_id > max_ports)) { 1981 xhci_warn(xhci, "Port change event with invalid port ID %d\n", 1982 port_id); 1983 inc_deq(xhci, ir->event_ring); 1984 return; 1985 } 1986 1987 port = &xhci->hw_ports[port_id - 1]; 1988 if (!port || !port->rhub || port->hcd_portnum == DUPLICATE_ENTRY) { 1989 xhci_warn(xhci, "Port change event, no port for port ID %u\n", 1990 port_id); 1991 bogus_port_status = true; 1992 goto cleanup; 1993 } 1994 1995 /* We might get interrupts after shared_hcd is removed */ 1996 if (port->rhub == &xhci->usb3_rhub && xhci->shared_hcd == NULL) { 1997 xhci_dbg(xhci, "ignore port event for removed USB3 hcd\n"); 1998 bogus_port_status = true; 1999 goto cleanup; 2000 } 2001 2002 hcd = port->rhub->hcd; 2003 bus_state = &port->rhub->bus_state; 2004 hcd_portnum = port->hcd_portnum; 2005 portsc = readl(port->addr); 2006 2007 xhci_dbg(xhci, "Port change event, %d-%d, id %d, portsc: 0x%x\n", 2008 hcd->self.busnum, hcd_portnum + 1, port_id, portsc); 2009 2010 trace_xhci_handle_port_status(hcd_portnum, portsc); 2011 2012 if (hcd->state == HC_STATE_SUSPENDED) { 2013 xhci_dbg(xhci, "resume root hub\n"); 2014 usb_hcd_resume_root_hub(hcd); 2015 } 2016 2017 if (hcd->speed >= HCD_USB3 && 2018 (portsc & PORT_PLS_MASK) == XDEV_INACTIVE) { 2019 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1); 2020 if (slot_id && xhci->devs[slot_id]) 2021 xhci->devs[slot_id]->flags |= VDEV_PORT_ERROR; 2022 } 2023 2024 if ((portsc & PORT_PLC) && (portsc & PORT_PLS_MASK) == XDEV_RESUME) { 2025 xhci_dbg(xhci, "port resume event for port %d\n", port_id); 2026 2027 cmd_reg = readl(&xhci->op_regs->command); 2028 if (!(cmd_reg & CMD_RUN)) { 2029 xhci_warn(xhci, "xHC is not running.\n"); 2030 goto cleanup; 2031 } 2032 2033 if (DEV_SUPERSPEED_ANY(portsc)) { 2034 xhci_dbg(xhci, "remote wake SS port %d\n", port_id); 2035 /* Set a flag to say the port signaled remote wakeup, 2036 * so we can tell the difference between the end of 2037 * device and host initiated resume. 2038 */ 2039 bus_state->port_remote_wakeup |= 1 << hcd_portnum; 2040 xhci_test_and_clear_bit(xhci, port, PORT_PLC); 2041 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); 2042 xhci_set_link_state(xhci, port, XDEV_U0); 2043 /* Need to wait until the next link state change 2044 * indicates the device is actually in U0. 2045 */ 2046 bogus_port_status = true; 2047 goto cleanup; 2048 } else if (!test_bit(hcd_portnum, &bus_state->resuming_ports)) { 2049 xhci_dbg(xhci, "resume HS port %d\n", port_id); 2050 port->resume_timestamp = jiffies + 2051 msecs_to_jiffies(USB_RESUME_TIMEOUT); 2052 set_bit(hcd_portnum, &bus_state->resuming_ports); 2053 /* Do the rest in GetPortStatus after resume time delay. 2054 * Avoid polling roothub status before that so that a 2055 * usb device auto-resume latency around ~40ms. 2056 */ 2057 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); 2058 mod_timer(&hcd->rh_timer, 2059 port->resume_timestamp); 2060 usb_hcd_start_port_resume(&hcd->self, hcd_portnum); 2061 bogus_port_status = true; 2062 } 2063 } 2064 2065 if ((portsc & PORT_PLC) && 2066 DEV_SUPERSPEED_ANY(portsc) && 2067 ((portsc & PORT_PLS_MASK) == XDEV_U0 || 2068 (portsc & PORT_PLS_MASK) == XDEV_U1 || 2069 (portsc & PORT_PLS_MASK) == XDEV_U2)) { 2070 xhci_dbg(xhci, "resume SS port %d finished\n", port_id); 2071 complete(&port->u3exit_done); 2072 /* We've just brought the device into U0/1/2 through either the 2073 * Resume state after a device remote wakeup, or through the 2074 * U3Exit state after a host-initiated resume. If it's a device 2075 * initiated remote wake, don't pass up the link state change, 2076 * so the roothub behavior is consistent with external 2077 * USB 3.0 hub behavior. 2078 */ 2079 slot_id = xhci_find_slot_id_by_port(hcd, xhci, hcd_portnum + 1); 2080 if (slot_id && xhci->devs[slot_id]) 2081 xhci_ring_device(xhci, slot_id); 2082 if (bus_state->port_remote_wakeup & (1 << hcd_portnum)) { 2083 xhci_test_and_clear_bit(xhci, port, PORT_PLC); 2084 usb_wakeup_notification(hcd->self.root_hub, 2085 hcd_portnum + 1); 2086 bogus_port_status = true; 2087 goto cleanup; 2088 } 2089 } 2090 2091 /* 2092 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or 2093 * RExit to a disconnect state). If so, let the driver know it's 2094 * out of the RExit state. 2095 */ 2096 if (hcd->speed < HCD_USB3 && port->rexit_active) { 2097 complete(&port->rexit_done); 2098 port->rexit_active = false; 2099 bogus_port_status = true; 2100 goto cleanup; 2101 } 2102 2103 if (hcd->speed < HCD_USB3) { 2104 xhci_test_and_clear_bit(xhci, port, PORT_PLC); 2105 if ((xhci->quirks & XHCI_RESET_PLL_ON_DISCONNECT) && 2106 (portsc & PORT_CSC) && !(portsc & PORT_CONNECT)) 2107 xhci_cavium_reset_phy_quirk(xhci); 2108 } 2109 2110 cleanup: 2111 /* Update event ring dequeue pointer before dropping the lock */ 2112 inc_deq(xhci, ir->event_ring); 2113 2114 /* Don't make the USB core poll the roothub if we got a bad port status 2115 * change event. Besides, at that point we can't tell which roothub 2116 * (USB 2.0 or USB 3.0) to kick. 2117 */ 2118 if (bogus_port_status) 2119 return; 2120 2121 /* 2122 * xHCI port-status-change events occur when the "or" of all the 2123 * status-change bits in the portsc register changes from 0 to 1. 2124 * New status changes won't cause an event if any other change 2125 * bits are still set. When an event occurs, switch over to 2126 * polling to avoid losing status changes. 2127 */ 2128 xhci_dbg(xhci, "%s: starting usb%d port polling.\n", 2129 __func__, hcd->self.busnum); 2130 set_bit(HCD_FLAG_POLL_RH, &hcd->flags); 2131 spin_unlock(&xhci->lock); 2132 /* Pass this up to the core */ 2133 usb_hcd_poll_rh_status(hcd); 2134 spin_lock(&xhci->lock); 2135 } 2136 2137 /* 2138 * This TD is defined by the TRBs starting at start_trb in start_seg and ending 2139 * at end_trb, which may be in another segment. If the suspect DMA address is a 2140 * TRB in this TD, this function returns that TRB's segment. Otherwise it 2141 * returns 0. 2142 */ 2143 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci, 2144 struct xhci_segment *start_seg, 2145 union xhci_trb *start_trb, 2146 union xhci_trb *end_trb, 2147 dma_addr_t suspect_dma, 2148 bool debug) 2149 { 2150 dma_addr_t start_dma; 2151 dma_addr_t end_seg_dma; 2152 dma_addr_t end_trb_dma; 2153 struct xhci_segment *cur_seg; 2154 2155 start_dma = xhci_trb_virt_to_dma(start_seg, start_trb); 2156 cur_seg = start_seg; 2157 2158 do { 2159 if (start_dma == 0) 2160 return NULL; 2161 /* We may get an event for a Link TRB in the middle of a TD */ 2162 end_seg_dma = xhci_trb_virt_to_dma(cur_seg, 2163 &cur_seg->trbs[TRBS_PER_SEGMENT - 1]); 2164 /* If the end TRB isn't in this segment, this is set to 0 */ 2165 end_trb_dma = xhci_trb_virt_to_dma(cur_seg, end_trb); 2166 2167 if (debug) 2168 xhci_warn(xhci, 2169 "Looking for event-dma %016llx trb-start %016llx trb-end %016llx seg-start %016llx seg-end %016llx\n", 2170 (unsigned long long)suspect_dma, 2171 (unsigned long long)start_dma, 2172 (unsigned long long)end_trb_dma, 2173 (unsigned long long)cur_seg->dma, 2174 (unsigned long long)end_seg_dma); 2175 2176 if (end_trb_dma > 0) { 2177 /* The end TRB is in this segment, so suspect should be here */ 2178 if (start_dma <= end_trb_dma) { 2179 if (suspect_dma >= start_dma && suspect_dma <= end_trb_dma) 2180 return cur_seg; 2181 } else { 2182 /* Case for one segment with 2183 * a TD wrapped around to the top 2184 */ 2185 if ((suspect_dma >= start_dma && 2186 suspect_dma <= end_seg_dma) || 2187 (suspect_dma >= cur_seg->dma && 2188 suspect_dma <= end_trb_dma)) 2189 return cur_seg; 2190 } 2191 return NULL; 2192 } else { 2193 /* Might still be somewhere in this segment */ 2194 if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma) 2195 return cur_seg; 2196 } 2197 cur_seg = cur_seg->next; 2198 start_dma = xhci_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]); 2199 } while (cur_seg != start_seg); 2200 2201 return NULL; 2202 } 2203 2204 static void xhci_clear_hub_tt_buffer(struct xhci_hcd *xhci, struct xhci_td *td, 2205 struct xhci_virt_ep *ep) 2206 { 2207 /* 2208 * As part of low/full-speed endpoint-halt processing 2209 * we must clear the TT buffer (USB 2.0 specification 11.17.5). 2210 */ 2211 if (td->urb->dev->tt && !usb_pipeint(td->urb->pipe) && 2212 (td->urb->dev->tt->hub != xhci_to_hcd(xhci)->self.root_hub) && 2213 !(ep->ep_state & EP_CLEARING_TT)) { 2214 ep->ep_state |= EP_CLEARING_TT; 2215 td->urb->ep->hcpriv = td->urb->dev; 2216 if (usb_hub_clear_tt_buffer(td->urb)) 2217 ep->ep_state &= ~EP_CLEARING_TT; 2218 } 2219 } 2220 2221 /* Check if an error has halted the endpoint ring. The class driver will 2222 * cleanup the halt for a non-default control endpoint if we indicate a stall. 2223 * However, a babble and other errors also halt the endpoint ring, and the class 2224 * driver won't clear the halt in that case, so we need to issue a Set Transfer 2225 * Ring Dequeue Pointer command manually. 2226 */ 2227 static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci, 2228 struct xhci_ep_ctx *ep_ctx, 2229 unsigned int trb_comp_code) 2230 { 2231 /* TRB completion codes that may require a manual halt cleanup */ 2232 if (trb_comp_code == COMP_USB_TRANSACTION_ERROR || 2233 trb_comp_code == COMP_BABBLE_DETECTED_ERROR || 2234 trb_comp_code == COMP_SPLIT_TRANSACTION_ERROR) 2235 /* The 0.95 spec says a babbling control endpoint 2236 * is not halted. The 0.96 spec says it is. Some HW 2237 * claims to be 0.95 compliant, but it halts the control 2238 * endpoint anyway. Check if a babble halted the 2239 * endpoint. 2240 */ 2241 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_HALTED) 2242 return 1; 2243 2244 return 0; 2245 } 2246 2247 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code) 2248 { 2249 if (trb_comp_code >= 224 && trb_comp_code <= 255) { 2250 /* Vendor defined "informational" completion code, 2251 * treat as not-an-error. 2252 */ 2253 xhci_dbg(xhci, "Vendor defined info completion code %u\n", 2254 trb_comp_code); 2255 xhci_dbg(xhci, "Treating code as success.\n"); 2256 return 1; 2257 } 2258 return 0; 2259 } 2260 2261 static int finish_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, 2262 struct xhci_ring *ep_ring, struct xhci_td *td, 2263 u32 trb_comp_code) 2264 { 2265 struct xhci_ep_ctx *ep_ctx; 2266 2267 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); 2268 2269 switch (trb_comp_code) { 2270 case COMP_STOPPED_LENGTH_INVALID: 2271 case COMP_STOPPED_SHORT_PACKET: 2272 case COMP_STOPPED: 2273 /* 2274 * The "Stop Endpoint" completion will take care of any 2275 * stopped TDs. A stopped TD may be restarted, so don't update 2276 * the ring dequeue pointer or take this TD off any lists yet. 2277 */ 2278 return 0; 2279 case COMP_USB_TRANSACTION_ERROR: 2280 case COMP_BABBLE_DETECTED_ERROR: 2281 case COMP_SPLIT_TRANSACTION_ERROR: 2282 /* 2283 * If endpoint context state is not halted we might be 2284 * racing with a reset endpoint command issued by a unsuccessful 2285 * stop endpoint completion (context error). In that case the 2286 * td should be on the cancelled list, and EP_HALTED flag set. 2287 * 2288 * Or then it's not halted due to the 0.95 spec stating that a 2289 * babbling control endpoint should not halt. The 0.96 spec 2290 * again says it should. Some HW claims to be 0.95 compliant, 2291 * but it halts the control endpoint anyway. 2292 */ 2293 if (GET_EP_CTX_STATE(ep_ctx) != EP_STATE_HALTED) { 2294 /* 2295 * If EP_HALTED is set and TD is on the cancelled list 2296 * the TD and dequeue pointer will be handled by reset 2297 * ep command completion 2298 */ 2299 if ((ep->ep_state & EP_HALTED) && 2300 !list_empty(&td->cancelled_td_list)) { 2301 xhci_dbg(xhci, "Already resolving halted ep for 0x%llx\n", 2302 (unsigned long long)xhci_trb_virt_to_dma( 2303 td->start_seg, td->first_trb)); 2304 return 0; 2305 } 2306 /* endpoint not halted, don't reset it */ 2307 break; 2308 } 2309 /* Almost same procedure as for STALL_ERROR below */ 2310 xhci_clear_hub_tt_buffer(xhci, td, ep); 2311 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET); 2312 return 0; 2313 case COMP_STALL_ERROR: 2314 /* 2315 * xhci internal endpoint state will go to a "halt" state for 2316 * any stall, including default control pipe protocol stall. 2317 * To clear the host side halt we need to issue a reset endpoint 2318 * command, followed by a set dequeue command to move past the 2319 * TD. 2320 * Class drivers clear the device side halt from a functional 2321 * stall later. Hub TT buffer should only be cleared for FS/LS 2322 * devices behind HS hubs for functional stalls. 2323 */ 2324 if (ep->ep_index != 0) 2325 xhci_clear_hub_tt_buffer(xhci, td, ep); 2326 2327 xhci_handle_halted_endpoint(xhci, ep, td, EP_HARD_RESET); 2328 2329 return 0; /* xhci_handle_halted_endpoint marked td cancelled */ 2330 default: 2331 break; 2332 } 2333 2334 /* Update ring dequeue pointer */ 2335 ep_ring->dequeue = td->last_trb; 2336 ep_ring->deq_seg = td->last_trb_seg; 2337 inc_deq(xhci, ep_ring); 2338 2339 return xhci_td_cleanup(xhci, td, ep_ring, td->status); 2340 } 2341 2342 /* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */ 2343 static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring, 2344 union xhci_trb *stop_trb) 2345 { 2346 u32 sum; 2347 union xhci_trb *trb = ring->dequeue; 2348 struct xhci_segment *seg = ring->deq_seg; 2349 2350 for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) { 2351 if (!trb_is_noop(trb) && !trb_is_link(trb)) 2352 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2])); 2353 } 2354 return sum; 2355 } 2356 2357 /* 2358 * Process control tds, update urb status and actual_length. 2359 */ 2360 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, 2361 struct xhci_ring *ep_ring, struct xhci_td *td, 2362 union xhci_trb *ep_trb, struct xhci_transfer_event *event) 2363 { 2364 struct xhci_ep_ctx *ep_ctx; 2365 u32 trb_comp_code; 2366 u32 remaining, requested; 2367 u32 trb_type; 2368 2369 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3])); 2370 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep->ep_index); 2371 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 2372 requested = td->urb->transfer_buffer_length; 2373 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 2374 2375 switch (trb_comp_code) { 2376 case COMP_SUCCESS: 2377 if (trb_type != TRB_STATUS) { 2378 xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n", 2379 (trb_type == TRB_DATA) ? "data" : "setup"); 2380 td->status = -ESHUTDOWN; 2381 break; 2382 } 2383 td->status = 0; 2384 break; 2385 case COMP_SHORT_PACKET: 2386 td->status = 0; 2387 break; 2388 case COMP_STOPPED_SHORT_PACKET: 2389 if (trb_type == TRB_DATA || trb_type == TRB_NORMAL) 2390 td->urb->actual_length = remaining; 2391 else 2392 xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n"); 2393 goto finish_td; 2394 case COMP_STOPPED: 2395 switch (trb_type) { 2396 case TRB_SETUP: 2397 td->urb->actual_length = 0; 2398 goto finish_td; 2399 case TRB_DATA: 2400 case TRB_NORMAL: 2401 td->urb->actual_length = requested - remaining; 2402 goto finish_td; 2403 case TRB_STATUS: 2404 td->urb->actual_length = requested; 2405 goto finish_td; 2406 default: 2407 xhci_warn(xhci, "WARN: unexpected TRB Type %d\n", 2408 trb_type); 2409 goto finish_td; 2410 } 2411 case COMP_STOPPED_LENGTH_INVALID: 2412 goto finish_td; 2413 default: 2414 if (!xhci_requires_manual_halt_cleanup(xhci, 2415 ep_ctx, trb_comp_code)) 2416 break; 2417 xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n", 2418 trb_comp_code, ep->ep_index); 2419 fallthrough; 2420 case COMP_STALL_ERROR: 2421 /* Did we transfer part of the data (middle) phase? */ 2422 if (trb_type == TRB_DATA || trb_type == TRB_NORMAL) 2423 td->urb->actual_length = requested - remaining; 2424 else if (!td->urb_length_set) 2425 td->urb->actual_length = 0; 2426 goto finish_td; 2427 } 2428 2429 /* stopped at setup stage, no data transferred */ 2430 if (trb_type == TRB_SETUP) 2431 goto finish_td; 2432 2433 /* 2434 * if on data stage then update the actual_length of the URB and flag it 2435 * as set, so it won't be overwritten in the event for the last TRB. 2436 */ 2437 if (trb_type == TRB_DATA || 2438 trb_type == TRB_NORMAL) { 2439 td->urb_length_set = true; 2440 td->urb->actual_length = requested - remaining; 2441 xhci_dbg(xhci, "Waiting for status stage event\n"); 2442 return 0; 2443 } 2444 2445 /* at status stage */ 2446 if (!td->urb_length_set) 2447 td->urb->actual_length = requested; 2448 2449 finish_td: 2450 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); 2451 } 2452 2453 /* 2454 * Process isochronous tds, update urb packet status and actual_length. 2455 */ 2456 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, 2457 struct xhci_ring *ep_ring, struct xhci_td *td, 2458 union xhci_trb *ep_trb, struct xhci_transfer_event *event) 2459 { 2460 struct urb_priv *urb_priv; 2461 int idx; 2462 struct usb_iso_packet_descriptor *frame; 2463 u32 trb_comp_code; 2464 bool sum_trbs_for_length = false; 2465 u32 remaining, requested, ep_trb_len; 2466 int short_framestatus; 2467 2468 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 2469 urb_priv = td->urb->hcpriv; 2470 idx = urb_priv->num_tds_done; 2471 frame = &td->urb->iso_frame_desc[idx]; 2472 requested = frame->length; 2473 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 2474 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); 2475 short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ? 2476 -EREMOTEIO : 0; 2477 2478 /* handle completion code */ 2479 switch (trb_comp_code) { 2480 case COMP_SUCCESS: 2481 /* Don't overwrite status if TD had an error, see xHCI 4.9.1 */ 2482 if (td->error_mid_td) 2483 break; 2484 if (remaining) { 2485 frame->status = short_framestatus; 2486 sum_trbs_for_length = true; 2487 break; 2488 } 2489 frame->status = 0; 2490 break; 2491 case COMP_SHORT_PACKET: 2492 frame->status = short_framestatus; 2493 sum_trbs_for_length = true; 2494 break; 2495 case COMP_BANDWIDTH_OVERRUN_ERROR: 2496 frame->status = -ECOMM; 2497 break; 2498 case COMP_BABBLE_DETECTED_ERROR: 2499 sum_trbs_for_length = true; 2500 fallthrough; 2501 case COMP_ISOCH_BUFFER_OVERRUN: 2502 frame->status = -EOVERFLOW; 2503 if (ep_trb != td->last_trb) 2504 td->error_mid_td = true; 2505 break; 2506 case COMP_INCOMPATIBLE_DEVICE_ERROR: 2507 case COMP_STALL_ERROR: 2508 frame->status = -EPROTO; 2509 break; 2510 case COMP_USB_TRANSACTION_ERROR: 2511 frame->status = -EPROTO; 2512 sum_trbs_for_length = true; 2513 if (ep_trb != td->last_trb) 2514 td->error_mid_td = true; 2515 break; 2516 case COMP_STOPPED: 2517 sum_trbs_for_length = true; 2518 break; 2519 case COMP_STOPPED_SHORT_PACKET: 2520 /* field normally containing residue now contains tranferred */ 2521 frame->status = short_framestatus; 2522 requested = remaining; 2523 break; 2524 case COMP_STOPPED_LENGTH_INVALID: 2525 requested = 0; 2526 remaining = 0; 2527 break; 2528 default: 2529 sum_trbs_for_length = true; 2530 frame->status = -1; 2531 break; 2532 } 2533 2534 if (td->urb_length_set) 2535 goto finish_td; 2536 2537 if (sum_trbs_for_length) 2538 frame->actual_length = sum_trb_lengths(xhci, ep->ring, ep_trb) + 2539 ep_trb_len - remaining; 2540 else 2541 frame->actual_length = requested; 2542 2543 td->urb->actual_length += frame->actual_length; 2544 2545 finish_td: 2546 /* Don't give back TD yet if we encountered an error mid TD */ 2547 if (td->error_mid_td && ep_trb != td->last_trb) { 2548 xhci_dbg(xhci, "Error mid isoc TD, wait for final completion event\n"); 2549 td->urb_length_set = true; 2550 return 0; 2551 } 2552 2553 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); 2554 } 2555 2556 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td, 2557 struct xhci_virt_ep *ep, int status) 2558 { 2559 struct urb_priv *urb_priv; 2560 struct usb_iso_packet_descriptor *frame; 2561 int idx; 2562 2563 urb_priv = td->urb->hcpriv; 2564 idx = urb_priv->num_tds_done; 2565 frame = &td->urb->iso_frame_desc[idx]; 2566 2567 /* The transfer is partly done. */ 2568 frame->status = -EXDEV; 2569 2570 /* calc actual length */ 2571 frame->actual_length = 0; 2572 2573 /* Update ring dequeue pointer */ 2574 ep->ring->dequeue = td->last_trb; 2575 ep->ring->deq_seg = td->last_trb_seg; 2576 inc_deq(xhci, ep->ring); 2577 2578 return xhci_td_cleanup(xhci, td, ep->ring, status); 2579 } 2580 2581 /* 2582 * Process bulk and interrupt tds, update urb status and actual_length. 2583 */ 2584 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_virt_ep *ep, 2585 struct xhci_ring *ep_ring, struct xhci_td *td, 2586 union xhci_trb *ep_trb, struct xhci_transfer_event *event) 2587 { 2588 struct xhci_slot_ctx *slot_ctx; 2589 u32 trb_comp_code; 2590 u32 remaining, requested, ep_trb_len; 2591 2592 slot_ctx = xhci_get_slot_ctx(xhci, ep->vdev->out_ctx); 2593 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 2594 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 2595 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); 2596 requested = td->urb->transfer_buffer_length; 2597 2598 switch (trb_comp_code) { 2599 case COMP_SUCCESS: 2600 ep->err_count = 0; 2601 /* handle success with untransferred data as short packet */ 2602 if (ep_trb != td->last_trb || remaining) { 2603 xhci_warn(xhci, "WARN Successful completion on short TX\n"); 2604 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", 2605 td->urb->ep->desc.bEndpointAddress, 2606 requested, remaining); 2607 } 2608 td->status = 0; 2609 break; 2610 case COMP_SHORT_PACKET: 2611 xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n", 2612 td->urb->ep->desc.bEndpointAddress, 2613 requested, remaining); 2614 td->status = 0; 2615 break; 2616 case COMP_STOPPED_SHORT_PACKET: 2617 td->urb->actual_length = remaining; 2618 goto finish_td; 2619 case COMP_STOPPED_LENGTH_INVALID: 2620 /* stopped on ep trb with invalid length, exclude it */ 2621 td->urb->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb); 2622 goto finish_td; 2623 case COMP_USB_TRANSACTION_ERROR: 2624 if (xhci->quirks & XHCI_NO_SOFT_RETRY || 2625 (ep->err_count++ > MAX_SOFT_RETRY) || 2626 le32_to_cpu(slot_ctx->tt_info) & TT_SLOT) 2627 break; 2628 2629 td->status = 0; 2630 2631 xhci_handle_halted_endpoint(xhci, ep, td, EP_SOFT_RESET); 2632 return 0; 2633 default: 2634 /* do nothing */ 2635 break; 2636 } 2637 2638 if (ep_trb == td->last_trb) 2639 td->urb->actual_length = requested - remaining; 2640 else 2641 td->urb->actual_length = 2642 sum_trb_lengths(xhci, ep_ring, ep_trb) + 2643 ep_trb_len - remaining; 2644 finish_td: 2645 if (remaining > requested) { 2646 xhci_warn(xhci, "bad transfer trb length %d in event trb\n", 2647 remaining); 2648 td->urb->actual_length = 0; 2649 } 2650 2651 return finish_td(xhci, ep, ep_ring, td, trb_comp_code); 2652 } 2653 2654 /* 2655 * If this function returns an error condition, it means it got a Transfer 2656 * event with a corrupted Slot ID, Endpoint ID, or TRB DMA address. 2657 * At this point, the host controller is probably hosed and should be reset. 2658 */ 2659 static int handle_tx_event(struct xhci_hcd *xhci, 2660 struct xhci_interrupter *ir, 2661 struct xhci_transfer_event *event) 2662 { 2663 struct xhci_virt_ep *ep; 2664 struct xhci_ring *ep_ring; 2665 unsigned int slot_id; 2666 int ep_index; 2667 struct xhci_td *td = NULL; 2668 dma_addr_t ep_trb_dma; 2669 struct xhci_segment *ep_seg; 2670 union xhci_trb *ep_trb; 2671 int status = -EINPROGRESS; 2672 struct xhci_ep_ctx *ep_ctx; 2673 u32 trb_comp_code; 2674 int td_num = 0; 2675 bool handling_skipped_tds = false; 2676 2677 slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags)); 2678 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; 2679 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 2680 ep_trb_dma = le64_to_cpu(event->buffer); 2681 2682 ep = xhci_get_virt_ep(xhci, slot_id, ep_index); 2683 if (!ep) { 2684 xhci_err(xhci, "ERROR Invalid Transfer event\n"); 2685 goto err_out; 2686 } 2687 2688 ep_ring = xhci_dma_to_transfer_ring(ep, ep_trb_dma); 2689 ep_ctx = xhci_get_ep_ctx(xhci, ep->vdev->out_ctx, ep_index); 2690 2691 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) { 2692 xhci_err(xhci, 2693 "ERROR Transfer event for disabled endpoint slot %u ep %u\n", 2694 slot_id, ep_index); 2695 goto err_out; 2696 } 2697 2698 /* Some transfer events don't always point to a trb, see xhci 4.17.4 */ 2699 if (!ep_ring) { 2700 switch (trb_comp_code) { 2701 case COMP_STALL_ERROR: 2702 case COMP_USB_TRANSACTION_ERROR: 2703 case COMP_INVALID_STREAM_TYPE_ERROR: 2704 case COMP_INVALID_STREAM_ID_ERROR: 2705 xhci_dbg(xhci, "Stream transaction error ep %u no id\n", 2706 ep_index); 2707 if (ep->err_count++ > MAX_SOFT_RETRY) 2708 xhci_handle_halted_endpoint(xhci, ep, NULL, 2709 EP_HARD_RESET); 2710 else 2711 xhci_handle_halted_endpoint(xhci, ep, NULL, 2712 EP_SOFT_RESET); 2713 goto cleanup; 2714 case COMP_RING_UNDERRUN: 2715 case COMP_RING_OVERRUN: 2716 case COMP_STOPPED_LENGTH_INVALID: 2717 goto cleanup; 2718 default: 2719 xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n", 2720 slot_id, ep_index); 2721 goto err_out; 2722 } 2723 } 2724 2725 /* Count current td numbers if ep->skip is set */ 2726 if (ep->skip) 2727 td_num += list_count_nodes(&ep_ring->td_list); 2728 2729 /* Look for common error cases */ 2730 switch (trb_comp_code) { 2731 /* Skip codes that require special handling depending on 2732 * transfer type 2733 */ 2734 case COMP_SUCCESS: 2735 if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) != 0) { 2736 trb_comp_code = COMP_SHORT_PACKET; 2737 xhci_dbg(xhci, "Successful completion on short TX for slot %u ep %u with last td short %d\n", 2738 slot_id, ep_index, ep_ring->last_td_was_short); 2739 } 2740 break; 2741 case COMP_SHORT_PACKET: 2742 break; 2743 /* Completion codes for endpoint stopped state */ 2744 case COMP_STOPPED: 2745 xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n", 2746 slot_id, ep_index); 2747 break; 2748 case COMP_STOPPED_LENGTH_INVALID: 2749 xhci_dbg(xhci, 2750 "Stopped on No-op or Link TRB for slot %u ep %u\n", 2751 slot_id, ep_index); 2752 break; 2753 case COMP_STOPPED_SHORT_PACKET: 2754 xhci_dbg(xhci, 2755 "Stopped with short packet transfer detected for slot %u ep %u\n", 2756 slot_id, ep_index); 2757 break; 2758 /* Completion codes for endpoint halted state */ 2759 case COMP_STALL_ERROR: 2760 xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id, 2761 ep_index); 2762 status = -EPIPE; 2763 break; 2764 case COMP_SPLIT_TRANSACTION_ERROR: 2765 xhci_dbg(xhci, "Split transaction error for slot %u ep %u\n", 2766 slot_id, ep_index); 2767 status = -EPROTO; 2768 break; 2769 case COMP_USB_TRANSACTION_ERROR: 2770 xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n", 2771 slot_id, ep_index); 2772 status = -EPROTO; 2773 break; 2774 case COMP_BABBLE_DETECTED_ERROR: 2775 xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n", 2776 slot_id, ep_index); 2777 status = -EOVERFLOW; 2778 break; 2779 /* Completion codes for endpoint error state */ 2780 case COMP_TRB_ERROR: 2781 xhci_warn(xhci, 2782 "WARN: TRB error for slot %u ep %u on endpoint\n", 2783 slot_id, ep_index); 2784 status = -EILSEQ; 2785 break; 2786 /* completion codes not indicating endpoint state change */ 2787 case COMP_DATA_BUFFER_ERROR: 2788 xhci_warn(xhci, 2789 "WARN: HC couldn't access mem fast enough for slot %u ep %u\n", 2790 slot_id, ep_index); 2791 status = -ENOSR; 2792 break; 2793 case COMP_BANDWIDTH_OVERRUN_ERROR: 2794 xhci_warn(xhci, 2795 "WARN: bandwidth overrun event for slot %u ep %u on endpoint\n", 2796 slot_id, ep_index); 2797 break; 2798 case COMP_ISOCH_BUFFER_OVERRUN: 2799 xhci_warn(xhci, 2800 "WARN: buffer overrun event for slot %u ep %u on endpoint", 2801 slot_id, ep_index); 2802 break; 2803 case COMP_RING_UNDERRUN: 2804 /* 2805 * When the Isoch ring is empty, the xHC will generate 2806 * a Ring Overrun Event for IN Isoch endpoint or Ring 2807 * Underrun Event for OUT Isoch endpoint. 2808 */ 2809 xhci_dbg(xhci, "underrun event on endpoint\n"); 2810 if (!list_empty(&ep_ring->td_list)) 2811 xhci_dbg(xhci, "Underrun Event for slot %d ep %d " 2812 "still with TDs queued?\n", 2813 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)), 2814 ep_index); 2815 goto cleanup; 2816 case COMP_RING_OVERRUN: 2817 xhci_dbg(xhci, "overrun event on endpoint\n"); 2818 if (!list_empty(&ep_ring->td_list)) 2819 xhci_dbg(xhci, "Overrun Event for slot %d ep %d " 2820 "still with TDs queued?\n", 2821 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)), 2822 ep_index); 2823 goto cleanup; 2824 case COMP_MISSED_SERVICE_ERROR: 2825 /* 2826 * When encounter missed service error, one or more isoc tds 2827 * may be missed by xHC. 2828 * Set skip flag of the ep_ring; Complete the missed tds as 2829 * short transfer when process the ep_ring next time. 2830 */ 2831 ep->skip = true; 2832 xhci_dbg(xhci, 2833 "Miss service interval error for slot %u ep %u, set skip flag\n", 2834 slot_id, ep_index); 2835 goto cleanup; 2836 case COMP_NO_PING_RESPONSE_ERROR: 2837 ep->skip = true; 2838 xhci_dbg(xhci, 2839 "No Ping response error for slot %u ep %u, Skip one Isoc TD\n", 2840 slot_id, ep_index); 2841 goto cleanup; 2842 2843 case COMP_INCOMPATIBLE_DEVICE_ERROR: 2844 /* needs disable slot command to recover */ 2845 xhci_warn(xhci, 2846 "WARN: detect an incompatible device for slot %u ep %u", 2847 slot_id, ep_index); 2848 status = -EPROTO; 2849 break; 2850 default: 2851 if (xhci_is_vendor_info_code(xhci, trb_comp_code)) { 2852 status = 0; 2853 break; 2854 } 2855 xhci_warn(xhci, 2856 "ERROR Unknown event condition %u for slot %u ep %u , HC probably busted\n", 2857 trb_comp_code, slot_id, ep_index); 2858 goto cleanup; 2859 } 2860 2861 do { 2862 /* This TRB should be in the TD at the head of this ring's 2863 * TD list. 2864 */ 2865 if (list_empty(&ep_ring->td_list)) { 2866 /* 2867 * Don't print wanings if it's due to a stopped endpoint 2868 * generating an extra completion event if the device 2869 * was suspended. Or, a event for the last TRB of a 2870 * short TD we already got a short event for. 2871 * The short TD is already removed from the TD list. 2872 */ 2873 2874 if (!(trb_comp_code == COMP_STOPPED || 2875 trb_comp_code == COMP_STOPPED_LENGTH_INVALID || 2876 ep_ring->last_td_was_short)) { 2877 xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n", 2878 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)), 2879 ep_index); 2880 } 2881 if (ep->skip) { 2882 ep->skip = false; 2883 xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n", 2884 slot_id, ep_index); 2885 } 2886 if (trb_comp_code == COMP_STALL_ERROR || 2887 xhci_requires_manual_halt_cleanup(xhci, ep_ctx, 2888 trb_comp_code)) { 2889 xhci_handle_halted_endpoint(xhci, ep, NULL, 2890 EP_HARD_RESET); 2891 } 2892 goto cleanup; 2893 } 2894 2895 /* We've skipped all the TDs on the ep ring when ep->skip set */ 2896 if (ep->skip && td_num == 0) { 2897 ep->skip = false; 2898 xhci_dbg(xhci, "All tds on the ep_ring skipped. Clear skip flag for slot %u ep %u.\n", 2899 slot_id, ep_index); 2900 goto cleanup; 2901 } 2902 2903 td = list_first_entry(&ep_ring->td_list, struct xhci_td, 2904 td_list); 2905 if (ep->skip) 2906 td_num--; 2907 2908 /* Is this a TRB in the currently executing TD? */ 2909 ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue, 2910 td->last_trb, ep_trb_dma, false); 2911 2912 /* 2913 * Skip the Force Stopped Event. The event_trb(event_dma) of FSE 2914 * is not in the current TD pointed by ep_ring->dequeue because 2915 * that the hardware dequeue pointer still at the previous TRB 2916 * of the current TD. The previous TRB maybe a Link TD or the 2917 * last TRB of the previous TD. The command completion handle 2918 * will take care the rest. 2919 */ 2920 if (!ep_seg && (trb_comp_code == COMP_STOPPED || 2921 trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) { 2922 goto cleanup; 2923 } 2924 2925 if (!ep_seg) { 2926 2927 if (ep->skip && usb_endpoint_xfer_isoc(&td->urb->ep->desc)) { 2928 skip_isoc_td(xhci, td, ep, status); 2929 goto cleanup; 2930 } 2931 2932 /* 2933 * Some hosts give a spurious success event after a short 2934 * transfer. Ignore it. 2935 */ 2936 if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) && 2937 ep_ring->last_td_was_short) { 2938 ep_ring->last_td_was_short = false; 2939 goto cleanup; 2940 } 2941 2942 /* 2943 * xhci 4.10.2 states isoc endpoints should continue 2944 * processing the next TD if there was an error mid TD. 2945 * So host like NEC don't generate an event for the last 2946 * isoc TRB even if the IOC flag is set. 2947 * xhci 4.9.1 states that if there are errors in mult-TRB 2948 * TDs xHC should generate an error for that TRB, and if xHC 2949 * proceeds to the next TD it should genete an event for 2950 * any TRB with IOC flag on the way. Other host follow this. 2951 * So this event might be for the next TD. 2952 */ 2953 if (td->error_mid_td && 2954 !list_is_last(&td->td_list, &ep_ring->td_list)) { 2955 struct xhci_td *td_next = list_next_entry(td, td_list); 2956 2957 ep_seg = trb_in_td(xhci, td_next->start_seg, td_next->first_trb, 2958 td_next->last_trb, ep_trb_dma, false); 2959 if (ep_seg) { 2960 /* give back previous TD, start handling new */ 2961 xhci_dbg(xhci, "Missing TD completion event after mid TD error\n"); 2962 ep_ring->dequeue = td->last_trb; 2963 ep_ring->deq_seg = td->last_trb_seg; 2964 inc_deq(xhci, ep_ring); 2965 xhci_td_cleanup(xhci, td, ep_ring, td->status); 2966 td = td_next; 2967 } 2968 } 2969 2970 if (!ep_seg) { 2971 /* HC is busted, give up! */ 2972 xhci_err(xhci, 2973 "ERROR Transfer event TRB DMA ptr not " 2974 "part of current TD ep_index %d " 2975 "comp_code %u\n", ep_index, 2976 trb_comp_code); 2977 trb_in_td(xhci, ep_ring->deq_seg, 2978 ep_ring->dequeue, td->last_trb, 2979 ep_trb_dma, true); 2980 return -ESHUTDOWN; 2981 } 2982 } 2983 if (trb_comp_code == COMP_SHORT_PACKET) 2984 ep_ring->last_td_was_short = true; 2985 else 2986 ep_ring->last_td_was_short = false; 2987 2988 if (ep->skip) { 2989 xhci_dbg(xhci, 2990 "Found td. Clear skip flag for slot %u ep %u.\n", 2991 slot_id, ep_index); 2992 ep->skip = false; 2993 } 2994 2995 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) / 2996 sizeof(*ep_trb)]; 2997 2998 trace_xhci_handle_transfer(ep_ring, 2999 (struct xhci_generic_trb *) ep_trb); 3000 3001 /* 3002 * No-op TRB could trigger interrupts in a case where 3003 * a URB was killed and a STALL_ERROR happens right 3004 * after the endpoint ring stopped. Reset the halted 3005 * endpoint. Otherwise, the endpoint remains stalled 3006 * indefinitely. 3007 */ 3008 3009 if (trb_is_noop(ep_trb)) { 3010 if (trb_comp_code == COMP_STALL_ERROR || 3011 xhci_requires_manual_halt_cleanup(xhci, ep_ctx, 3012 trb_comp_code)) 3013 xhci_handle_halted_endpoint(xhci, ep, td, 3014 EP_HARD_RESET); 3015 goto cleanup; 3016 } 3017 3018 td->status = status; 3019 3020 /* update the urb's actual_length and give back to the core */ 3021 if (usb_endpoint_xfer_control(&td->urb->ep->desc)) 3022 process_ctrl_td(xhci, ep, ep_ring, td, ep_trb, event); 3023 else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc)) 3024 process_isoc_td(xhci, ep, ep_ring, td, ep_trb, event); 3025 else 3026 process_bulk_intr_td(xhci, ep, ep_ring, td, ep_trb, event); 3027 cleanup: 3028 handling_skipped_tds = ep->skip && 3029 trb_comp_code != COMP_MISSED_SERVICE_ERROR && 3030 trb_comp_code != COMP_NO_PING_RESPONSE_ERROR; 3031 3032 /* 3033 * Do not update event ring dequeue pointer if we're in a loop 3034 * processing missed tds. 3035 */ 3036 if (!handling_skipped_tds) 3037 inc_deq(xhci, ir->event_ring); 3038 3039 /* 3040 * If ep->skip is set, it means there are missed tds on the 3041 * endpoint ring need to take care of. 3042 * Process them as short transfer until reach the td pointed by 3043 * the event. 3044 */ 3045 } while (handling_skipped_tds); 3046 3047 return 0; 3048 3049 err_out: 3050 xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n", 3051 (unsigned long long) xhci_trb_virt_to_dma( 3052 ir->event_ring->deq_seg, 3053 ir->event_ring->dequeue), 3054 lower_32_bits(le64_to_cpu(event->buffer)), 3055 upper_32_bits(le64_to_cpu(event->buffer)), 3056 le32_to_cpu(event->transfer_len), 3057 le32_to_cpu(event->flags)); 3058 return -ENODEV; 3059 } 3060 3061 /* 3062 * This function handles all OS-owned events on the event ring. It may drop 3063 * xhci->lock between event processing (e.g. to pass up port status changes). 3064 * Returns >0 for "possibly more events to process" (caller should call again), 3065 * otherwise 0 if done. In future, <0 returns should indicate error code. 3066 */ 3067 static int xhci_handle_event(struct xhci_hcd *xhci, struct xhci_interrupter *ir) 3068 { 3069 union xhci_trb *event; 3070 int update_ptrs = 1; 3071 u32 trb_type; 3072 int ret; 3073 3074 /* Event ring hasn't been allocated yet. */ 3075 if (!ir || !ir->event_ring || !ir->event_ring->dequeue) { 3076 xhci_err(xhci, "ERROR interrupter not ready\n"); 3077 return -ENOMEM; 3078 } 3079 3080 event = ir->event_ring->dequeue; 3081 /* Does the HC or OS own the TRB? */ 3082 if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) != 3083 ir->event_ring->cycle_state) 3084 return 0; 3085 3086 trace_xhci_handle_event(ir->event_ring, &event->generic); 3087 3088 /* 3089 * Barrier between reading the TRB_CYCLE (valid) flag above and any 3090 * speculative reads of the event's flags/data below. 3091 */ 3092 rmb(); 3093 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->event_cmd.flags)); 3094 /* FIXME: Handle more event types. */ 3095 3096 switch (trb_type) { 3097 case TRB_COMPLETION: 3098 handle_cmd_completion(xhci, &event->event_cmd); 3099 break; 3100 case TRB_PORT_STATUS: 3101 handle_port_status(xhci, ir, event); 3102 update_ptrs = 0; 3103 break; 3104 case TRB_TRANSFER: 3105 ret = handle_tx_event(xhci, ir, &event->trans_event); 3106 if (ret >= 0) 3107 update_ptrs = 0; 3108 break; 3109 case TRB_DEV_NOTE: 3110 handle_device_notification(xhci, event); 3111 break; 3112 default: 3113 if (trb_type >= TRB_VENDOR_DEFINED_LOW) 3114 handle_vendor_event(xhci, event, trb_type); 3115 else 3116 xhci_warn(xhci, "ERROR unknown event type %d\n", trb_type); 3117 } 3118 /* Any of the above functions may drop and re-acquire the lock, so check 3119 * to make sure a watchdog timer didn't mark the host as non-responsive. 3120 */ 3121 if (xhci->xhc_state & XHCI_STATE_DYING) { 3122 xhci_dbg(xhci, "xHCI host dying, returning from " 3123 "event handler.\n"); 3124 return 0; 3125 } 3126 3127 if (update_ptrs) 3128 /* Update SW event ring dequeue pointer */ 3129 inc_deq(xhci, ir->event_ring); 3130 3131 /* Are there more items on the event ring? Caller will call us again to 3132 * check. 3133 */ 3134 return 1; 3135 } 3136 3137 /* 3138 * Update Event Ring Dequeue Pointer: 3139 * - When all events have finished 3140 * - To avoid "Event Ring Full Error" condition 3141 */ 3142 static void xhci_update_erst_dequeue(struct xhci_hcd *xhci, 3143 struct xhci_interrupter *ir, 3144 union xhci_trb *event_ring_deq, 3145 bool clear_ehb) 3146 { 3147 u64 temp_64; 3148 dma_addr_t deq; 3149 3150 temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); 3151 /* If necessary, update the HW's version of the event ring deq ptr. */ 3152 if (event_ring_deq != ir->event_ring->dequeue) { 3153 deq = xhci_trb_virt_to_dma(ir->event_ring->deq_seg, 3154 ir->event_ring->dequeue); 3155 if (deq == 0) 3156 xhci_warn(xhci, "WARN something wrong with SW event ring dequeue ptr\n"); 3157 /* 3158 * Per 4.9.4, Software writes to the ERDP register shall 3159 * always advance the Event Ring Dequeue Pointer value. 3160 */ 3161 if ((temp_64 & (u64) ~ERST_PTR_MASK) == 3162 ((u64) deq & (u64) ~ERST_PTR_MASK)) 3163 return; 3164 3165 /* Update HC event ring dequeue pointer */ 3166 temp_64 &= ERST_DESI_MASK; 3167 temp_64 |= ((u64) deq & (u64) ~ERST_PTR_MASK); 3168 } 3169 3170 /* Clear the event handler busy flag (RW1C) */ 3171 if (clear_ehb) 3172 temp_64 |= ERST_EHB; 3173 xhci_write_64(xhci, temp_64, &ir->ir_set->erst_dequeue); 3174 } 3175 3176 /* 3177 * xHCI spec says we can get an interrupt, and if the HC has an error condition, 3178 * we might get bad data out of the event ring. Section 4.10.2.7 has a list of 3179 * indicators of an event TRB error, but we check the status *first* to be safe. 3180 */ 3181 irqreturn_t xhci_irq(struct usb_hcd *hcd) 3182 { 3183 struct xhci_hcd *xhci = hcd_to_xhci(hcd); 3184 union xhci_trb *event_ring_deq; 3185 struct xhci_interrupter *ir; 3186 irqreturn_t ret = IRQ_NONE; 3187 u64 temp_64; 3188 u32 status; 3189 int event_loop = 0; 3190 3191 spin_lock(&xhci->lock); 3192 /* Check if the xHC generated the interrupt, or the irq is shared */ 3193 status = readl(&xhci->op_regs->status); 3194 if (status == ~(u32)0) { 3195 xhci_hc_died(xhci); 3196 ret = IRQ_HANDLED; 3197 goto out; 3198 } 3199 3200 if (!(status & STS_EINT)) 3201 goto out; 3202 3203 if (status & STS_HCE) { 3204 xhci_warn(xhci, "WARNING: Host Controller Error\n"); 3205 goto out; 3206 } 3207 3208 if (status & STS_FATAL) { 3209 xhci_warn(xhci, "WARNING: Host System Error\n"); 3210 xhci_halt(xhci); 3211 ret = IRQ_HANDLED; 3212 goto out; 3213 } 3214 3215 /* 3216 * Clear the op reg interrupt status first, 3217 * so we can receive interrupts from other MSI-X interrupters. 3218 * Write 1 to clear the interrupt status. 3219 */ 3220 status |= STS_EINT; 3221 writel(status, &xhci->op_regs->status); 3222 3223 /* This is the handler of the primary interrupter */ 3224 ir = xhci->interrupter; 3225 if (!hcd->msi_enabled) { 3226 u32 irq_pending; 3227 irq_pending = readl(&ir->ir_set->irq_pending); 3228 irq_pending |= IMAN_IP; 3229 writel(irq_pending, &ir->ir_set->irq_pending); 3230 } 3231 3232 if (xhci->xhc_state & XHCI_STATE_DYING || 3233 xhci->xhc_state & XHCI_STATE_HALTED) { 3234 xhci_dbg(xhci, "xHCI dying, ignoring interrupt. " 3235 "Shouldn't IRQs be disabled?\n"); 3236 /* Clear the event handler busy flag (RW1C); 3237 * the event ring should be empty. 3238 */ 3239 temp_64 = xhci_read_64(xhci, &ir->ir_set->erst_dequeue); 3240 xhci_write_64(xhci, temp_64 | ERST_EHB, 3241 &ir->ir_set->erst_dequeue); 3242 ret = IRQ_HANDLED; 3243 goto out; 3244 } 3245 3246 event_ring_deq = ir->event_ring->dequeue; 3247 /* FIXME this should be a delayed service routine 3248 * that clears the EHB. 3249 */ 3250 while (xhci_handle_event(xhci, ir) > 0) { 3251 if (event_loop++ < TRBS_PER_SEGMENT / 2) 3252 continue; 3253 xhci_update_erst_dequeue(xhci, ir, event_ring_deq, false); 3254 event_ring_deq = ir->event_ring->dequeue; 3255 3256 /* ring is half-full, force isoc trbs to interrupt more often */ 3257 if (xhci->isoc_bei_interval > AVOID_BEI_INTERVAL_MIN) 3258 xhci->isoc_bei_interval = xhci->isoc_bei_interval / 2; 3259 3260 event_loop = 0; 3261 } 3262 3263 xhci_update_erst_dequeue(xhci, ir, event_ring_deq, true); 3264 ret = IRQ_HANDLED; 3265 3266 out: 3267 spin_unlock(&xhci->lock); 3268 3269 return ret; 3270 } 3271 3272 irqreturn_t xhci_msi_irq(int irq, void *hcd) 3273 { 3274 return xhci_irq(hcd); 3275 } 3276 EXPORT_SYMBOL_GPL(xhci_msi_irq); 3277 3278 /**** Endpoint Ring Operations ****/ 3279 3280 /* 3281 * Generic function for queueing a TRB on a ring. 3282 * The caller must have checked to make sure there's room on the ring. 3283 * 3284 * @more_trbs_coming: Will you enqueue more TRBs before calling 3285 * prepare_transfer()? 3286 */ 3287 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring, 3288 bool more_trbs_coming, 3289 u32 field1, u32 field2, u32 field3, u32 field4) 3290 { 3291 struct xhci_generic_trb *trb; 3292 3293 trb = &ring->enqueue->generic; 3294 trb->field[0] = cpu_to_le32(field1); 3295 trb->field[1] = cpu_to_le32(field2); 3296 trb->field[2] = cpu_to_le32(field3); 3297 /* make sure TRB is fully written before giving it to the controller */ 3298 wmb(); 3299 trb->field[3] = cpu_to_le32(field4); 3300 3301 trace_xhci_queue_trb(ring, trb); 3302 3303 inc_enq(xhci, ring, more_trbs_coming); 3304 } 3305 3306 /* 3307 * Does various checks on the endpoint ring, and makes it ready to queue num_trbs. 3308 * expand ring if it start to be full. 3309 */ 3310 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring, 3311 u32 ep_state, unsigned int num_trbs, gfp_t mem_flags) 3312 { 3313 unsigned int link_trb_count = 0; 3314 unsigned int new_segs = 0; 3315 3316 /* Make sure the endpoint has been added to xHC schedule */ 3317 switch (ep_state) { 3318 case EP_STATE_DISABLED: 3319 /* 3320 * USB core changed config/interfaces without notifying us, 3321 * or hardware is reporting the wrong state. 3322 */ 3323 xhci_warn(xhci, "WARN urb submitted to disabled ep\n"); 3324 return -ENOENT; 3325 case EP_STATE_ERROR: 3326 xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n"); 3327 /* FIXME event handling code for error needs to clear it */ 3328 /* XXX not sure if this should be -ENOENT or not */ 3329 return -EINVAL; 3330 case EP_STATE_HALTED: 3331 xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n"); 3332 break; 3333 case EP_STATE_STOPPED: 3334 case EP_STATE_RUNNING: 3335 break; 3336 default: 3337 xhci_err(xhci, "ERROR unknown endpoint state for ep\n"); 3338 /* 3339 * FIXME issue Configure Endpoint command to try to get the HC 3340 * back into a known state. 3341 */ 3342 return -EINVAL; 3343 } 3344 3345 if (ep_ring != xhci->cmd_ring) { 3346 new_segs = xhci_ring_expansion_needed(xhci, ep_ring, num_trbs); 3347 } else if (xhci_num_trbs_free(xhci, ep_ring) <= num_trbs) { 3348 xhci_err(xhci, "Do not support expand command ring\n"); 3349 return -ENOMEM; 3350 } 3351 3352 if (new_segs) { 3353 xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion, 3354 "ERROR no room on ep ring, try ring expansion"); 3355 if (xhci_ring_expansion(xhci, ep_ring, new_segs, mem_flags)) { 3356 xhci_err(xhci, "Ring expansion failed\n"); 3357 return -ENOMEM; 3358 } 3359 } 3360 3361 while (trb_is_link(ep_ring->enqueue)) { 3362 /* If we're not dealing with 0.95 hardware or isoc rings 3363 * on AMD 0.96 host, clear the chain bit. 3364 */ 3365 if (!xhci_link_trb_quirk(xhci) && 3366 !(ep_ring->type == TYPE_ISOC && 3367 (xhci->quirks & XHCI_AMD_0x96_HOST))) 3368 ep_ring->enqueue->link.control &= 3369 cpu_to_le32(~TRB_CHAIN); 3370 else 3371 ep_ring->enqueue->link.control |= 3372 cpu_to_le32(TRB_CHAIN); 3373 3374 wmb(); 3375 ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE); 3376 3377 /* Toggle the cycle bit after the last ring segment. */ 3378 if (link_trb_toggles_cycle(ep_ring->enqueue)) 3379 ep_ring->cycle_state ^= 1; 3380 3381 ep_ring->enq_seg = ep_ring->enq_seg->next; 3382 ep_ring->enqueue = ep_ring->enq_seg->trbs; 3383 3384 /* prevent infinite loop if all first trbs are link trbs */ 3385 if (link_trb_count++ > ep_ring->num_segs) { 3386 xhci_warn(xhci, "Ring is an endless link TRB loop\n"); 3387 return -EINVAL; 3388 } 3389 } 3390 3391 if (last_trb_on_seg(ep_ring->enq_seg, ep_ring->enqueue)) { 3392 xhci_warn(xhci, "Missing link TRB at end of ring segment\n"); 3393 return -EINVAL; 3394 } 3395 3396 return 0; 3397 } 3398 3399 static int prepare_transfer(struct xhci_hcd *xhci, 3400 struct xhci_virt_device *xdev, 3401 unsigned int ep_index, 3402 unsigned int stream_id, 3403 unsigned int num_trbs, 3404 struct urb *urb, 3405 unsigned int td_index, 3406 gfp_t mem_flags) 3407 { 3408 int ret; 3409 struct urb_priv *urb_priv; 3410 struct xhci_td *td; 3411 struct xhci_ring *ep_ring; 3412 struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); 3413 3414 ep_ring = xhci_triad_to_transfer_ring(xhci, xdev->slot_id, ep_index, 3415 stream_id); 3416 if (!ep_ring) { 3417 xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n", 3418 stream_id); 3419 return -EINVAL; 3420 } 3421 3422 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), 3423 num_trbs, mem_flags); 3424 if (ret) 3425 return ret; 3426 3427 urb_priv = urb->hcpriv; 3428 td = &urb_priv->td[td_index]; 3429 3430 INIT_LIST_HEAD(&td->td_list); 3431 INIT_LIST_HEAD(&td->cancelled_td_list); 3432 3433 if (td_index == 0) { 3434 ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb); 3435 if (unlikely(ret)) 3436 return ret; 3437 } 3438 3439 td->urb = urb; 3440 /* Add this TD to the tail of the endpoint ring's TD list */ 3441 list_add_tail(&td->td_list, &ep_ring->td_list); 3442 td->start_seg = ep_ring->enq_seg; 3443 td->first_trb = ep_ring->enqueue; 3444 3445 return 0; 3446 } 3447 3448 unsigned int count_trbs(u64 addr, u64 len) 3449 { 3450 unsigned int num_trbs; 3451 3452 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)), 3453 TRB_MAX_BUFF_SIZE); 3454 if (num_trbs == 0) 3455 num_trbs++; 3456 3457 return num_trbs; 3458 } 3459 3460 static inline unsigned int count_trbs_needed(struct urb *urb) 3461 { 3462 return count_trbs(urb->transfer_dma, urb->transfer_buffer_length); 3463 } 3464 3465 static unsigned int count_sg_trbs_needed(struct urb *urb) 3466 { 3467 struct scatterlist *sg; 3468 unsigned int i, len, full_len, num_trbs = 0; 3469 3470 full_len = urb->transfer_buffer_length; 3471 3472 for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) { 3473 len = sg_dma_len(sg); 3474 num_trbs += count_trbs(sg_dma_address(sg), len); 3475 len = min_t(unsigned int, len, full_len); 3476 full_len -= len; 3477 if (full_len == 0) 3478 break; 3479 } 3480 3481 return num_trbs; 3482 } 3483 3484 static unsigned int count_isoc_trbs_needed(struct urb *urb, int i) 3485 { 3486 u64 addr, len; 3487 3488 addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset); 3489 len = urb->iso_frame_desc[i].length; 3490 3491 return count_trbs(addr, len); 3492 } 3493 3494 static void check_trb_math(struct urb *urb, int running_total) 3495 { 3496 if (unlikely(running_total != urb->transfer_buffer_length)) 3497 dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, " 3498 "queued %#x (%d), asked for %#x (%d)\n", 3499 __func__, 3500 urb->ep->desc.bEndpointAddress, 3501 running_total, running_total, 3502 urb->transfer_buffer_length, 3503 urb->transfer_buffer_length); 3504 } 3505 3506 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id, 3507 unsigned int ep_index, unsigned int stream_id, int start_cycle, 3508 struct xhci_generic_trb *start_trb) 3509 { 3510 /* 3511 * Pass all the TRBs to the hardware at once and make sure this write 3512 * isn't reordered. 3513 */ 3514 wmb(); 3515 if (start_cycle) 3516 start_trb->field[3] |= cpu_to_le32(start_cycle); 3517 else 3518 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); 3519 xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id); 3520 } 3521 3522 static void check_interval(struct xhci_hcd *xhci, struct urb *urb, 3523 struct xhci_ep_ctx *ep_ctx) 3524 { 3525 int xhci_interval; 3526 int ep_interval; 3527 3528 xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info)); 3529 ep_interval = urb->interval; 3530 3531 /* Convert to microframes */ 3532 if (urb->dev->speed == USB_SPEED_LOW || 3533 urb->dev->speed == USB_SPEED_FULL) 3534 ep_interval *= 8; 3535 3536 /* FIXME change this to a warning and a suggestion to use the new API 3537 * to set the polling interval (once the API is added). 3538 */ 3539 if (xhci_interval != ep_interval) { 3540 dev_dbg_ratelimited(&urb->dev->dev, 3541 "Driver uses different interval (%d microframe%s) than xHCI (%d microframe%s)\n", 3542 ep_interval, ep_interval == 1 ? "" : "s", 3543 xhci_interval, xhci_interval == 1 ? "" : "s"); 3544 urb->interval = xhci_interval; 3545 /* Convert back to frames for LS/FS devices */ 3546 if (urb->dev->speed == USB_SPEED_LOW || 3547 urb->dev->speed == USB_SPEED_FULL) 3548 urb->interval /= 8; 3549 } 3550 } 3551 3552 /* 3553 * xHCI uses normal TRBs for both bulk and interrupt. When the interrupt 3554 * endpoint is to be serviced, the xHC will consume (at most) one TD. A TD 3555 * (comprised of sg list entries) can take several service intervals to 3556 * transmit. 3557 */ 3558 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags, 3559 struct urb *urb, int slot_id, unsigned int ep_index) 3560 { 3561 struct xhci_ep_ctx *ep_ctx; 3562 3563 ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index); 3564 check_interval(xhci, urb, ep_ctx); 3565 3566 return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index); 3567 } 3568 3569 /* 3570 * For xHCI 1.0 host controllers, TD size is the number of max packet sized 3571 * packets remaining in the TD (*not* including this TRB). 3572 * 3573 * Total TD packet count = total_packet_count = 3574 * DIV_ROUND_UP(TD size in bytes / wMaxPacketSize) 3575 * 3576 * Packets transferred up to and including this TRB = packets_transferred = 3577 * rounddown(total bytes transferred including this TRB / wMaxPacketSize) 3578 * 3579 * TD size = total_packet_count - packets_transferred 3580 * 3581 * For xHCI 0.96 and older, TD size field should be the remaining bytes 3582 * including this TRB, right shifted by 10 3583 * 3584 * For all hosts it must fit in bits 21:17, so it can't be bigger than 31. 3585 * This is taken care of in the TRB_TD_SIZE() macro 3586 * 3587 * The last TRB in a TD must have the TD size set to zero. 3588 */ 3589 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred, 3590 int trb_buff_len, unsigned int td_total_len, 3591 struct urb *urb, bool more_trbs_coming) 3592 { 3593 u32 maxp, total_packet_count; 3594 3595 /* MTK xHCI 0.96 contains some features from 1.0 */ 3596 if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST)) 3597 return ((td_total_len - transferred) >> 10); 3598 3599 /* One TRB with a zero-length data packet. */ 3600 if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) || 3601 trb_buff_len == td_total_len) 3602 return 0; 3603 3604 /* for MTK xHCI 0.96, TD size include this TRB, but not in 1.x */ 3605 if ((xhci->quirks & XHCI_MTK_HOST) && (xhci->hci_version < 0x100)) 3606 trb_buff_len = 0; 3607 3608 maxp = usb_endpoint_maxp(&urb->ep->desc); 3609 total_packet_count = DIV_ROUND_UP(td_total_len, maxp); 3610 3611 /* Queueing functions don't count the current TRB into transferred */ 3612 return (total_packet_count - ((transferred + trb_buff_len) / maxp)); 3613 } 3614 3615 3616 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len, 3617 u32 *trb_buff_len, struct xhci_segment *seg) 3618 { 3619 struct device *dev = xhci_to_hcd(xhci)->self.sysdev; 3620 unsigned int unalign; 3621 unsigned int max_pkt; 3622 u32 new_buff_len; 3623 size_t len; 3624 3625 max_pkt = usb_endpoint_maxp(&urb->ep->desc); 3626 unalign = (enqd_len + *trb_buff_len) % max_pkt; 3627 3628 /* we got lucky, last normal TRB data on segment is packet aligned */ 3629 if (unalign == 0) 3630 return 0; 3631 3632 xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n", 3633 unalign, *trb_buff_len); 3634 3635 /* is the last nornal TRB alignable by splitting it */ 3636 if (*trb_buff_len > unalign) { 3637 *trb_buff_len -= unalign; 3638 xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len); 3639 return 0; 3640 } 3641 3642 /* 3643 * We want enqd_len + trb_buff_len to sum up to a number aligned to 3644 * number which is divisible by the endpoint's wMaxPacketSize. IOW: 3645 * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0. 3646 */ 3647 new_buff_len = max_pkt - (enqd_len % max_pkt); 3648 3649 if (new_buff_len > (urb->transfer_buffer_length - enqd_len)) 3650 new_buff_len = (urb->transfer_buffer_length - enqd_len); 3651 3652 /* create a max max_pkt sized bounce buffer pointed to by last trb */ 3653 if (usb_urb_dir_out(urb)) { 3654 if (urb->num_sgs) { 3655 len = sg_pcopy_to_buffer(urb->sg, urb->num_sgs, 3656 seg->bounce_buf, new_buff_len, enqd_len); 3657 if (len != new_buff_len) 3658 xhci_warn(xhci, "WARN Wrong bounce buffer write length: %zu != %d\n", 3659 len, new_buff_len); 3660 } else { 3661 memcpy(seg->bounce_buf, urb->transfer_buffer + enqd_len, new_buff_len); 3662 } 3663 3664 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, 3665 max_pkt, DMA_TO_DEVICE); 3666 } else { 3667 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, 3668 max_pkt, DMA_FROM_DEVICE); 3669 } 3670 3671 if (dma_mapping_error(dev, seg->bounce_dma)) { 3672 /* try without aligning. Some host controllers survive */ 3673 xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n"); 3674 return 0; 3675 } 3676 *trb_buff_len = new_buff_len; 3677 seg->bounce_len = new_buff_len; 3678 seg->bounce_offs = enqd_len; 3679 3680 xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len); 3681 3682 return 1; 3683 } 3684 3685 /* This is very similar to what ehci-q.c qtd_fill() does */ 3686 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags, 3687 struct urb *urb, int slot_id, unsigned int ep_index) 3688 { 3689 struct xhci_ring *ring; 3690 struct urb_priv *urb_priv; 3691 struct xhci_td *td; 3692 struct xhci_generic_trb *start_trb; 3693 struct scatterlist *sg = NULL; 3694 bool more_trbs_coming = true; 3695 bool need_zero_pkt = false; 3696 bool first_trb = true; 3697 unsigned int num_trbs; 3698 unsigned int start_cycle, num_sgs = 0; 3699 unsigned int enqd_len, block_len, trb_buff_len, full_len; 3700 int sent_len, ret; 3701 u32 field, length_field, remainder; 3702 u64 addr, send_addr; 3703 3704 ring = xhci_urb_to_transfer_ring(xhci, urb); 3705 if (!ring) 3706 return -EINVAL; 3707 3708 full_len = urb->transfer_buffer_length; 3709 /* If we have scatter/gather list, we use it. */ 3710 if (urb->num_sgs && !(urb->transfer_flags & URB_DMA_MAP_SINGLE)) { 3711 num_sgs = urb->num_mapped_sgs; 3712 sg = urb->sg; 3713 addr = (u64) sg_dma_address(sg); 3714 block_len = sg_dma_len(sg); 3715 num_trbs = count_sg_trbs_needed(urb); 3716 } else { 3717 num_trbs = count_trbs_needed(urb); 3718 addr = (u64) urb->transfer_dma; 3719 block_len = full_len; 3720 } 3721 ret = prepare_transfer(xhci, xhci->devs[slot_id], 3722 ep_index, urb->stream_id, 3723 num_trbs, urb, 0, mem_flags); 3724 if (unlikely(ret < 0)) 3725 return ret; 3726 3727 urb_priv = urb->hcpriv; 3728 3729 /* Deal with URB_ZERO_PACKET - need one more td/trb */ 3730 if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1) 3731 need_zero_pkt = true; 3732 3733 td = &urb_priv->td[0]; 3734 3735 /* 3736 * Don't give the first TRB to the hardware (by toggling the cycle bit) 3737 * until we've finished creating all the other TRBs. The ring's cycle 3738 * state may change as we enqueue the other TRBs, so save it too. 3739 */ 3740 start_trb = &ring->enqueue->generic; 3741 start_cycle = ring->cycle_state; 3742 send_addr = addr; 3743 3744 /* Queue the TRBs, even if they are zero-length */ 3745 for (enqd_len = 0; first_trb || enqd_len < full_len; 3746 enqd_len += trb_buff_len) { 3747 field = TRB_TYPE(TRB_NORMAL); 3748 3749 /* TRB buffer should not cross 64KB boundaries */ 3750 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr); 3751 trb_buff_len = min_t(unsigned int, trb_buff_len, block_len); 3752 3753 if (enqd_len + trb_buff_len > full_len) 3754 trb_buff_len = full_len - enqd_len; 3755 3756 /* Don't change the cycle bit of the first TRB until later */ 3757 if (first_trb) { 3758 first_trb = false; 3759 if (start_cycle == 0) 3760 field |= TRB_CYCLE; 3761 } else 3762 field |= ring->cycle_state; 3763 3764 /* Chain all the TRBs together; clear the chain bit in the last 3765 * TRB to indicate it's the last TRB in the chain. 3766 */ 3767 if (enqd_len + trb_buff_len < full_len) { 3768 field |= TRB_CHAIN; 3769 if (trb_is_link(ring->enqueue + 1)) { 3770 if (xhci_align_td(xhci, urb, enqd_len, 3771 &trb_buff_len, 3772 ring->enq_seg)) { 3773 send_addr = ring->enq_seg->bounce_dma; 3774 /* assuming TD won't span 2 segs */ 3775 td->bounce_seg = ring->enq_seg; 3776 } 3777 } 3778 } 3779 if (enqd_len + trb_buff_len >= full_len) { 3780 field &= ~TRB_CHAIN; 3781 field |= TRB_IOC; 3782 more_trbs_coming = false; 3783 td->last_trb = ring->enqueue; 3784 td->last_trb_seg = ring->enq_seg; 3785 if (xhci_urb_suitable_for_idt(urb)) { 3786 memcpy(&send_addr, urb->transfer_buffer, 3787 trb_buff_len); 3788 le64_to_cpus(&send_addr); 3789 field |= TRB_IDT; 3790 } 3791 } 3792 3793 /* Only set interrupt on short packet for IN endpoints */ 3794 if (usb_urb_dir_in(urb)) 3795 field |= TRB_ISP; 3796 3797 /* Set the TRB length, TD size, and interrupter fields. */ 3798 remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len, 3799 full_len, urb, more_trbs_coming); 3800 3801 length_field = TRB_LEN(trb_buff_len) | 3802 TRB_TD_SIZE(remainder) | 3803 TRB_INTR_TARGET(0); 3804 3805 queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt, 3806 lower_32_bits(send_addr), 3807 upper_32_bits(send_addr), 3808 length_field, 3809 field); 3810 td->num_trbs++; 3811 addr += trb_buff_len; 3812 sent_len = trb_buff_len; 3813 3814 while (sg && sent_len >= block_len) { 3815 /* New sg entry */ 3816 --num_sgs; 3817 sent_len -= block_len; 3818 sg = sg_next(sg); 3819 if (num_sgs != 0 && sg) { 3820 block_len = sg_dma_len(sg); 3821 addr = (u64) sg_dma_address(sg); 3822 addr += sent_len; 3823 } 3824 } 3825 block_len -= sent_len; 3826 send_addr = addr; 3827 } 3828 3829 if (need_zero_pkt) { 3830 ret = prepare_transfer(xhci, xhci->devs[slot_id], 3831 ep_index, urb->stream_id, 3832 1, urb, 1, mem_flags); 3833 urb_priv->td[1].last_trb = ring->enqueue; 3834 urb_priv->td[1].last_trb_seg = ring->enq_seg; 3835 field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC; 3836 queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field); 3837 urb_priv->td[1].num_trbs++; 3838 } 3839 3840 check_trb_math(urb, enqd_len); 3841 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, 3842 start_cycle, start_trb); 3843 return 0; 3844 } 3845 3846 /* Caller must have locked xhci->lock */ 3847 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags, 3848 struct urb *urb, int slot_id, unsigned int ep_index) 3849 { 3850 struct xhci_ring *ep_ring; 3851 int num_trbs; 3852 int ret; 3853 struct usb_ctrlrequest *setup; 3854 struct xhci_generic_trb *start_trb; 3855 int start_cycle; 3856 u32 field; 3857 struct urb_priv *urb_priv; 3858 struct xhci_td *td; 3859 3860 ep_ring = xhci_urb_to_transfer_ring(xhci, urb); 3861 if (!ep_ring) 3862 return -EINVAL; 3863 3864 /* 3865 * Need to copy setup packet into setup TRB, so we can't use the setup 3866 * DMA address. 3867 */ 3868 if (!urb->setup_packet) 3869 return -EINVAL; 3870 3871 if ((xhci->quirks & XHCI_ETRON_HOST) && 3872 urb->dev->speed >= USB_SPEED_SUPER) { 3873 /* 3874 * If next available TRB is the Link TRB in the ring segment then 3875 * enqueue a No Op TRB, this can prevent the Setup and Data Stage 3876 * TRB to be breaked by the Link TRB. 3877 */ 3878 if (trb_is_link(ep_ring->enqueue + 1)) { 3879 field = TRB_TYPE(TRB_TR_NOOP) | ep_ring->cycle_state; 3880 queue_trb(xhci, ep_ring, false, 0, 0, 3881 TRB_INTR_TARGET(0), field); 3882 } 3883 } 3884 3885 /* 1 TRB for setup, 1 for status */ 3886 num_trbs = 2; 3887 /* 3888 * Don't need to check if we need additional event data and normal TRBs, 3889 * since data in control transfers will never get bigger than 16MB 3890 * XXX: can we get a buffer that crosses 64KB boundaries? 3891 */ 3892 if (urb->transfer_buffer_length > 0) 3893 num_trbs++; 3894 ret = prepare_transfer(xhci, xhci->devs[slot_id], 3895 ep_index, urb->stream_id, 3896 num_trbs, urb, 0, mem_flags); 3897 if (ret < 0) 3898 return ret; 3899 3900 urb_priv = urb->hcpriv; 3901 td = &urb_priv->td[0]; 3902 td->num_trbs = num_trbs; 3903 3904 /* 3905 * Don't give the first TRB to the hardware (by toggling the cycle bit) 3906 * until we've finished creating all the other TRBs. The ring's cycle 3907 * state may change as we enqueue the other TRBs, so save it too. 3908 */ 3909 start_trb = &ep_ring->enqueue->generic; 3910 start_cycle = ep_ring->cycle_state; 3911 3912 /* Queue setup TRB - see section 6.4.1.2.1 */ 3913 /* FIXME better way to translate setup_packet into two u32 fields? */ 3914 setup = (struct usb_ctrlrequest *) urb->setup_packet; 3915 field = 0; 3916 field |= TRB_IDT | TRB_TYPE(TRB_SETUP); 3917 if (start_cycle == 0) 3918 field |= 0x1; 3919 3920 /* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */ 3921 if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) { 3922 if (urb->transfer_buffer_length > 0) { 3923 if (setup->bRequestType & USB_DIR_IN) 3924 field |= TRB_TX_TYPE(TRB_DATA_IN); 3925 else 3926 field |= TRB_TX_TYPE(TRB_DATA_OUT); 3927 } 3928 } 3929 3930 queue_trb(xhci, ep_ring, true, 3931 setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16, 3932 le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16, 3933 TRB_LEN(8) | TRB_INTR_TARGET(0), 3934 /* Immediate data in pointer */ 3935 field); 3936 3937 /* If there's data, queue data TRBs */ 3938 /* Only set interrupt on short packet for IN endpoints */ 3939 if (usb_urb_dir_in(urb)) 3940 field = TRB_ISP | TRB_TYPE(TRB_DATA); 3941 else 3942 field = TRB_TYPE(TRB_DATA); 3943 3944 if (urb->transfer_buffer_length > 0) { 3945 u32 length_field, remainder; 3946 u64 addr; 3947 3948 if (xhci_urb_suitable_for_idt(urb)) { 3949 memcpy(&addr, urb->transfer_buffer, 3950 urb->transfer_buffer_length); 3951 le64_to_cpus(&addr); 3952 field |= TRB_IDT; 3953 } else { 3954 addr = (u64) urb->transfer_dma; 3955 } 3956 3957 remainder = xhci_td_remainder(xhci, 0, 3958 urb->transfer_buffer_length, 3959 urb->transfer_buffer_length, 3960 urb, 1); 3961 length_field = TRB_LEN(urb->transfer_buffer_length) | 3962 TRB_TD_SIZE(remainder) | 3963 TRB_INTR_TARGET(0); 3964 if (setup->bRequestType & USB_DIR_IN) 3965 field |= TRB_DIR_IN; 3966 queue_trb(xhci, ep_ring, true, 3967 lower_32_bits(addr), 3968 upper_32_bits(addr), 3969 length_field, 3970 field | ep_ring->cycle_state); 3971 } 3972 3973 /* Save the DMA address of the last TRB in the TD */ 3974 td->last_trb = ep_ring->enqueue; 3975 td->last_trb_seg = ep_ring->enq_seg; 3976 3977 /* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */ 3978 /* If the device sent data, the status stage is an OUT transfer */ 3979 if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN) 3980 field = 0; 3981 else 3982 field = TRB_DIR_IN; 3983 queue_trb(xhci, ep_ring, false, 3984 0, 3985 0, 3986 TRB_INTR_TARGET(0), 3987 /* Event on completion */ 3988 field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state); 3989 3990 giveback_first_trb(xhci, slot_id, ep_index, 0, 3991 start_cycle, start_trb); 3992 return 0; 3993 } 3994 3995 /* 3996 * The transfer burst count field of the isochronous TRB defines the number of 3997 * bursts that are required to move all packets in this TD. Only SuperSpeed 3998 * devices can burst up to bMaxBurst number of packets per service interval. 3999 * This field is zero based, meaning a value of zero in the field means one 4000 * burst. Basically, for everything but SuperSpeed devices, this field will be 4001 * zero. Only xHCI 1.0 host controllers support this field. 4002 */ 4003 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci, 4004 struct urb *urb, unsigned int total_packet_count) 4005 { 4006 unsigned int max_burst; 4007 4008 if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER) 4009 return 0; 4010 4011 max_burst = urb->ep->ss_ep_comp.bMaxBurst; 4012 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1; 4013 } 4014 4015 /* 4016 * Returns the number of packets in the last "burst" of packets. This field is 4017 * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so 4018 * the last burst packet count is equal to the total number of packets in the 4019 * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst 4020 * must contain (bMaxBurst + 1) number of packets, but the last burst can 4021 * contain 1 to (bMaxBurst + 1) packets. 4022 */ 4023 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci, 4024 struct urb *urb, unsigned int total_packet_count) 4025 { 4026 unsigned int max_burst; 4027 unsigned int residue; 4028 4029 if (xhci->hci_version < 0x100) 4030 return 0; 4031 4032 if (urb->dev->speed >= USB_SPEED_SUPER) { 4033 /* bMaxBurst is zero based: 0 means 1 packet per burst */ 4034 max_burst = urb->ep->ss_ep_comp.bMaxBurst; 4035 residue = total_packet_count % (max_burst + 1); 4036 /* If residue is zero, the last burst contains (max_burst + 1) 4037 * number of packets, but the TLBPC field is zero-based. 4038 */ 4039 if (residue == 0) 4040 return max_burst; 4041 return residue - 1; 4042 } 4043 if (total_packet_count == 0) 4044 return 0; 4045 return total_packet_count - 1; 4046 } 4047 4048 /* 4049 * Calculates Frame ID field of the isochronous TRB identifies the 4050 * target frame that the Interval associated with this Isochronous 4051 * Transfer Descriptor will start on. Refer to 4.11.2.5 in 1.1 spec. 4052 * 4053 * Returns actual frame id on success, negative value on error. 4054 */ 4055 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci, 4056 struct urb *urb, int index) 4057 { 4058 int start_frame, ist, ret = 0; 4059 int start_frame_id, end_frame_id, current_frame_id; 4060 4061 if (urb->dev->speed == USB_SPEED_LOW || 4062 urb->dev->speed == USB_SPEED_FULL) 4063 start_frame = urb->start_frame + index * urb->interval; 4064 else 4065 start_frame = (urb->start_frame + index * urb->interval) >> 3; 4066 4067 /* Isochronous Scheduling Threshold (IST, bits 0~3 in HCSPARAMS2): 4068 * 4069 * If bit [3] of IST is cleared to '0', software can add a TRB no 4070 * later than IST[2:0] Microframes before that TRB is scheduled to 4071 * be executed. 4072 * If bit [3] of IST is set to '1', software can add a TRB no later 4073 * than IST[2:0] Frames before that TRB is scheduled to be executed. 4074 */ 4075 ist = HCS_IST(xhci->hcs_params2) & 0x7; 4076 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) 4077 ist <<= 3; 4078 4079 /* Software shall not schedule an Isoch TD with a Frame ID value that 4080 * is less than the Start Frame ID or greater than the End Frame ID, 4081 * where: 4082 * 4083 * End Frame ID = (Current MFINDEX register value + 895 ms.) MOD 2048 4084 * Start Frame ID = (Current MFINDEX register value + IST + 1) MOD 2048 4085 * 4086 * Both the End Frame ID and Start Frame ID values are calculated 4087 * in microframes. When software determines the valid Frame ID value; 4088 * The End Frame ID value should be rounded down to the nearest Frame 4089 * boundary, and the Start Frame ID value should be rounded up to the 4090 * nearest Frame boundary. 4091 */ 4092 current_frame_id = readl(&xhci->run_regs->microframe_index); 4093 start_frame_id = roundup(current_frame_id + ist + 1, 8); 4094 end_frame_id = rounddown(current_frame_id + 895 * 8, 8); 4095 4096 start_frame &= 0x7ff; 4097 start_frame_id = (start_frame_id >> 3) & 0x7ff; 4098 end_frame_id = (end_frame_id >> 3) & 0x7ff; 4099 4100 xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n", 4101 __func__, index, readl(&xhci->run_regs->microframe_index), 4102 start_frame_id, end_frame_id, start_frame); 4103 4104 if (start_frame_id < end_frame_id) { 4105 if (start_frame > end_frame_id || 4106 start_frame < start_frame_id) 4107 ret = -EINVAL; 4108 } else if (start_frame_id > end_frame_id) { 4109 if ((start_frame > end_frame_id && 4110 start_frame < start_frame_id)) 4111 ret = -EINVAL; 4112 } else { 4113 ret = -EINVAL; 4114 } 4115 4116 if (index == 0) { 4117 if (ret == -EINVAL || start_frame == start_frame_id) { 4118 start_frame = start_frame_id + 1; 4119 if (urb->dev->speed == USB_SPEED_LOW || 4120 urb->dev->speed == USB_SPEED_FULL) 4121 urb->start_frame = start_frame; 4122 else 4123 urb->start_frame = start_frame << 3; 4124 ret = 0; 4125 } 4126 } 4127 4128 if (ret) { 4129 xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n", 4130 start_frame, current_frame_id, index, 4131 start_frame_id, end_frame_id); 4132 xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n"); 4133 return ret; 4134 } 4135 4136 return start_frame; 4137 } 4138 4139 /* Check if we should generate event interrupt for a TD in an isoc URB */ 4140 static bool trb_block_event_intr(struct xhci_hcd *xhci, int num_tds, int i) 4141 { 4142 if (xhci->hci_version < 0x100) 4143 return false; 4144 /* always generate an event interrupt for the last TD */ 4145 if (i == num_tds - 1) 4146 return false; 4147 /* 4148 * If AVOID_BEI is set the host handles full event rings poorly, 4149 * generate an event at least every 8th TD to clear the event ring 4150 */ 4151 if (i && xhci->quirks & XHCI_AVOID_BEI) 4152 return !!(i % xhci->isoc_bei_interval); 4153 4154 return true; 4155 } 4156 4157 /* This is for isoc transfer */ 4158 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags, 4159 struct urb *urb, int slot_id, unsigned int ep_index) 4160 { 4161 struct xhci_ring *ep_ring; 4162 struct urb_priv *urb_priv; 4163 struct xhci_td *td; 4164 int num_tds, trbs_per_td; 4165 struct xhci_generic_trb *start_trb; 4166 bool first_trb; 4167 int start_cycle; 4168 u32 field, length_field; 4169 int running_total, trb_buff_len, td_len, td_remain_len, ret; 4170 u64 start_addr, addr; 4171 int i, j; 4172 bool more_trbs_coming; 4173 struct xhci_virt_ep *xep; 4174 int frame_id; 4175 4176 xep = &xhci->devs[slot_id]->eps[ep_index]; 4177 ep_ring = xhci->devs[slot_id]->eps[ep_index].ring; 4178 4179 num_tds = urb->number_of_packets; 4180 if (num_tds < 1) { 4181 xhci_dbg(xhci, "Isoc URB with zero packets?\n"); 4182 return -EINVAL; 4183 } 4184 start_addr = (u64) urb->transfer_dma; 4185 start_trb = &ep_ring->enqueue->generic; 4186 start_cycle = ep_ring->cycle_state; 4187 4188 urb_priv = urb->hcpriv; 4189 /* Queue the TRBs for each TD, even if they are zero-length */ 4190 for (i = 0; i < num_tds; i++) { 4191 unsigned int total_pkt_count, max_pkt; 4192 unsigned int burst_count, last_burst_pkt_count; 4193 u32 sia_frame_id; 4194 4195 first_trb = true; 4196 running_total = 0; 4197 addr = start_addr + urb->iso_frame_desc[i].offset; 4198 td_len = urb->iso_frame_desc[i].length; 4199 td_remain_len = td_len; 4200 max_pkt = usb_endpoint_maxp(&urb->ep->desc); 4201 total_pkt_count = DIV_ROUND_UP(td_len, max_pkt); 4202 4203 /* A zero-length transfer still involves at least one packet. */ 4204 if (total_pkt_count == 0) 4205 total_pkt_count++; 4206 burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count); 4207 last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci, 4208 urb, total_pkt_count); 4209 4210 trbs_per_td = count_isoc_trbs_needed(urb, i); 4211 4212 ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index, 4213 urb->stream_id, trbs_per_td, urb, i, mem_flags); 4214 if (ret < 0) { 4215 if (i == 0) 4216 return ret; 4217 goto cleanup; 4218 } 4219 td = &urb_priv->td[i]; 4220 td->num_trbs = trbs_per_td; 4221 /* use SIA as default, if frame id is used overwrite it */ 4222 sia_frame_id = TRB_SIA; 4223 if (!(urb->transfer_flags & URB_ISO_ASAP) && 4224 HCC_CFC(xhci->hcc_params)) { 4225 frame_id = xhci_get_isoc_frame_id(xhci, urb, i); 4226 if (frame_id >= 0) 4227 sia_frame_id = TRB_FRAME_ID(frame_id); 4228 } 4229 /* 4230 * Set isoc specific data for the first TRB in a TD. 4231 * Prevent HW from getting the TRBs by keeping the cycle state 4232 * inverted in the first TDs isoc TRB. 4233 */ 4234 field = TRB_TYPE(TRB_ISOC) | 4235 TRB_TLBPC(last_burst_pkt_count) | 4236 sia_frame_id | 4237 (i ? ep_ring->cycle_state : !start_cycle); 4238 4239 /* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */ 4240 if (!xep->use_extended_tbc) 4241 field |= TRB_TBC(burst_count); 4242 4243 /* fill the rest of the TRB fields, and remaining normal TRBs */ 4244 for (j = 0; j < trbs_per_td; j++) { 4245 u32 remainder = 0; 4246 4247 /* only first TRB is isoc, overwrite otherwise */ 4248 if (!first_trb) 4249 field = TRB_TYPE(TRB_NORMAL) | 4250 ep_ring->cycle_state; 4251 4252 /* Only set interrupt on short packet for IN EPs */ 4253 if (usb_urb_dir_in(urb)) 4254 field |= TRB_ISP; 4255 4256 /* Set the chain bit for all except the last TRB */ 4257 if (j < trbs_per_td - 1) { 4258 more_trbs_coming = true; 4259 field |= TRB_CHAIN; 4260 } else { 4261 more_trbs_coming = false; 4262 td->last_trb = ep_ring->enqueue; 4263 td->last_trb_seg = ep_ring->enq_seg; 4264 field |= TRB_IOC; 4265 if (trb_block_event_intr(xhci, num_tds, i)) 4266 field |= TRB_BEI; 4267 } 4268 /* Calculate TRB length */ 4269 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr); 4270 if (trb_buff_len > td_remain_len) 4271 trb_buff_len = td_remain_len; 4272 4273 /* Set the TRB length, TD size, & interrupter fields. */ 4274 remainder = xhci_td_remainder(xhci, running_total, 4275 trb_buff_len, td_len, 4276 urb, more_trbs_coming); 4277 4278 length_field = TRB_LEN(trb_buff_len) | 4279 TRB_INTR_TARGET(0); 4280 4281 /* xhci 1.1 with ETE uses TD Size field for TBC */ 4282 if (first_trb && xep->use_extended_tbc) 4283 length_field |= TRB_TD_SIZE_TBC(burst_count); 4284 else 4285 length_field |= TRB_TD_SIZE(remainder); 4286 first_trb = false; 4287 4288 queue_trb(xhci, ep_ring, more_trbs_coming, 4289 lower_32_bits(addr), 4290 upper_32_bits(addr), 4291 length_field, 4292 field); 4293 running_total += trb_buff_len; 4294 4295 addr += trb_buff_len; 4296 td_remain_len -= trb_buff_len; 4297 } 4298 4299 /* Check TD length */ 4300 if (running_total != td_len) { 4301 xhci_err(xhci, "ISOC TD length unmatch\n"); 4302 ret = -EINVAL; 4303 goto cleanup; 4304 } 4305 } 4306 4307 /* store the next frame id */ 4308 if (HCC_CFC(xhci->hcc_params)) 4309 xep->next_frame_id = urb->start_frame + num_tds * urb->interval; 4310 4311 if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) { 4312 if (xhci->quirks & XHCI_AMD_PLL_FIX) 4313 usb_amd_quirk_pll_disable(); 4314 } 4315 xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++; 4316 4317 giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id, 4318 start_cycle, start_trb); 4319 return 0; 4320 cleanup: 4321 /* Clean up a partially enqueued isoc transfer. */ 4322 4323 for (i--; i >= 0; i--) 4324 list_del_init(&urb_priv->td[i].td_list); 4325 4326 /* Use the first TD as a temporary variable to turn the TDs we've queued 4327 * into No-ops with a software-owned cycle bit. That way the hardware 4328 * won't accidentally start executing bogus TDs when we partially 4329 * overwrite them. td->first_trb and td->start_seg are already set. 4330 */ 4331 urb_priv->td[0].last_trb = ep_ring->enqueue; 4332 /* Every TRB except the first & last will have its cycle bit flipped. */ 4333 td_to_noop(xhci, ep_ring, &urb_priv->td[0], true); 4334 4335 /* Reset the ring enqueue back to the first TRB and its cycle bit. */ 4336 ep_ring->enqueue = urb_priv->td[0].first_trb; 4337 ep_ring->enq_seg = urb_priv->td[0].start_seg; 4338 ep_ring->cycle_state = start_cycle; 4339 usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb); 4340 return ret; 4341 } 4342 4343 /* 4344 * Check transfer ring to guarantee there is enough room for the urb. 4345 * Update ISO URB start_frame and interval. 4346 * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to 4347 * update urb->start_frame if URB_ISO_ASAP is set in transfer_flags or 4348 * Contiguous Frame ID is not supported by HC. 4349 */ 4350 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags, 4351 struct urb *urb, int slot_id, unsigned int ep_index) 4352 { 4353 struct xhci_virt_device *xdev; 4354 struct xhci_ring *ep_ring; 4355 struct xhci_ep_ctx *ep_ctx; 4356 int start_frame; 4357 int num_tds, num_trbs, i; 4358 int ret; 4359 struct xhci_virt_ep *xep; 4360 int ist; 4361 4362 xdev = xhci->devs[slot_id]; 4363 xep = &xhci->devs[slot_id]->eps[ep_index]; 4364 ep_ring = xdev->eps[ep_index].ring; 4365 ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index); 4366 4367 num_trbs = 0; 4368 num_tds = urb->number_of_packets; 4369 for (i = 0; i < num_tds; i++) 4370 num_trbs += count_isoc_trbs_needed(urb, i); 4371 4372 /* Check the ring to guarantee there is enough room for the whole urb. 4373 * Do not insert any td of the urb to the ring if the check failed. 4374 */ 4375 ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx), 4376 num_trbs, mem_flags); 4377 if (ret) 4378 return ret; 4379 4380 /* 4381 * Check interval value. This should be done before we start to 4382 * calculate the start frame value. 4383 */ 4384 check_interval(xhci, urb, ep_ctx); 4385 4386 /* Calculate the start frame and put it in urb->start_frame. */ 4387 if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) { 4388 if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_RUNNING) { 4389 urb->start_frame = xep->next_frame_id; 4390 goto skip_start_over; 4391 } 4392 } 4393 4394 start_frame = readl(&xhci->run_regs->microframe_index); 4395 start_frame &= 0x3fff; 4396 /* 4397 * Round up to the next frame and consider the time before trb really 4398 * gets scheduled by hardare. 4399 */ 4400 ist = HCS_IST(xhci->hcs_params2) & 0x7; 4401 if (HCS_IST(xhci->hcs_params2) & (1 << 3)) 4402 ist <<= 3; 4403 start_frame += ist + XHCI_CFC_DELAY; 4404 start_frame = roundup(start_frame, 8); 4405 4406 /* 4407 * Round up to the next ESIT (Endpoint Service Interval Time) if ESIT 4408 * is greate than 8 microframes. 4409 */ 4410 if (urb->dev->speed == USB_SPEED_LOW || 4411 urb->dev->speed == USB_SPEED_FULL) { 4412 start_frame = roundup(start_frame, urb->interval << 3); 4413 urb->start_frame = start_frame >> 3; 4414 } else { 4415 start_frame = roundup(start_frame, urb->interval); 4416 urb->start_frame = start_frame; 4417 } 4418 4419 skip_start_over: 4420 4421 return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index); 4422 } 4423 4424 /**** Command Ring Operations ****/ 4425 4426 /* Generic function for queueing a command TRB on the command ring. 4427 * Check to make sure there's room on the command ring for one command TRB. 4428 * Also check that there's room reserved for commands that must not fail. 4429 * If this is a command that must not fail, meaning command_must_succeed = TRUE, 4430 * then only check for the number of reserved spots. 4431 * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB 4432 * because the command event handler may want to resubmit a failed command. 4433 */ 4434 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd, 4435 u32 field1, u32 field2, 4436 u32 field3, u32 field4, bool command_must_succeed) 4437 { 4438 int reserved_trbs = xhci->cmd_ring_reserved_trbs; 4439 int ret; 4440 4441 if ((xhci->xhc_state & XHCI_STATE_DYING) || 4442 (xhci->xhc_state & XHCI_STATE_HALTED)) { 4443 xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n"); 4444 return -ESHUTDOWN; 4445 } 4446 4447 if (!command_must_succeed) 4448 reserved_trbs++; 4449 4450 ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING, 4451 reserved_trbs, GFP_ATOMIC); 4452 if (ret < 0) { 4453 xhci_err(xhci, "ERR: No room for command on command ring\n"); 4454 if (command_must_succeed) 4455 xhci_err(xhci, "ERR: Reserved TRB counting for " 4456 "unfailable commands failed.\n"); 4457 return ret; 4458 } 4459 4460 cmd->command_trb = xhci->cmd_ring->enqueue; 4461 4462 /* if there are no other commands queued we start the timeout timer */ 4463 if (list_empty(&xhci->cmd_list)) { 4464 xhci->current_cmd = cmd; 4465 xhci_mod_cmd_timer(xhci); 4466 } 4467 4468 list_add_tail(&cmd->cmd_list, &xhci->cmd_list); 4469 4470 queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3, 4471 field4 | xhci->cmd_ring->cycle_state); 4472 return 0; 4473 } 4474 4475 /* Queue a slot enable or disable request on the command ring */ 4476 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd, 4477 u32 trb_type, u32 slot_id) 4478 { 4479 return queue_command(xhci, cmd, 0, 0, 0, 4480 TRB_TYPE(trb_type) | SLOT_ID_FOR_TRB(slot_id), false); 4481 } 4482 4483 /* Queue an address device command TRB */ 4484 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd, 4485 dma_addr_t in_ctx_ptr, u32 slot_id, enum xhci_setup_dev setup) 4486 { 4487 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), 4488 upper_32_bits(in_ctx_ptr), 0, 4489 TRB_TYPE(TRB_ADDR_DEV) | SLOT_ID_FOR_TRB(slot_id) 4490 | (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0), false); 4491 } 4492 4493 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd, 4494 u32 field1, u32 field2, u32 field3, u32 field4) 4495 { 4496 return queue_command(xhci, cmd, field1, field2, field3, field4, false); 4497 } 4498 4499 /* Queue a reset device command TRB */ 4500 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd, 4501 u32 slot_id) 4502 { 4503 return queue_command(xhci, cmd, 0, 0, 0, 4504 TRB_TYPE(TRB_RESET_DEV) | SLOT_ID_FOR_TRB(slot_id), 4505 false); 4506 } 4507 4508 /* Queue a configure endpoint command TRB */ 4509 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci, 4510 struct xhci_command *cmd, dma_addr_t in_ctx_ptr, 4511 u32 slot_id, bool command_must_succeed) 4512 { 4513 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), 4514 upper_32_bits(in_ctx_ptr), 0, 4515 TRB_TYPE(TRB_CONFIG_EP) | SLOT_ID_FOR_TRB(slot_id), 4516 command_must_succeed); 4517 } 4518 4519 /* Queue an evaluate context command TRB */ 4520 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd, 4521 dma_addr_t in_ctx_ptr, u32 slot_id, bool command_must_succeed) 4522 { 4523 return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr), 4524 upper_32_bits(in_ctx_ptr), 0, 4525 TRB_TYPE(TRB_EVAL_CONTEXT) | SLOT_ID_FOR_TRB(slot_id), 4526 command_must_succeed); 4527 } 4528 4529 /* 4530 * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop 4531 * activity on an endpoint that is about to be suspended. 4532 */ 4533 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd, 4534 int slot_id, unsigned int ep_index, int suspend) 4535 { 4536 u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id); 4537 u32 trb_ep_index = EP_ID_FOR_TRB(ep_index); 4538 u32 type = TRB_TYPE(TRB_STOP_RING); 4539 u32 trb_suspend = SUSPEND_PORT_FOR_TRB(suspend); 4540 4541 return queue_command(xhci, cmd, 0, 0, 0, 4542 trb_slot_id | trb_ep_index | type | trb_suspend, false); 4543 } 4544 4545 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd, 4546 int slot_id, unsigned int ep_index, 4547 enum xhci_ep_reset_type reset_type) 4548 { 4549 u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id); 4550 u32 trb_ep_index = EP_ID_FOR_TRB(ep_index); 4551 u32 type = TRB_TYPE(TRB_RESET_EP); 4552 4553 if (reset_type == EP_SOFT_RESET) 4554 type |= TRB_TSP; 4555 4556 return queue_command(xhci, cmd, 0, 0, 0, 4557 trb_slot_id | trb_ep_index | type, false); 4558 } 4559