1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Cadence CDNSP DRD Driver. 4 * 5 * Copyright (C) 2020 Cadence. 6 * 7 * Author: Pawel Laszczak <pawell@cadence.com> 8 * 9 * Code based on Linux XHCI driver. 10 * Origin: Copyright (C) 2008 Intel Corp 11 */ 12 13 /* 14 * Ring initialization rules: 15 * 1. Each segment is initialized to zero, except for link TRBs. 16 * 2. Ring cycle state = 0. This represents Producer Cycle State (PCS) or 17 * Consumer Cycle State (CCS), depending on ring function. 18 * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment. 19 * 20 * Ring behavior rules: 21 * 1. A ring is empty if enqueue == dequeue. This means there will always be at 22 * least one free TRB in the ring. This is useful if you want to turn that 23 * into a link TRB and expand the ring. 24 * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a 25 * link TRB, then load the pointer with the address in the link TRB. If the 26 * link TRB had its toggle bit set, you may need to update the ring cycle 27 * state (see cycle bit rules). You may have to do this multiple times 28 * until you reach a non-link TRB. 29 * 3. A ring is full if enqueue++ (for the definition of increment above) 30 * equals the dequeue pointer. 31 * 32 * Cycle bit rules: 33 * 1. When a consumer increments a dequeue pointer and encounters a toggle bit 34 * in a link TRB, it must toggle the ring cycle state. 35 * 2. When a producer increments an enqueue pointer and encounters a toggle bit 36 * in a link TRB, it must toggle the ring cycle state. 37 * 38 * Producer rules: 39 * 1. Check if ring is full before you enqueue. 40 * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing. 41 * Update enqueue pointer between each write (which may update the ring 42 * cycle state). 43 * 3. Notify consumer. If SW is producer, it rings the doorbell for command 44 * and endpoint rings. If controller is the producer for the event ring, 45 * and it generates an interrupt according to interrupt modulation rules. 46 * 47 * Consumer rules: 48 * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state, 49 * the TRB is owned by the consumer. 50 * 2. Update dequeue pointer (which may update the ring cycle state) and 51 * continue processing TRBs until you reach a TRB which is not owned by you. 52 * 3. Notify the producer. SW is the consumer for the event ring, and it 53 * updates event ring dequeue pointer. Controller is the consumer for the 54 * command and endpoint rings; it generates events on the event ring 55 * for these. 56 */ 57 58 #include <linux/scatterlist.h> 59 #include <linux/dma-mapping.h> 60 #include <linux/delay.h> 61 #include <linux/slab.h> 62 #include <linux/irq.h> 63 64 #include "cdnsp-trace.h" 65 #include "cdnsp-gadget.h" 66 67 /* 68 * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA 69 * address of the TRB. 70 */ 71 dma_addr_t cdnsp_trb_virt_to_dma(struct cdnsp_segment *seg, 72 union cdnsp_trb *trb) 73 { 74 unsigned long segment_offset = trb - seg->trbs; 75 76 if (trb < seg->trbs || segment_offset >= TRBS_PER_SEGMENT) 77 return 0; 78 79 return seg->dma + (segment_offset * sizeof(*trb)); 80 } 81 82 static bool cdnsp_trb_is_noop(union cdnsp_trb *trb) 83 { 84 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]); 85 } 86 87 static bool cdnsp_trb_is_link(union cdnsp_trb *trb) 88 { 89 return TRB_TYPE_LINK_LE32(trb->link.control); 90 } 91 92 bool cdnsp_last_trb_on_seg(struct cdnsp_segment *seg, union cdnsp_trb *trb) 93 { 94 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1]; 95 } 96 97 bool cdnsp_last_trb_on_ring(struct cdnsp_ring *ring, 98 struct cdnsp_segment *seg, 99 union cdnsp_trb *trb) 100 { 101 return cdnsp_last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg); 102 } 103 104 static bool cdnsp_link_trb_toggles_cycle(union cdnsp_trb *trb) 105 { 106 return le32_to_cpu(trb->link.control) & LINK_TOGGLE; 107 } 108 109 static void cdnsp_trb_to_noop(union cdnsp_trb *trb, u32 noop_type) 110 { 111 if (cdnsp_trb_is_link(trb)) { 112 /* Unchain chained link TRBs. */ 113 trb->link.control &= cpu_to_le32(~TRB_CHAIN); 114 } else { 115 trb->generic.field[0] = 0; 116 trb->generic.field[1] = 0; 117 trb->generic.field[2] = 0; 118 /* Preserve only the cycle bit of this TRB. */ 119 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE); 120 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type)); 121 } 122 } 123 124 /* 125 * Updates trb to point to the next TRB in the ring, and updates seg if the next 126 * TRB is in a new segment. This does not skip over link TRBs, and it does not 127 * effect the ring dequeue or enqueue pointers. 128 */ 129 static void cdnsp_next_trb(struct cdnsp_device *pdev, 130 struct cdnsp_ring *ring, 131 struct cdnsp_segment **seg, 132 union cdnsp_trb **trb) 133 { 134 if (cdnsp_trb_is_link(*trb)) { 135 *seg = (*seg)->next; 136 *trb = ((*seg)->trbs); 137 } else { 138 (*trb)++; 139 } 140 } 141 142 /* 143 * See Cycle bit rules. SW is the consumer for the event ring only. 144 * Don't make a ring full of link TRBs. That would be dumb and this would loop. 145 */ 146 void cdnsp_inc_deq(struct cdnsp_device *pdev, struct cdnsp_ring *ring) 147 { 148 /* event ring doesn't have link trbs, check for last trb. */ 149 if (ring->type == TYPE_EVENT) { 150 if (!cdnsp_last_trb_on_seg(ring->deq_seg, ring->dequeue)) { 151 ring->dequeue++; 152 goto out; 153 } 154 155 if (cdnsp_last_trb_on_ring(ring, ring->deq_seg, ring->dequeue)) 156 ring->cycle_state ^= 1; 157 158 ring->deq_seg = ring->deq_seg->next; 159 ring->dequeue = ring->deq_seg->trbs; 160 goto out; 161 } 162 163 /* All other rings have link trbs. */ 164 if (!cdnsp_trb_is_link(ring->dequeue)) { 165 ring->dequeue++; 166 ring->num_trbs_free++; 167 } 168 while (cdnsp_trb_is_link(ring->dequeue)) { 169 ring->deq_seg = ring->deq_seg->next; 170 ring->dequeue = ring->deq_seg->trbs; 171 } 172 out: 173 trace_cdnsp_inc_deq(ring); 174 } 175 176 /* 177 * See Cycle bit rules. SW is the consumer for the event ring only. 178 * Don't make a ring full of link TRBs. That would be dumb and this would loop. 179 * 180 * If we've just enqueued a TRB that is in the middle of a TD (meaning the 181 * chain bit is set), then set the chain bit in all the following link TRBs. 182 * If we've enqueued the last TRB in a TD, make sure the following link TRBs 183 * have their chain bit cleared (so that each Link TRB is a separate TD). 184 * 185 * @more_trbs_coming: Will you enqueue more TRBs before ringing the doorbell. 186 */ 187 static void cdnsp_inc_enq(struct cdnsp_device *pdev, 188 struct cdnsp_ring *ring, 189 bool more_trbs_coming) 190 { 191 union cdnsp_trb *next; 192 u32 chain; 193 194 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN; 195 196 /* If this is not event ring, there is one less usable TRB. */ 197 if (!cdnsp_trb_is_link(ring->enqueue)) 198 ring->num_trbs_free--; 199 next = ++(ring->enqueue); 200 201 /* Update the dequeue pointer further if that was a link TRB */ 202 while (cdnsp_trb_is_link(next)) { 203 /* 204 * If the caller doesn't plan on enqueuing more TDs before 205 * ringing the doorbell, then we don't want to give the link TRB 206 * to the hardware just yet. We'll give the link TRB back in 207 * cdnsp_prepare_ring() just before we enqueue the TD at the 208 * top of the ring. 209 */ 210 if (!chain && !more_trbs_coming) 211 break; 212 213 next->link.control &= cpu_to_le32(~TRB_CHAIN); 214 next->link.control |= cpu_to_le32(chain); 215 216 /* Give this link TRB to the hardware */ 217 wmb(); 218 next->link.control ^= cpu_to_le32(TRB_CYCLE); 219 220 /* Toggle the cycle bit after the last ring segment. */ 221 if (cdnsp_link_trb_toggles_cycle(next)) 222 ring->cycle_state ^= 1; 223 224 ring->enq_seg = ring->enq_seg->next; 225 ring->enqueue = ring->enq_seg->trbs; 226 next = ring->enqueue; 227 } 228 229 trace_cdnsp_inc_enq(ring); 230 } 231 232 /* 233 * Check to see if there's room to enqueue num_trbs on the ring and make sure 234 * enqueue pointer will not advance into dequeue segment. 235 */ 236 static bool cdnsp_room_on_ring(struct cdnsp_device *pdev, 237 struct cdnsp_ring *ring, 238 unsigned int num_trbs) 239 { 240 int num_trbs_in_deq_seg; 241 242 if (ring->num_trbs_free < num_trbs) 243 return false; 244 245 if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) { 246 num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs; 247 248 if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg) 249 return false; 250 } 251 252 return true; 253 } 254 255 /* 256 * Workaround for L1: controller has issue with resuming from L1 after 257 * setting doorbell for endpoint during L1 state. This function forces 258 * resume signal in such case. 259 */ 260 static void cdnsp_force_l0_go(struct cdnsp_device *pdev) 261 { 262 if (pdev->active_port == &pdev->usb2_port && pdev->gadget.lpm_capable) 263 cdnsp_set_link_state(pdev, &pdev->active_port->regs->portsc, XDEV_U0); 264 } 265 266 /* Ring the doorbell after placing a command on the ring. */ 267 void cdnsp_ring_cmd_db(struct cdnsp_device *pdev) 268 { 269 writel(DB_VALUE_CMD, &pdev->dba->cmd_db); 270 } 271 272 /* 273 * Ring the doorbell after placing a transfer on the ring. 274 * Returns true if doorbell was set, otherwise false. 275 */ 276 static bool cdnsp_ring_ep_doorbell(struct cdnsp_device *pdev, 277 struct cdnsp_ep *pep, 278 unsigned int stream_id) 279 { 280 __le32 __iomem *reg_addr = &pdev->dba->ep_db; 281 unsigned int ep_state = pep->ep_state; 282 unsigned int db_value; 283 284 /* 285 * Don't ring the doorbell for this endpoint if endpoint is halted or 286 * disabled. 287 */ 288 if (ep_state & EP_HALTED || !(ep_state & EP_ENABLED)) 289 return false; 290 291 /* For stream capable endpoints driver can ring doorbell only twice. */ 292 if (pep->ep_state & EP_HAS_STREAMS) { 293 if (pep->stream_info.drbls_count >= 2) 294 return false; 295 296 pep->stream_info.drbls_count++; 297 } 298 299 pep->ep_state &= ~EP_STOPPED; 300 301 if (pep->idx == 0 && pdev->ep0_stage == CDNSP_DATA_STAGE && 302 !pdev->ep0_expect_in) 303 db_value = DB_VALUE_EP0_OUT(pep->idx, stream_id); 304 else 305 db_value = DB_VALUE(pep->idx, stream_id); 306 307 trace_cdnsp_tr_drbl(pep, stream_id); 308 309 writel(db_value, reg_addr); 310 311 cdnsp_force_l0_go(pdev); 312 313 /* Doorbell was set. */ 314 return true; 315 } 316 317 /* 318 * Get the right ring for the given pep and stream_id. 319 * If the endpoint supports streams, boundary check the USB request's stream ID. 320 * If the endpoint doesn't support streams, return the singular endpoint ring. 321 */ 322 static struct cdnsp_ring *cdnsp_get_transfer_ring(struct cdnsp_device *pdev, 323 struct cdnsp_ep *pep, 324 unsigned int stream_id) 325 { 326 if (!(pep->ep_state & EP_HAS_STREAMS)) 327 return pep->ring; 328 329 if (stream_id == 0 || stream_id >= pep->stream_info.num_streams) { 330 dev_err(pdev->dev, "ERR: %s ring doesn't exist for SID: %d.\n", 331 pep->name, stream_id); 332 return NULL; 333 } 334 335 return pep->stream_info.stream_rings[stream_id]; 336 } 337 338 static struct cdnsp_ring * 339 cdnsp_request_to_transfer_ring(struct cdnsp_device *pdev, 340 struct cdnsp_request *preq) 341 { 342 return cdnsp_get_transfer_ring(pdev, preq->pep, 343 preq->request.stream_id); 344 } 345 346 /* Ring the doorbell for any rings with pending requests. */ 347 void cdnsp_ring_doorbell_for_active_rings(struct cdnsp_device *pdev, 348 struct cdnsp_ep *pep) 349 { 350 struct cdnsp_stream_info *stream_info; 351 unsigned int stream_id; 352 int ret; 353 354 if (pep->ep_state & EP_DIS_IN_RROGRESS) 355 return; 356 357 /* A ring has pending Request if its TD list is not empty. */ 358 if (!(pep->ep_state & EP_HAS_STREAMS) && pep->number) { 359 if (pep->ring && !list_empty(&pep->ring->td_list)) 360 cdnsp_ring_ep_doorbell(pdev, pep, 0); 361 return; 362 } 363 364 stream_info = &pep->stream_info; 365 366 for (stream_id = 1; stream_id < stream_info->num_streams; stream_id++) { 367 struct cdnsp_td *td, *td_temp; 368 struct cdnsp_ring *ep_ring; 369 370 if (stream_info->drbls_count >= 2) 371 return; 372 373 ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id); 374 if (!ep_ring) 375 continue; 376 377 if (!ep_ring->stream_active || ep_ring->stream_rejected) 378 continue; 379 380 list_for_each_entry_safe(td, td_temp, &ep_ring->td_list, 381 td_list) { 382 if (td->drbl) 383 continue; 384 385 ret = cdnsp_ring_ep_doorbell(pdev, pep, stream_id); 386 if (ret) 387 td->drbl = 1; 388 } 389 } 390 } 391 392 /* 393 * Get the hw dequeue pointer controller stopped on, either directly from the 394 * endpoint context, or if streams are in use from the stream context. 395 * The returned hw_dequeue contains the lowest four bits with cycle state 396 * and possible stream context type. 397 */ 398 static u64 cdnsp_get_hw_deq(struct cdnsp_device *pdev, 399 unsigned int ep_index, 400 unsigned int stream_id) 401 { 402 struct cdnsp_stream_ctx *st_ctx; 403 struct cdnsp_ep *pep; 404 405 pep = &pdev->eps[stream_id]; 406 407 if (pep->ep_state & EP_HAS_STREAMS) { 408 st_ctx = &pep->stream_info.stream_ctx_array[stream_id]; 409 return le64_to_cpu(st_ctx->stream_ring); 410 } 411 412 return le64_to_cpu(pep->out_ctx->deq); 413 } 414 415 /* 416 * Move the controller endpoint ring dequeue pointer past cur_td. 417 * Record the new state of the controller endpoint ring dequeue segment, 418 * dequeue pointer, and new consumer cycle state in state. 419 * Update internal representation of the ring's dequeue pointer. 420 * 421 * We do this in three jumps: 422 * - First we update our new ring state to be the same as when the 423 * controller stopped. 424 * - Then we traverse the ring to find the segment that contains 425 * the last TRB in the TD. We toggle the controller new cycle state 426 * when we pass any link TRBs with the toggle cycle bit set. 427 * - Finally we move the dequeue state one TRB further, toggling the cycle bit 428 * if we've moved it past a link TRB with the toggle cycle bit set. 429 */ 430 static void cdnsp_find_new_dequeue_state(struct cdnsp_device *pdev, 431 struct cdnsp_ep *pep, 432 unsigned int stream_id, 433 struct cdnsp_td *cur_td, 434 struct cdnsp_dequeue_state *state) 435 { 436 bool td_last_trb_found = false; 437 struct cdnsp_segment *new_seg; 438 struct cdnsp_ring *ep_ring; 439 union cdnsp_trb *new_deq; 440 bool cycle_found = false; 441 u64 hw_dequeue; 442 443 ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id); 444 if (!ep_ring) 445 return; 446 447 /* 448 * Dig out the cycle state saved by the controller during the 449 * stop endpoint command. 450 */ 451 hw_dequeue = cdnsp_get_hw_deq(pdev, pep->idx, stream_id); 452 new_seg = ep_ring->deq_seg; 453 new_deq = ep_ring->dequeue; 454 state->new_cycle_state = hw_dequeue & 0x1; 455 state->stream_id = stream_id; 456 457 /* 458 * We want to find the pointer, segment and cycle state of the new trb 459 * (the one after current TD's last_trb). We know the cycle state at 460 * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are 461 * found. 462 */ 463 do { 464 if (!cycle_found && cdnsp_trb_virt_to_dma(new_seg, new_deq) 465 == (dma_addr_t)(hw_dequeue & ~0xf)) { 466 cycle_found = true; 467 468 if (td_last_trb_found) 469 break; 470 } 471 472 if (new_deq == cur_td->last_trb) 473 td_last_trb_found = true; 474 475 if (cycle_found && cdnsp_trb_is_link(new_deq) && 476 cdnsp_link_trb_toggles_cycle(new_deq)) 477 state->new_cycle_state ^= 0x1; 478 479 cdnsp_next_trb(pdev, ep_ring, &new_seg, &new_deq); 480 481 /* Search wrapped around, bail out. */ 482 if (new_deq == pep->ring->dequeue) { 483 dev_err(pdev->dev, 484 "Error: Failed finding new dequeue state\n"); 485 state->new_deq_seg = NULL; 486 state->new_deq_ptr = NULL; 487 return; 488 } 489 490 } while (!cycle_found || !td_last_trb_found); 491 492 state->new_deq_seg = new_seg; 493 state->new_deq_ptr = new_deq; 494 495 trace_cdnsp_new_deq_state(state); 496 } 497 498 /* 499 * flip_cycle means flip the cycle bit of all but the first and last TRB. 500 * (The last TRB actually points to the ring enqueue pointer, which is not part 501 * of this TD.) This is used to remove partially enqueued isoc TDs from a ring. 502 */ 503 static void cdnsp_td_to_noop(struct cdnsp_device *pdev, 504 struct cdnsp_ring *ep_ring, 505 struct cdnsp_td *td, 506 bool flip_cycle) 507 { 508 struct cdnsp_segment *seg = td->start_seg; 509 union cdnsp_trb *trb = td->first_trb; 510 511 while (1) { 512 cdnsp_trb_to_noop(trb, TRB_TR_NOOP); 513 514 /* flip cycle if asked to */ 515 if (flip_cycle && trb != td->first_trb && trb != td->last_trb) 516 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE); 517 518 if (trb == td->last_trb) 519 break; 520 521 cdnsp_next_trb(pdev, ep_ring, &seg, &trb); 522 } 523 } 524 525 /* 526 * This TD is defined by the TRBs starting at start_trb in start_seg and ending 527 * at end_trb, which may be in another segment. If the suspect DMA address is a 528 * TRB in this TD, this function returns that TRB's segment. Otherwise it 529 * returns 0. 530 */ 531 static struct cdnsp_segment *cdnsp_trb_in_td(struct cdnsp_device *pdev, 532 struct cdnsp_segment *start_seg, 533 union cdnsp_trb *start_trb, 534 union cdnsp_trb *end_trb, 535 dma_addr_t suspect_dma) 536 { 537 struct cdnsp_segment *cur_seg; 538 union cdnsp_trb *temp_trb; 539 dma_addr_t end_seg_dma; 540 dma_addr_t end_trb_dma; 541 dma_addr_t start_dma; 542 543 start_dma = cdnsp_trb_virt_to_dma(start_seg, start_trb); 544 cur_seg = start_seg; 545 546 do { 547 if (start_dma == 0) 548 return NULL; 549 550 temp_trb = &cur_seg->trbs[TRBS_PER_SEGMENT - 1]; 551 /* We may get an event for a Link TRB in the middle of a TD */ 552 end_seg_dma = cdnsp_trb_virt_to_dma(cur_seg, temp_trb); 553 /* If the end TRB isn't in this segment, this is set to 0 */ 554 end_trb_dma = cdnsp_trb_virt_to_dma(cur_seg, end_trb); 555 556 trace_cdnsp_looking_trb_in_td(suspect_dma, start_dma, 557 end_trb_dma, cur_seg->dma, 558 end_seg_dma); 559 560 if (end_trb_dma > 0) { 561 /* 562 * The end TRB is in this segment, so suspect should 563 * be here 564 */ 565 if (start_dma <= end_trb_dma) { 566 if (suspect_dma >= start_dma && 567 suspect_dma <= end_trb_dma) { 568 return cur_seg; 569 } 570 } else { 571 /* 572 * Case for one segment with a 573 * TD wrapped around to the top 574 */ 575 if ((suspect_dma >= start_dma && 576 suspect_dma <= end_seg_dma) || 577 (suspect_dma >= cur_seg->dma && 578 suspect_dma <= end_trb_dma)) { 579 return cur_seg; 580 } 581 } 582 583 return NULL; 584 } 585 586 /* Might still be somewhere in this segment */ 587 if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma) 588 return cur_seg; 589 590 cur_seg = cur_seg->next; 591 start_dma = cdnsp_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]); 592 } while (cur_seg != start_seg); 593 594 return NULL; 595 } 596 597 static void cdnsp_unmap_td_bounce_buffer(struct cdnsp_device *pdev, 598 struct cdnsp_ring *ring, 599 struct cdnsp_td *td) 600 { 601 struct cdnsp_segment *seg = td->bounce_seg; 602 struct cdnsp_request *preq; 603 size_t len; 604 605 if (!seg) 606 return; 607 608 preq = td->preq; 609 610 trace_cdnsp_bounce_unmap(td->preq, seg->bounce_len, seg->bounce_offs, 611 seg->bounce_dma, 0); 612 613 if (!preq->direction) { 614 dma_unmap_single(pdev->dev, seg->bounce_dma, 615 ring->bounce_buf_len, DMA_TO_DEVICE); 616 return; 617 } 618 619 dma_unmap_single(pdev->dev, seg->bounce_dma, ring->bounce_buf_len, 620 DMA_FROM_DEVICE); 621 622 /* For in transfers we need to copy the data from bounce to sg */ 623 len = sg_pcopy_from_buffer(preq->request.sg, preq->request.num_sgs, 624 seg->bounce_buf, seg->bounce_len, 625 seg->bounce_offs); 626 if (len != seg->bounce_len) 627 dev_warn(pdev->dev, "WARN Wrong bounce buffer read length: %zu != %d\n", 628 len, seg->bounce_len); 629 630 seg->bounce_len = 0; 631 seg->bounce_offs = 0; 632 } 633 634 static int cdnsp_cmd_set_deq(struct cdnsp_device *pdev, 635 struct cdnsp_ep *pep, 636 struct cdnsp_dequeue_state *deq_state) 637 { 638 struct cdnsp_ring *ep_ring; 639 int ret; 640 641 if (!deq_state->new_deq_ptr || !deq_state->new_deq_seg) { 642 cdnsp_ring_doorbell_for_active_rings(pdev, pep); 643 return 0; 644 } 645 646 cdnsp_queue_new_dequeue_state(pdev, pep, deq_state); 647 cdnsp_ring_cmd_db(pdev); 648 ret = cdnsp_wait_for_cmd_compl(pdev); 649 650 trace_cdnsp_handle_cmd_set_deq(cdnsp_get_slot_ctx(&pdev->out_ctx)); 651 trace_cdnsp_handle_cmd_set_deq_ep(pep->out_ctx); 652 653 /* 654 * Update the ring's dequeue segment and dequeue pointer 655 * to reflect the new position. 656 */ 657 ep_ring = cdnsp_get_transfer_ring(pdev, pep, deq_state->stream_id); 658 659 if (cdnsp_trb_is_link(ep_ring->dequeue)) { 660 ep_ring->deq_seg = ep_ring->deq_seg->next; 661 ep_ring->dequeue = ep_ring->deq_seg->trbs; 662 } 663 664 while (ep_ring->dequeue != deq_state->new_deq_ptr) { 665 ep_ring->num_trbs_free++; 666 ep_ring->dequeue++; 667 668 if (cdnsp_trb_is_link(ep_ring->dequeue)) { 669 if (ep_ring->dequeue == deq_state->new_deq_ptr) 670 break; 671 672 ep_ring->deq_seg = ep_ring->deq_seg->next; 673 ep_ring->dequeue = ep_ring->deq_seg->trbs; 674 } 675 } 676 677 /* 678 * Probably there was TIMEOUT during handling Set Dequeue Pointer 679 * command. It's critical error and controller will be stopped. 680 */ 681 if (ret) 682 return -ESHUTDOWN; 683 684 /* Restart any rings with pending requests */ 685 cdnsp_ring_doorbell_for_active_rings(pdev, pep); 686 687 return 0; 688 } 689 690 int cdnsp_remove_request(struct cdnsp_device *pdev, 691 struct cdnsp_request *preq, 692 struct cdnsp_ep *pep) 693 { 694 struct cdnsp_dequeue_state deq_state; 695 struct cdnsp_td *cur_td = NULL; 696 struct cdnsp_ring *ep_ring; 697 struct cdnsp_segment *seg; 698 int status = -ECONNRESET; 699 int ret = 0; 700 u64 hw_deq; 701 702 memset(&deq_state, 0, sizeof(deq_state)); 703 704 trace_cdnsp_remove_request(pep->out_ctx); 705 trace_cdnsp_remove_request_td(preq); 706 707 cur_td = &preq->td; 708 ep_ring = cdnsp_request_to_transfer_ring(pdev, preq); 709 710 /* 711 * If we stopped on the TD we need to cancel, then we have to 712 * move the controller endpoint ring dequeue pointer past 713 * this TD. 714 */ 715 hw_deq = cdnsp_get_hw_deq(pdev, pep->idx, preq->request.stream_id); 716 hw_deq &= ~0xf; 717 718 seg = cdnsp_trb_in_td(pdev, cur_td->start_seg, cur_td->first_trb, 719 cur_td->last_trb, hw_deq); 720 721 if (seg && (pep->ep_state & EP_ENABLED)) 722 cdnsp_find_new_dequeue_state(pdev, pep, preq->request.stream_id, 723 cur_td, &deq_state); 724 else 725 cdnsp_td_to_noop(pdev, ep_ring, cur_td, false); 726 727 /* 728 * The event handler won't see a completion for this TD anymore, 729 * so remove it from the endpoint ring's TD list. 730 */ 731 list_del_init(&cur_td->td_list); 732 ep_ring->num_tds--; 733 pep->stream_info.td_count--; 734 735 /* 736 * During disconnecting all endpoint will be disabled so we don't 737 * have to worry about updating dequeue pointer. 738 */ 739 if (pdev->cdnsp_state & CDNSP_STATE_DISCONNECT_PENDING) { 740 status = -ESHUTDOWN; 741 ret = cdnsp_cmd_set_deq(pdev, pep, &deq_state); 742 } 743 744 cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, cur_td); 745 cdnsp_gadget_giveback(pep, cur_td->preq, status); 746 747 return ret; 748 } 749 750 static int cdnsp_update_port_id(struct cdnsp_device *pdev, u32 port_id) 751 { 752 struct cdnsp_port *port = pdev->active_port; 753 u8 old_port = 0; 754 755 if (port && port->port_num == port_id) 756 return 0; 757 758 if (port) 759 old_port = port->port_num; 760 761 if (port_id == pdev->usb2_port.port_num) { 762 port = &pdev->usb2_port; 763 } else if (port_id == pdev->usb3_port.port_num) { 764 port = &pdev->usb3_port; 765 } else { 766 dev_err(pdev->dev, "Port event with invalid port ID %d\n", 767 port_id); 768 return -EINVAL; 769 } 770 771 if (port_id != old_port) { 772 cdnsp_disable_slot(pdev); 773 pdev->active_port = port; 774 cdnsp_enable_slot(pdev); 775 } 776 777 if (port_id == pdev->usb2_port.port_num) 778 cdnsp_set_usb2_hardware_lpm(pdev, NULL, 1); 779 else 780 writel(PORT_U1_TIMEOUT(1) | PORT_U2_TIMEOUT(1), 781 &pdev->usb3_port.regs->portpmsc); 782 783 return 0; 784 } 785 786 static void cdnsp_handle_port_status(struct cdnsp_device *pdev, 787 union cdnsp_trb *event) 788 { 789 struct cdnsp_port_regs __iomem *port_regs; 790 u32 portsc, cmd_regs; 791 bool port2 = false; 792 u32 link_state; 793 u32 port_id; 794 795 /* Port status change events always have a successful completion code */ 796 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) 797 dev_err(pdev->dev, "ERR: incorrect PSC event\n"); 798 799 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); 800 801 if (cdnsp_update_port_id(pdev, port_id)) 802 goto cleanup; 803 804 port_regs = pdev->active_port->regs; 805 806 if (port_id == pdev->usb2_port.port_num) 807 port2 = true; 808 809 new_event: 810 portsc = readl(&port_regs->portsc); 811 writel(cdnsp_port_state_to_neutral(portsc) | 812 (portsc & PORT_CHANGE_BITS), &port_regs->portsc); 813 814 trace_cdnsp_handle_port_status(pdev->active_port->port_num, portsc); 815 816 pdev->gadget.speed = cdnsp_port_speed(portsc); 817 link_state = portsc & PORT_PLS_MASK; 818 819 /* Port Link State change detected. */ 820 if ((portsc & PORT_PLC)) { 821 if (!(pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) && 822 link_state == XDEV_RESUME) { 823 cmd_regs = readl(&pdev->op_regs->command); 824 if (!(cmd_regs & CMD_R_S)) 825 goto cleanup; 826 827 if (DEV_SUPERSPEED_ANY(portsc)) { 828 cdnsp_set_link_state(pdev, &port_regs->portsc, 829 XDEV_U0); 830 831 cdnsp_resume_gadget(pdev); 832 } 833 } 834 835 if ((pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) && 836 link_state == XDEV_U0) { 837 pdev->cdnsp_state &= ~CDNSP_WAKEUP_PENDING; 838 839 cdnsp_force_header_wakeup(pdev, 1); 840 cdnsp_ring_cmd_db(pdev); 841 cdnsp_wait_for_cmd_compl(pdev); 842 } 843 844 if (link_state == XDEV_U0 && pdev->link_state == XDEV_U3 && 845 !DEV_SUPERSPEED_ANY(portsc)) 846 cdnsp_resume_gadget(pdev); 847 848 if (link_state == XDEV_U3 && pdev->link_state != XDEV_U3) 849 cdnsp_suspend_gadget(pdev); 850 851 pdev->link_state = link_state; 852 } 853 854 if (portsc & PORT_CSC) { 855 /* Detach device. */ 856 if (pdev->gadget.connected && !(portsc & PORT_CONNECT)) 857 cdnsp_disconnect_gadget(pdev); 858 859 /* Attach device. */ 860 if (portsc & PORT_CONNECT) { 861 if (!port2) 862 cdnsp_irq_reset(pdev); 863 864 usb_gadget_set_state(&pdev->gadget, USB_STATE_ATTACHED); 865 } 866 } 867 868 /* Port reset. */ 869 if ((portsc & (PORT_RC | PORT_WRC)) && (portsc & PORT_CONNECT)) { 870 cdnsp_irq_reset(pdev); 871 pdev->u1_allowed = 0; 872 pdev->u2_allowed = 0; 873 pdev->may_wakeup = 0; 874 } 875 876 if (portsc & PORT_CEC) 877 dev_err(pdev->dev, "Port Over Current detected\n"); 878 879 if (portsc & PORT_CEC) 880 dev_err(pdev->dev, "Port Configure Error detected\n"); 881 882 if (readl(&port_regs->portsc) & PORT_CHANGE_BITS) 883 goto new_event; 884 885 cleanup: 886 cdnsp_inc_deq(pdev, pdev->event_ring); 887 } 888 889 static void cdnsp_td_cleanup(struct cdnsp_device *pdev, 890 struct cdnsp_td *td, 891 struct cdnsp_ring *ep_ring, 892 int *status) 893 { 894 struct cdnsp_request *preq = td->preq; 895 896 /* if a bounce buffer was used to align this td then unmap it */ 897 cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, td); 898 899 /* 900 * If the controller said we transferred more data than the buffer 901 * length, Play it safe and say we didn't transfer anything. 902 */ 903 if (preq->request.actual > preq->request.length) { 904 preq->request.actual = 0; 905 *status = 0; 906 } 907 908 list_del_init(&td->td_list); 909 ep_ring->num_tds--; 910 preq->pep->stream_info.td_count--; 911 912 cdnsp_gadget_giveback(preq->pep, preq, *status); 913 } 914 915 static void cdnsp_finish_td(struct cdnsp_device *pdev, 916 struct cdnsp_td *td, 917 struct cdnsp_transfer_event *event, 918 struct cdnsp_ep *ep, 919 int *status) 920 { 921 struct cdnsp_ring *ep_ring; 922 u32 trb_comp_code; 923 924 ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); 925 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 926 927 if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID || 928 trb_comp_code == COMP_STOPPED || 929 trb_comp_code == COMP_STOPPED_SHORT_PACKET) { 930 /* 931 * The Endpoint Stop Command completion will take care of any 932 * stopped TDs. A stopped TD may be restarted, so don't update 933 * the ring dequeue pointer or take this TD off any lists yet. 934 */ 935 return; 936 } 937 938 /* Update ring dequeue pointer */ 939 while (ep_ring->dequeue != td->last_trb) 940 cdnsp_inc_deq(pdev, ep_ring); 941 942 cdnsp_inc_deq(pdev, ep_ring); 943 944 cdnsp_td_cleanup(pdev, td, ep_ring, status); 945 } 946 947 /* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */ 948 static int cdnsp_sum_trb_lengths(struct cdnsp_device *pdev, 949 struct cdnsp_ring *ring, 950 union cdnsp_trb *stop_trb) 951 { 952 struct cdnsp_segment *seg = ring->deq_seg; 953 union cdnsp_trb *trb = ring->dequeue; 954 u32 sum; 955 956 for (sum = 0; trb != stop_trb; cdnsp_next_trb(pdev, ring, &seg, &trb)) { 957 if (!cdnsp_trb_is_noop(trb) && !cdnsp_trb_is_link(trb)) 958 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2])); 959 } 960 return sum; 961 } 962 963 static int cdnsp_giveback_first_trb(struct cdnsp_device *pdev, 964 struct cdnsp_ep *pep, 965 unsigned int stream_id, 966 int start_cycle, 967 struct cdnsp_generic_trb *start_trb) 968 { 969 /* 970 * Pass all the TRBs to the hardware at once and make sure this write 971 * isn't reordered. 972 */ 973 wmb(); 974 975 if (start_cycle) 976 start_trb->field[3] |= cpu_to_le32(start_cycle); 977 else 978 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); 979 980 if ((pep->ep_state & EP_HAS_STREAMS) && 981 !pep->stream_info.first_prime_det) { 982 trace_cdnsp_wait_for_prime(pep, stream_id); 983 return 0; 984 } 985 986 return cdnsp_ring_ep_doorbell(pdev, pep, stream_id); 987 } 988 989 /* 990 * Process control tds, update USB request status and actual_length. 991 */ 992 static void cdnsp_process_ctrl_td(struct cdnsp_device *pdev, 993 struct cdnsp_td *td, 994 union cdnsp_trb *event_trb, 995 struct cdnsp_transfer_event *event, 996 struct cdnsp_ep *pep, 997 int *status) 998 { 999 struct cdnsp_ring *ep_ring; 1000 u32 remaining; 1001 u32 trb_type; 1002 1003 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event_trb->generic.field[3])); 1004 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 1005 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 1006 1007 /* 1008 * if on data stage then update the actual_length of the USB 1009 * request and flag it as set, so it won't be overwritten in the event 1010 * for the last TRB. 1011 */ 1012 if (trb_type == TRB_DATA) { 1013 td->request_length_set = true; 1014 td->preq->request.actual = td->preq->request.length - remaining; 1015 } 1016 1017 /* at status stage */ 1018 if (!td->request_length_set) 1019 td->preq->request.actual = td->preq->request.length; 1020 1021 if (pdev->ep0_stage == CDNSP_DATA_STAGE && pep->number == 0 && 1022 pdev->three_stage_setup) { 1023 td = list_entry(ep_ring->td_list.next, struct cdnsp_td, 1024 td_list); 1025 pdev->ep0_stage = CDNSP_STATUS_STAGE; 1026 1027 cdnsp_giveback_first_trb(pdev, pep, 0, ep_ring->cycle_state, 1028 &td->last_trb->generic); 1029 return; 1030 } 1031 1032 cdnsp_finish_td(pdev, td, event, pep, status); 1033 } 1034 1035 /* 1036 * Process isochronous tds, update usb request status and actual_length. 1037 */ 1038 static void cdnsp_process_isoc_td(struct cdnsp_device *pdev, 1039 struct cdnsp_td *td, 1040 union cdnsp_trb *ep_trb, 1041 struct cdnsp_transfer_event *event, 1042 struct cdnsp_ep *pep, 1043 int status) 1044 { 1045 struct cdnsp_request *preq = td->preq; 1046 u32 remaining, requested, ep_trb_len; 1047 bool sum_trbs_for_length = false; 1048 struct cdnsp_ring *ep_ring; 1049 u32 trb_comp_code; 1050 u32 td_length; 1051 1052 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 1053 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 1054 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 1055 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); 1056 1057 requested = preq->request.length; 1058 1059 /* handle completion code */ 1060 switch (trb_comp_code) { 1061 case COMP_SUCCESS: 1062 preq->request.status = 0; 1063 break; 1064 case COMP_SHORT_PACKET: 1065 preq->request.status = 0; 1066 sum_trbs_for_length = true; 1067 break; 1068 case COMP_ISOCH_BUFFER_OVERRUN: 1069 case COMP_BABBLE_DETECTED_ERROR: 1070 preq->request.status = -EOVERFLOW; 1071 break; 1072 case COMP_STOPPED: 1073 sum_trbs_for_length = true; 1074 break; 1075 case COMP_STOPPED_SHORT_PACKET: 1076 /* field normally containing residue now contains transferred */ 1077 preq->request.status = 0; 1078 requested = remaining; 1079 break; 1080 case COMP_STOPPED_LENGTH_INVALID: 1081 requested = 0; 1082 remaining = 0; 1083 break; 1084 default: 1085 sum_trbs_for_length = true; 1086 preq->request.status = -1; 1087 break; 1088 } 1089 1090 if (sum_trbs_for_length) { 1091 td_length = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb); 1092 td_length += ep_trb_len - remaining; 1093 } else { 1094 td_length = requested; 1095 } 1096 1097 td->preq->request.actual += td_length; 1098 1099 cdnsp_finish_td(pdev, td, event, pep, &status); 1100 } 1101 1102 static void cdnsp_skip_isoc_td(struct cdnsp_device *pdev, 1103 struct cdnsp_td *td, 1104 struct cdnsp_transfer_event *event, 1105 struct cdnsp_ep *pep, 1106 int status) 1107 { 1108 struct cdnsp_ring *ep_ring; 1109 1110 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 1111 td->preq->request.status = -EXDEV; 1112 td->preq->request.actual = 0; 1113 1114 /* Update ring dequeue pointer */ 1115 while (ep_ring->dequeue != td->last_trb) 1116 cdnsp_inc_deq(pdev, ep_ring); 1117 1118 cdnsp_inc_deq(pdev, ep_ring); 1119 1120 cdnsp_td_cleanup(pdev, td, ep_ring, &status); 1121 } 1122 1123 /* 1124 * Process bulk and interrupt tds, update usb request status and actual_length. 1125 */ 1126 static void cdnsp_process_bulk_intr_td(struct cdnsp_device *pdev, 1127 struct cdnsp_td *td, 1128 union cdnsp_trb *ep_trb, 1129 struct cdnsp_transfer_event *event, 1130 struct cdnsp_ep *ep, 1131 int *status) 1132 { 1133 u32 remaining, requested, ep_trb_len; 1134 struct cdnsp_ring *ep_ring; 1135 u32 trb_comp_code; 1136 1137 ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); 1138 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 1139 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 1140 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); 1141 requested = td->preq->request.length; 1142 1143 switch (trb_comp_code) { 1144 case COMP_SUCCESS: 1145 case COMP_SHORT_PACKET: 1146 *status = 0; 1147 break; 1148 case COMP_STOPPED_SHORT_PACKET: 1149 td->preq->request.actual = remaining; 1150 goto finish_td; 1151 case COMP_STOPPED_LENGTH_INVALID: 1152 /* Stopped on ep trb with invalid length, exclude it. */ 1153 ep_trb_len = 0; 1154 remaining = 0; 1155 break; 1156 } 1157 1158 if (ep_trb == td->last_trb) 1159 ep_trb_len = requested - remaining; 1160 else 1161 ep_trb_len = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb) + 1162 ep_trb_len - remaining; 1163 td->preq->request.actual = ep_trb_len; 1164 1165 finish_td: 1166 ep->stream_info.drbls_count--; 1167 1168 cdnsp_finish_td(pdev, td, event, ep, status); 1169 } 1170 1171 static void cdnsp_handle_tx_nrdy(struct cdnsp_device *pdev, 1172 struct cdnsp_transfer_event *event) 1173 { 1174 struct cdnsp_generic_trb *generic; 1175 struct cdnsp_ring *ep_ring; 1176 struct cdnsp_ep *pep; 1177 int cur_stream; 1178 int ep_index; 1179 int host_sid; 1180 int dev_sid; 1181 1182 generic = (struct cdnsp_generic_trb *)event; 1183 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; 1184 dev_sid = TRB_TO_DEV_STREAM(le32_to_cpu(generic->field[0])); 1185 host_sid = TRB_TO_HOST_STREAM(le32_to_cpu(generic->field[2])); 1186 1187 pep = &pdev->eps[ep_index]; 1188 1189 if (!(pep->ep_state & EP_HAS_STREAMS)) 1190 return; 1191 1192 if (host_sid == STREAM_PRIME_ACK) { 1193 pep->stream_info.first_prime_det = 1; 1194 for (cur_stream = 1; cur_stream < pep->stream_info.num_streams; 1195 cur_stream++) { 1196 ep_ring = pep->stream_info.stream_rings[cur_stream]; 1197 ep_ring->stream_active = 1; 1198 ep_ring->stream_rejected = 0; 1199 } 1200 } 1201 1202 if (host_sid == STREAM_REJECTED) { 1203 struct cdnsp_td *td, *td_temp; 1204 1205 pep->stream_info.drbls_count--; 1206 ep_ring = pep->stream_info.stream_rings[dev_sid]; 1207 ep_ring->stream_active = 0; 1208 ep_ring->stream_rejected = 1; 1209 1210 list_for_each_entry_safe(td, td_temp, &ep_ring->td_list, 1211 td_list) { 1212 td->drbl = 0; 1213 } 1214 } 1215 1216 cdnsp_ring_doorbell_for_active_rings(pdev, pep); 1217 } 1218 1219 /* 1220 * If this function returns an error condition, it means it got a Transfer 1221 * event with a corrupted TRB DMA address or endpoint is disabled. 1222 */ 1223 static int cdnsp_handle_tx_event(struct cdnsp_device *pdev, 1224 struct cdnsp_transfer_event *event) 1225 { 1226 const struct usb_endpoint_descriptor *desc; 1227 bool handling_skipped_tds = false; 1228 struct cdnsp_segment *ep_seg; 1229 struct cdnsp_ring *ep_ring; 1230 int status = -EINPROGRESS; 1231 union cdnsp_trb *ep_trb; 1232 dma_addr_t ep_trb_dma; 1233 struct cdnsp_ep *pep; 1234 struct cdnsp_td *td; 1235 u32 trb_comp_code; 1236 int invalidate; 1237 int ep_index; 1238 1239 invalidate = le32_to_cpu(event->flags) & TRB_EVENT_INVALIDATE; 1240 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; 1241 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 1242 ep_trb_dma = le64_to_cpu(event->buffer); 1243 1244 pep = &pdev->eps[ep_index]; 1245 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 1246 1247 /* 1248 * If device is disconnect then all requests will be dequeued 1249 * by upper layers as part of disconnect sequence. 1250 * We don't want handle such event to avoid racing. 1251 */ 1252 if (invalidate || !pdev->gadget.connected) 1253 goto cleanup; 1254 1255 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_DISABLED) { 1256 trace_cdnsp_ep_disabled(pep->out_ctx); 1257 goto err_out; 1258 } 1259 1260 /* Some transfer events don't always point to a trb*/ 1261 if (!ep_ring) { 1262 switch (trb_comp_code) { 1263 case COMP_INVALID_STREAM_TYPE_ERROR: 1264 case COMP_INVALID_STREAM_ID_ERROR: 1265 case COMP_RING_UNDERRUN: 1266 case COMP_RING_OVERRUN: 1267 goto cleanup; 1268 default: 1269 dev_err(pdev->dev, "ERROR: %s event for unknown ring\n", 1270 pep->name); 1271 goto err_out; 1272 } 1273 } 1274 1275 /* Look for some error cases that need special treatment. */ 1276 switch (trb_comp_code) { 1277 case COMP_BABBLE_DETECTED_ERROR: 1278 status = -EOVERFLOW; 1279 break; 1280 case COMP_RING_UNDERRUN: 1281 case COMP_RING_OVERRUN: 1282 /* 1283 * When the Isoch ring is empty, the controller will generate 1284 * a Ring Overrun Event for IN Isoch endpoint or Ring 1285 * Underrun Event for OUT Isoch endpoint. 1286 */ 1287 goto cleanup; 1288 case COMP_MISSED_SERVICE_ERROR: 1289 /* 1290 * When encounter missed service error, one or more isoc tds 1291 * may be missed by controller. 1292 * Set skip flag of the ep_ring; Complete the missed tds as 1293 * short transfer when process the ep_ring next time. 1294 */ 1295 pep->skip = true; 1296 break; 1297 } 1298 1299 do { 1300 /* 1301 * This TRB should be in the TD at the head of this ring's TD 1302 * list. 1303 */ 1304 if (list_empty(&ep_ring->td_list)) { 1305 /* 1306 * Don't print warnings if it's due to a stopped 1307 * endpoint generating an extra completion event, or 1308 * a event for the last TRB of a short TD we already 1309 * got a short event for. 1310 * The short TD is already removed from the TD list. 1311 */ 1312 if (!(trb_comp_code == COMP_STOPPED || 1313 trb_comp_code == COMP_STOPPED_LENGTH_INVALID || 1314 ep_ring->last_td_was_short)) 1315 trace_cdnsp_trb_without_td(ep_ring, 1316 (struct cdnsp_generic_trb *)event); 1317 1318 if (pep->skip) { 1319 pep->skip = false; 1320 trace_cdnsp_ep_list_empty_with_skip(pep, 0); 1321 } 1322 1323 goto cleanup; 1324 } 1325 1326 td = list_entry(ep_ring->td_list.next, struct cdnsp_td, 1327 td_list); 1328 1329 /* Is this a TRB in the currently executing TD? */ 1330 ep_seg = cdnsp_trb_in_td(pdev, ep_ring->deq_seg, 1331 ep_ring->dequeue, td->last_trb, 1332 ep_trb_dma); 1333 1334 /* 1335 * Skip the Force Stopped Event. The event_trb(ep_trb_dma) 1336 * of FSE is not in the current TD pointed by ep_ring->dequeue 1337 * because that the hardware dequeue pointer still at the 1338 * previous TRB of the current TD. The previous TRB maybe a 1339 * Link TD or the last TRB of the previous TD. The command 1340 * completion handle will take care the rest. 1341 */ 1342 if (!ep_seg && (trb_comp_code == COMP_STOPPED || 1343 trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) { 1344 pep->skip = false; 1345 goto cleanup; 1346 } 1347 1348 desc = td->preq->pep->endpoint.desc; 1349 if (!ep_seg) { 1350 if (!pep->skip || !usb_endpoint_xfer_isoc(desc)) { 1351 /* Something is busted, give up! */ 1352 dev_err(pdev->dev, 1353 "ERROR Transfer event TRB DMA ptr not " 1354 "part of current TD ep_index %d " 1355 "comp_code %u\n", ep_index, 1356 trb_comp_code); 1357 return -EINVAL; 1358 } 1359 1360 cdnsp_skip_isoc_td(pdev, td, event, pep, status); 1361 goto cleanup; 1362 } 1363 1364 if (trb_comp_code == COMP_SHORT_PACKET) 1365 ep_ring->last_td_was_short = true; 1366 else 1367 ep_ring->last_td_was_short = false; 1368 1369 if (pep->skip) { 1370 pep->skip = false; 1371 cdnsp_skip_isoc_td(pdev, td, event, pep, status); 1372 goto cleanup; 1373 } 1374 1375 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) 1376 / sizeof(*ep_trb)]; 1377 1378 trace_cdnsp_handle_transfer(ep_ring, 1379 (struct cdnsp_generic_trb *)ep_trb); 1380 1381 if (cdnsp_trb_is_noop(ep_trb)) 1382 goto cleanup; 1383 1384 if (usb_endpoint_xfer_control(desc)) 1385 cdnsp_process_ctrl_td(pdev, td, ep_trb, event, pep, 1386 &status); 1387 else if (usb_endpoint_xfer_isoc(desc)) 1388 cdnsp_process_isoc_td(pdev, td, ep_trb, event, pep, 1389 status); 1390 else 1391 cdnsp_process_bulk_intr_td(pdev, td, ep_trb, event, pep, 1392 &status); 1393 cleanup: 1394 handling_skipped_tds = pep->skip; 1395 1396 /* 1397 * Do not update event ring dequeue pointer if we're in a loop 1398 * processing missed tds. 1399 */ 1400 if (!handling_skipped_tds) 1401 cdnsp_inc_deq(pdev, pdev->event_ring); 1402 1403 /* 1404 * If ep->skip is set, it means there are missed tds on the 1405 * endpoint ring need to take care of. 1406 * Process them as short transfer until reach the td pointed by 1407 * the event. 1408 */ 1409 } while (handling_skipped_tds); 1410 return 0; 1411 1412 err_out: 1413 dev_err(pdev->dev, "@%016llx %08x %08x %08x %08x\n", 1414 (unsigned long long) 1415 cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg, 1416 pdev->event_ring->dequeue), 1417 lower_32_bits(le64_to_cpu(event->buffer)), 1418 upper_32_bits(le64_to_cpu(event->buffer)), 1419 le32_to_cpu(event->transfer_len), 1420 le32_to_cpu(event->flags)); 1421 return -EINVAL; 1422 } 1423 1424 /* 1425 * This function handles all events on the event ring. 1426 * Returns true for "possibly more events to process" (caller should call 1427 * again), otherwise false if done. 1428 */ 1429 static bool cdnsp_handle_event(struct cdnsp_device *pdev) 1430 { 1431 unsigned int comp_code; 1432 union cdnsp_trb *event; 1433 bool update_ptrs = true; 1434 u32 cycle_bit; 1435 int ret = 0; 1436 u32 flags; 1437 1438 event = pdev->event_ring->dequeue; 1439 flags = le32_to_cpu(event->event_cmd.flags); 1440 cycle_bit = (flags & TRB_CYCLE); 1441 1442 /* Does the controller or driver own the TRB? */ 1443 if (cycle_bit != pdev->event_ring->cycle_state) 1444 return false; 1445 1446 trace_cdnsp_handle_event(pdev->event_ring, &event->generic); 1447 1448 /* 1449 * Barrier between reading the TRB_CYCLE (valid) flag above and any 1450 * reads of the event's flags/data below. 1451 */ 1452 rmb(); 1453 1454 switch (flags & TRB_TYPE_BITMASK) { 1455 case TRB_TYPE(TRB_COMPLETION): 1456 /* 1457 * Command can't be handled in interrupt context so just 1458 * increment command ring dequeue pointer. 1459 */ 1460 cdnsp_inc_deq(pdev, pdev->cmd_ring); 1461 break; 1462 case TRB_TYPE(TRB_PORT_STATUS): 1463 cdnsp_handle_port_status(pdev, event); 1464 update_ptrs = false; 1465 break; 1466 case TRB_TYPE(TRB_TRANSFER): 1467 ret = cdnsp_handle_tx_event(pdev, &event->trans_event); 1468 if (ret >= 0) 1469 update_ptrs = false; 1470 break; 1471 case TRB_TYPE(TRB_SETUP): 1472 pdev->ep0_stage = CDNSP_SETUP_STAGE; 1473 pdev->setup_id = TRB_SETUPID_TO_TYPE(flags); 1474 pdev->setup_speed = TRB_SETUP_SPEEDID(flags); 1475 pdev->setup = *((struct usb_ctrlrequest *) 1476 &event->trans_event.buffer); 1477 1478 cdnsp_setup_analyze(pdev); 1479 break; 1480 case TRB_TYPE(TRB_ENDPOINT_NRDY): 1481 cdnsp_handle_tx_nrdy(pdev, &event->trans_event); 1482 break; 1483 case TRB_TYPE(TRB_HC_EVENT): { 1484 comp_code = GET_COMP_CODE(le32_to_cpu(event->generic.field[2])); 1485 1486 switch (comp_code) { 1487 case COMP_EVENT_RING_FULL_ERROR: 1488 dev_err(pdev->dev, "Event Ring Full\n"); 1489 break; 1490 default: 1491 dev_err(pdev->dev, "Controller error code 0x%02x\n", 1492 comp_code); 1493 } 1494 1495 break; 1496 } 1497 case TRB_TYPE(TRB_MFINDEX_WRAP): 1498 case TRB_TYPE(TRB_DRB_OVERFLOW): 1499 break; 1500 default: 1501 dev_warn(pdev->dev, "ERROR unknown event type %ld\n", 1502 TRB_FIELD_TO_TYPE(flags)); 1503 } 1504 1505 if (update_ptrs) 1506 /* Update SW event ring dequeue pointer. */ 1507 cdnsp_inc_deq(pdev, pdev->event_ring); 1508 1509 /* 1510 * Caller will call us again to check if there are more items 1511 * on the event ring. 1512 */ 1513 return true; 1514 } 1515 1516 irqreturn_t cdnsp_thread_irq_handler(int irq, void *data) 1517 { 1518 struct cdnsp_device *pdev = (struct cdnsp_device *)data; 1519 union cdnsp_trb *event_ring_deq; 1520 int counter = 0; 1521 1522 spin_lock(&pdev->lock); 1523 1524 if (pdev->cdnsp_state & (CDNSP_STATE_HALTED | CDNSP_STATE_DYING)) { 1525 cdnsp_died(pdev); 1526 spin_unlock(&pdev->lock); 1527 return IRQ_HANDLED; 1528 } 1529 1530 event_ring_deq = pdev->event_ring->dequeue; 1531 1532 while (cdnsp_handle_event(pdev)) { 1533 if (++counter >= TRBS_PER_EV_DEQ_UPDATE) { 1534 cdnsp_update_erst_dequeue(pdev, event_ring_deq, 0); 1535 event_ring_deq = pdev->event_ring->dequeue; 1536 counter = 0; 1537 } 1538 } 1539 1540 cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1); 1541 1542 spin_unlock(&pdev->lock); 1543 1544 return IRQ_HANDLED; 1545 } 1546 1547 irqreturn_t cdnsp_irq_handler(int irq, void *priv) 1548 { 1549 struct cdnsp_device *pdev = (struct cdnsp_device *)priv; 1550 u32 irq_pending; 1551 u32 status; 1552 1553 status = readl(&pdev->op_regs->status); 1554 1555 if (status == ~(u32)0) { 1556 cdnsp_died(pdev); 1557 return IRQ_HANDLED; 1558 } 1559 1560 if (!(status & STS_EINT)) 1561 return IRQ_NONE; 1562 1563 writel(status | STS_EINT, &pdev->op_regs->status); 1564 irq_pending = readl(&pdev->ir_set->irq_pending); 1565 irq_pending |= IMAN_IP; 1566 writel(irq_pending, &pdev->ir_set->irq_pending); 1567 1568 if (status & STS_FATAL) { 1569 cdnsp_died(pdev); 1570 return IRQ_HANDLED; 1571 } 1572 1573 return IRQ_WAKE_THREAD; 1574 } 1575 1576 /* 1577 * Generic function for queuing a TRB on a ring. 1578 * The caller must have checked to make sure there's room on the ring. 1579 * 1580 * @more_trbs_coming: Will you enqueue more TRBs before setting doorbell? 1581 */ 1582 static void cdnsp_queue_trb(struct cdnsp_device *pdev, struct cdnsp_ring *ring, 1583 bool more_trbs_coming, u32 field1, u32 field2, 1584 u32 field3, u32 field4) 1585 { 1586 struct cdnsp_generic_trb *trb; 1587 1588 trb = &ring->enqueue->generic; 1589 1590 trb->field[0] = cpu_to_le32(field1); 1591 trb->field[1] = cpu_to_le32(field2); 1592 trb->field[2] = cpu_to_le32(field3); 1593 trb->field[3] = cpu_to_le32(field4); 1594 1595 trace_cdnsp_queue_trb(ring, trb); 1596 cdnsp_inc_enq(pdev, ring, more_trbs_coming); 1597 } 1598 1599 /* 1600 * Does various checks on the endpoint ring, and makes it ready to 1601 * queue num_trbs. 1602 */ 1603 static int cdnsp_prepare_ring(struct cdnsp_device *pdev, 1604 struct cdnsp_ring *ep_ring, 1605 u32 ep_state, unsigned 1606 int num_trbs, 1607 gfp_t mem_flags) 1608 { 1609 unsigned int num_trbs_needed; 1610 1611 /* Make sure the endpoint has been added to controller schedule. */ 1612 switch (ep_state) { 1613 case EP_STATE_STOPPED: 1614 case EP_STATE_RUNNING: 1615 case EP_STATE_HALTED: 1616 break; 1617 default: 1618 dev_err(pdev->dev, "ERROR: incorrect endpoint state\n"); 1619 return -EINVAL; 1620 } 1621 1622 while (1) { 1623 if (cdnsp_room_on_ring(pdev, ep_ring, num_trbs)) 1624 break; 1625 1626 trace_cdnsp_no_room_on_ring("try ring expansion"); 1627 1628 num_trbs_needed = num_trbs - ep_ring->num_trbs_free; 1629 if (cdnsp_ring_expansion(pdev, ep_ring, num_trbs_needed, 1630 mem_flags)) { 1631 dev_err(pdev->dev, "Ring expansion failed\n"); 1632 return -ENOMEM; 1633 } 1634 } 1635 1636 while (cdnsp_trb_is_link(ep_ring->enqueue)) { 1637 ep_ring->enqueue->link.control |= cpu_to_le32(TRB_CHAIN); 1638 /* The cycle bit must be set as the last operation. */ 1639 wmb(); 1640 ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE); 1641 1642 /* Toggle the cycle bit after the last ring segment. */ 1643 if (cdnsp_link_trb_toggles_cycle(ep_ring->enqueue)) 1644 ep_ring->cycle_state ^= 1; 1645 ep_ring->enq_seg = ep_ring->enq_seg->next; 1646 ep_ring->enqueue = ep_ring->enq_seg->trbs; 1647 } 1648 return 0; 1649 } 1650 1651 static int cdnsp_prepare_transfer(struct cdnsp_device *pdev, 1652 struct cdnsp_request *preq, 1653 unsigned int num_trbs) 1654 { 1655 struct cdnsp_ring *ep_ring; 1656 int ret; 1657 1658 ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep, 1659 preq->request.stream_id); 1660 if (!ep_ring) 1661 return -EINVAL; 1662 1663 ret = cdnsp_prepare_ring(pdev, ep_ring, 1664 GET_EP_CTX_STATE(preq->pep->out_ctx), 1665 num_trbs, GFP_ATOMIC); 1666 if (ret) 1667 return ret; 1668 1669 INIT_LIST_HEAD(&preq->td.td_list); 1670 preq->td.preq = preq; 1671 1672 /* Add this TD to the tail of the endpoint ring's TD list. */ 1673 list_add_tail(&preq->td.td_list, &ep_ring->td_list); 1674 ep_ring->num_tds++; 1675 preq->pep->stream_info.td_count++; 1676 1677 preq->td.start_seg = ep_ring->enq_seg; 1678 preq->td.first_trb = ep_ring->enqueue; 1679 1680 return 0; 1681 } 1682 1683 static unsigned int cdnsp_count_trbs(u64 addr, u64 len) 1684 { 1685 unsigned int num_trbs; 1686 1687 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)), 1688 TRB_MAX_BUFF_SIZE); 1689 if (num_trbs == 0) 1690 num_trbs++; 1691 1692 return num_trbs; 1693 } 1694 1695 static unsigned int count_trbs_needed(struct cdnsp_request *preq) 1696 { 1697 return cdnsp_count_trbs(preq->request.dma, preq->request.length); 1698 } 1699 1700 static unsigned int count_sg_trbs_needed(struct cdnsp_request *preq) 1701 { 1702 unsigned int i, len, full_len, num_trbs = 0; 1703 struct scatterlist *sg; 1704 1705 full_len = preq->request.length; 1706 1707 for_each_sg(preq->request.sg, sg, preq->request.num_sgs, i) { 1708 len = sg_dma_len(sg); 1709 num_trbs += cdnsp_count_trbs(sg_dma_address(sg), len); 1710 len = min(len, full_len); 1711 full_len -= len; 1712 if (full_len == 0) 1713 break; 1714 } 1715 1716 return num_trbs; 1717 } 1718 1719 static unsigned int count_isoc_trbs_needed(struct cdnsp_request *preq) 1720 { 1721 return cdnsp_count_trbs(preq->request.dma, preq->request.length); 1722 } 1723 1724 static void cdnsp_check_trb_math(struct cdnsp_request *preq, int running_total) 1725 { 1726 if (running_total != preq->request.length) 1727 dev_err(preq->pep->pdev->dev, 1728 "%s - Miscalculated tx length, " 1729 "queued %#x, asked for %#x (%d)\n", 1730 preq->pep->name, running_total, 1731 preq->request.length, preq->request.actual); 1732 } 1733 1734 /* 1735 * TD size is the number of max packet sized packets remaining in the TD 1736 * (*not* including this TRB). 1737 * 1738 * Total TD packet count = total_packet_count = 1739 * DIV_ROUND_UP(TD size in bytes / wMaxPacketSize) 1740 * 1741 * Packets transferred up to and including this TRB = packets_transferred = 1742 * rounddown(total bytes transferred including this TRB / wMaxPacketSize) 1743 * 1744 * TD size = total_packet_count - packets_transferred 1745 * 1746 * It must fit in bits 21:17, so it can't be bigger than 31. 1747 * This is taken care of in the TRB_TD_SIZE() macro 1748 * 1749 * The last TRB in a TD must have the TD size set to zero. 1750 */ 1751 static u32 cdnsp_td_remainder(struct cdnsp_device *pdev, 1752 int transferred, 1753 int trb_buff_len, 1754 unsigned int td_total_len, 1755 struct cdnsp_request *preq, 1756 bool more_trbs_coming) 1757 { 1758 u32 maxp, total_packet_count; 1759 1760 /* One TRB with a zero-length data packet. */ 1761 if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) || 1762 trb_buff_len == td_total_len) 1763 return 0; 1764 1765 maxp = usb_endpoint_maxp(preq->pep->endpoint.desc); 1766 total_packet_count = DIV_ROUND_UP(td_total_len, maxp); 1767 1768 /* Queuing functions don't count the current TRB into transferred. */ 1769 return (total_packet_count - ((transferred + trb_buff_len) / maxp)); 1770 } 1771 1772 static int cdnsp_align_td(struct cdnsp_device *pdev, 1773 struct cdnsp_request *preq, u32 enqd_len, 1774 u32 *trb_buff_len, struct cdnsp_segment *seg) 1775 { 1776 struct device *dev = pdev->dev; 1777 unsigned int unalign; 1778 unsigned int max_pkt; 1779 u32 new_buff_len; 1780 1781 max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc); 1782 unalign = (enqd_len + *trb_buff_len) % max_pkt; 1783 1784 /* We got lucky, last normal TRB data on segment is packet aligned. */ 1785 if (unalign == 0) 1786 return 0; 1787 1788 /* Is the last nornal TRB alignable by splitting it. */ 1789 if (*trb_buff_len > unalign) { 1790 *trb_buff_len -= unalign; 1791 trace_cdnsp_bounce_align_td_split(preq, *trb_buff_len, 1792 enqd_len, 0, unalign); 1793 return 0; 1794 } 1795 1796 /* 1797 * We want enqd_len + trb_buff_len to sum up to a number aligned to 1798 * number which is divisible by the endpoint's wMaxPacketSize. IOW: 1799 * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0. 1800 */ 1801 new_buff_len = max_pkt - (enqd_len % max_pkt); 1802 1803 if (new_buff_len > (preq->request.length - enqd_len)) 1804 new_buff_len = (preq->request.length - enqd_len); 1805 1806 /* Create a max max_pkt sized bounce buffer pointed to by last trb. */ 1807 if (preq->direction) { 1808 sg_pcopy_to_buffer(preq->request.sg, 1809 preq->request.num_mapped_sgs, 1810 seg->bounce_buf, new_buff_len, enqd_len); 1811 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, 1812 max_pkt, DMA_TO_DEVICE); 1813 } else { 1814 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, 1815 max_pkt, DMA_FROM_DEVICE); 1816 } 1817 1818 if (dma_mapping_error(dev, seg->bounce_dma)) { 1819 /* Try without aligning.*/ 1820 dev_warn(pdev->dev, 1821 "Failed mapping bounce buffer, not aligning\n"); 1822 return 0; 1823 } 1824 1825 *trb_buff_len = new_buff_len; 1826 seg->bounce_len = new_buff_len; 1827 seg->bounce_offs = enqd_len; 1828 1829 trace_cdnsp_bounce_map(preq, new_buff_len, enqd_len, seg->bounce_dma, 1830 unalign); 1831 1832 /* 1833 * Bounce buffer successful aligned and seg->bounce_dma will be used 1834 * in transfer TRB as new transfer buffer address. 1835 */ 1836 return 1; 1837 } 1838 1839 int cdnsp_queue_bulk_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq) 1840 { 1841 unsigned int enqd_len, block_len, trb_buff_len, full_len; 1842 unsigned int start_cycle, num_sgs = 0; 1843 struct cdnsp_generic_trb *start_trb; 1844 u32 field, length_field, remainder; 1845 struct scatterlist *sg = NULL; 1846 bool more_trbs_coming = true; 1847 bool need_zero_pkt = false; 1848 bool zero_len_trb = false; 1849 struct cdnsp_ring *ring; 1850 bool first_trb = true; 1851 unsigned int num_trbs; 1852 struct cdnsp_ep *pep; 1853 u64 addr, send_addr; 1854 int sent_len, ret; 1855 1856 ring = cdnsp_request_to_transfer_ring(pdev, preq); 1857 if (!ring) 1858 return -EINVAL; 1859 1860 full_len = preq->request.length; 1861 1862 if (preq->request.num_sgs) { 1863 num_sgs = preq->request.num_sgs; 1864 sg = preq->request.sg; 1865 addr = (u64)sg_dma_address(sg); 1866 block_len = sg_dma_len(sg); 1867 num_trbs = count_sg_trbs_needed(preq); 1868 } else { 1869 num_trbs = count_trbs_needed(preq); 1870 addr = (u64)preq->request.dma; 1871 block_len = full_len; 1872 } 1873 1874 pep = preq->pep; 1875 1876 /* Deal with request.zero - need one more td/trb. */ 1877 if (preq->request.zero && preq->request.length && 1878 IS_ALIGNED(full_len, usb_endpoint_maxp(pep->endpoint.desc))) { 1879 need_zero_pkt = true; 1880 num_trbs++; 1881 } 1882 1883 ret = cdnsp_prepare_transfer(pdev, preq, num_trbs); 1884 if (ret) 1885 return ret; 1886 1887 /* 1888 * Don't give the first TRB to the hardware (by toggling the cycle bit) 1889 * until we've finished creating all the other TRBs. The ring's cycle 1890 * state may change as we enqueue the other TRBs, so save it too. 1891 */ 1892 start_trb = &ring->enqueue->generic; 1893 start_cycle = ring->cycle_state; 1894 send_addr = addr; 1895 1896 /* Queue the TRBs, even if they are zero-length */ 1897 for (enqd_len = 0; zero_len_trb || first_trb || enqd_len < full_len; 1898 enqd_len += trb_buff_len) { 1899 field = TRB_TYPE(TRB_NORMAL); 1900 1901 /* TRB buffer should not cross 64KB boundaries */ 1902 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr); 1903 trb_buff_len = min(trb_buff_len, block_len); 1904 if (enqd_len + trb_buff_len > full_len) 1905 trb_buff_len = full_len - enqd_len; 1906 1907 /* Don't change the cycle bit of the first TRB until later */ 1908 if (first_trb) { 1909 first_trb = false; 1910 if (start_cycle == 0) 1911 field |= TRB_CYCLE; 1912 } else { 1913 field |= ring->cycle_state; 1914 } 1915 1916 /* 1917 * Chain all the TRBs together; clear the chain bit in the last 1918 * TRB to indicate it's the last TRB in the chain. 1919 */ 1920 if (enqd_len + trb_buff_len < full_len || need_zero_pkt) { 1921 field |= TRB_CHAIN; 1922 if (cdnsp_trb_is_link(ring->enqueue + 1)) { 1923 if (cdnsp_align_td(pdev, preq, enqd_len, 1924 &trb_buff_len, 1925 ring->enq_seg)) { 1926 send_addr = ring->enq_seg->bounce_dma; 1927 /* Assuming TD won't span 2 segs */ 1928 preq->td.bounce_seg = ring->enq_seg; 1929 } 1930 } 1931 } 1932 1933 if (enqd_len + trb_buff_len >= full_len) { 1934 if (need_zero_pkt && zero_len_trb) { 1935 zero_len_trb = true; 1936 } else { 1937 field &= ~TRB_CHAIN; 1938 field |= TRB_IOC; 1939 more_trbs_coming = false; 1940 need_zero_pkt = false; 1941 preq->td.last_trb = ring->enqueue; 1942 } 1943 } 1944 1945 /* Only set interrupt on short packet for OUT endpoints. */ 1946 if (!preq->direction) 1947 field |= TRB_ISP; 1948 1949 /* Set the TRB length, TD size, and interrupter fields. */ 1950 remainder = cdnsp_td_remainder(pdev, enqd_len, trb_buff_len, 1951 full_len, preq, 1952 more_trbs_coming); 1953 1954 length_field = TRB_LEN(trb_buff_len) | TRB_TD_SIZE(remainder) | 1955 TRB_INTR_TARGET(0); 1956 1957 cdnsp_queue_trb(pdev, ring, more_trbs_coming | need_zero_pkt, 1958 lower_32_bits(send_addr), 1959 upper_32_bits(send_addr), 1960 length_field, 1961 field); 1962 1963 addr += trb_buff_len; 1964 sent_len = trb_buff_len; 1965 while (sg && sent_len >= block_len) { 1966 /* New sg entry */ 1967 --num_sgs; 1968 sent_len -= block_len; 1969 if (num_sgs != 0) { 1970 sg = sg_next(sg); 1971 block_len = sg_dma_len(sg); 1972 addr = (u64)sg_dma_address(sg); 1973 addr += sent_len; 1974 } 1975 } 1976 block_len -= sent_len; 1977 send_addr = addr; 1978 } 1979 1980 cdnsp_check_trb_math(preq, enqd_len); 1981 ret = cdnsp_giveback_first_trb(pdev, pep, preq->request.stream_id, 1982 start_cycle, start_trb); 1983 1984 if (ret) 1985 preq->td.drbl = 1; 1986 1987 return 0; 1988 } 1989 1990 int cdnsp_queue_ctrl_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq) 1991 { 1992 u32 field, length_field, remainder; 1993 struct cdnsp_ep *pep = preq->pep; 1994 struct cdnsp_ring *ep_ring; 1995 int num_trbs; 1996 int ret; 1997 1998 ep_ring = cdnsp_request_to_transfer_ring(pdev, preq); 1999 if (!ep_ring) 2000 return -EINVAL; 2001 2002 /* 1 TRB for data, 1 for status */ 2003 num_trbs = (pdev->three_stage_setup) ? 2 : 1; 2004 2005 ret = cdnsp_prepare_transfer(pdev, preq, num_trbs); 2006 if (ret) 2007 return ret; 2008 2009 /* If there's data, queue data TRBs */ 2010 if (pdev->ep0_expect_in) 2011 field = TRB_TYPE(TRB_DATA) | TRB_IOC; 2012 else 2013 field = TRB_ISP | TRB_TYPE(TRB_DATA) | TRB_IOC; 2014 2015 if (preq->request.length > 0) { 2016 remainder = cdnsp_td_remainder(pdev, 0, preq->request.length, 2017 preq->request.length, preq, 1); 2018 2019 length_field = TRB_LEN(preq->request.length) | 2020 TRB_TD_SIZE(remainder) | TRB_INTR_TARGET(0); 2021 2022 if (pdev->ep0_expect_in) 2023 field |= TRB_DIR_IN; 2024 2025 cdnsp_queue_trb(pdev, ep_ring, true, 2026 lower_32_bits(preq->request.dma), 2027 upper_32_bits(preq->request.dma), length_field, 2028 field | ep_ring->cycle_state | 2029 TRB_SETUPID(pdev->setup_id) | 2030 pdev->setup_speed); 2031 2032 pdev->ep0_stage = CDNSP_DATA_STAGE; 2033 } 2034 2035 /* Save the DMA address of the last TRB in the TD. */ 2036 preq->td.last_trb = ep_ring->enqueue; 2037 2038 /* Queue status TRB. */ 2039 if (preq->request.length == 0) 2040 field = ep_ring->cycle_state; 2041 else 2042 field = (ep_ring->cycle_state ^ 1); 2043 2044 if (preq->request.length > 0 && pdev->ep0_expect_in) 2045 field |= TRB_DIR_IN; 2046 2047 if (pep->ep_state & EP0_HALTED_STATUS) { 2048 pep->ep_state &= ~EP0_HALTED_STATUS; 2049 field |= TRB_SETUPSTAT(TRB_SETUPSTAT_STALL); 2050 } else { 2051 field |= TRB_SETUPSTAT(TRB_SETUPSTAT_ACK); 2052 } 2053 2054 cdnsp_queue_trb(pdev, ep_ring, false, 0, 0, TRB_INTR_TARGET(0), 2055 field | TRB_IOC | TRB_SETUPID(pdev->setup_id) | 2056 TRB_TYPE(TRB_STATUS) | pdev->setup_speed); 2057 2058 cdnsp_ring_ep_doorbell(pdev, pep, preq->request.stream_id); 2059 2060 return 0; 2061 } 2062 2063 int cdnsp_cmd_stop_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep) 2064 { 2065 u32 ep_state = GET_EP_CTX_STATE(pep->out_ctx); 2066 int ret = 0; 2067 2068 if (ep_state == EP_STATE_STOPPED || ep_state == EP_STATE_DISABLED) { 2069 trace_cdnsp_ep_stopped_or_disabled(pep->out_ctx); 2070 goto ep_stopped; 2071 } 2072 2073 cdnsp_queue_stop_endpoint(pdev, pep->idx); 2074 cdnsp_ring_cmd_db(pdev); 2075 ret = cdnsp_wait_for_cmd_compl(pdev); 2076 2077 trace_cdnsp_handle_cmd_stop_ep(pep->out_ctx); 2078 2079 ep_stopped: 2080 pep->ep_state |= EP_STOPPED; 2081 return ret; 2082 } 2083 2084 int cdnsp_cmd_flush_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep) 2085 { 2086 int ret; 2087 2088 cdnsp_queue_flush_endpoint(pdev, pep->idx); 2089 cdnsp_ring_cmd_db(pdev); 2090 ret = cdnsp_wait_for_cmd_compl(pdev); 2091 2092 trace_cdnsp_handle_cmd_flush_ep(pep->out_ctx); 2093 2094 return ret; 2095 } 2096 2097 /* 2098 * The transfer burst count field of the isochronous TRB defines the number of 2099 * bursts that are required to move all packets in this TD. Only SuperSpeed 2100 * devices can burst up to bMaxBurst number of packets per service interval. 2101 * This field is zero based, meaning a value of zero in the field means one 2102 * burst. Basically, for everything but SuperSpeed devices, this field will be 2103 * zero. 2104 */ 2105 static unsigned int cdnsp_get_burst_count(struct cdnsp_device *pdev, 2106 struct cdnsp_request *preq, 2107 unsigned int total_packet_count) 2108 { 2109 unsigned int max_burst; 2110 2111 if (pdev->gadget.speed < USB_SPEED_SUPER) 2112 return 0; 2113 2114 max_burst = preq->pep->endpoint.comp_desc->bMaxBurst; 2115 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1; 2116 } 2117 2118 /* 2119 * Returns the number of packets in the last "burst" of packets. This field is 2120 * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so 2121 * the last burst packet count is equal to the total number of packets in the 2122 * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst 2123 * must contain (bMaxBurst + 1) number of packets, but the last burst can 2124 * contain 1 to (bMaxBurst + 1) packets. 2125 */ 2126 static unsigned int 2127 cdnsp_get_last_burst_packet_count(struct cdnsp_device *pdev, 2128 struct cdnsp_request *preq, 2129 unsigned int total_packet_count) 2130 { 2131 unsigned int max_burst; 2132 unsigned int residue; 2133 2134 if (pdev->gadget.speed >= USB_SPEED_SUPER) { 2135 /* bMaxBurst is zero based: 0 means 1 packet per burst. */ 2136 max_burst = preq->pep->endpoint.comp_desc->bMaxBurst; 2137 residue = total_packet_count % (max_burst + 1); 2138 2139 /* 2140 * If residue is zero, the last burst contains (max_burst + 1) 2141 * number of packets, but the TLBPC field is zero-based. 2142 */ 2143 if (residue == 0) 2144 return max_burst; 2145 2146 return residue - 1; 2147 } 2148 if (total_packet_count == 0) 2149 return 0; 2150 2151 return total_packet_count - 1; 2152 } 2153 2154 /* Queue function isoc transfer */ 2155 static int cdnsp_queue_isoc_tx(struct cdnsp_device *pdev, 2156 struct cdnsp_request *preq) 2157 { 2158 int trb_buff_len, td_len, td_remain_len, ret; 2159 unsigned int burst_count, last_burst_pkt; 2160 unsigned int total_pkt_count, max_pkt; 2161 struct cdnsp_generic_trb *start_trb; 2162 bool more_trbs_coming = true; 2163 struct cdnsp_ring *ep_ring; 2164 int running_total = 0; 2165 u32 field, length_field; 2166 int start_cycle; 2167 int trbs_per_td; 2168 u64 addr; 2169 int i; 2170 2171 ep_ring = preq->pep->ring; 2172 start_trb = &ep_ring->enqueue->generic; 2173 start_cycle = ep_ring->cycle_state; 2174 td_len = preq->request.length; 2175 addr = (u64)preq->request.dma; 2176 td_remain_len = td_len; 2177 2178 max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc); 2179 total_pkt_count = DIV_ROUND_UP(td_len, max_pkt); 2180 2181 /* A zero-length transfer still involves at least one packet. */ 2182 if (total_pkt_count == 0) 2183 total_pkt_count++; 2184 2185 burst_count = cdnsp_get_burst_count(pdev, preq, total_pkt_count); 2186 last_burst_pkt = cdnsp_get_last_burst_packet_count(pdev, preq, 2187 total_pkt_count); 2188 trbs_per_td = count_isoc_trbs_needed(preq); 2189 2190 ret = cdnsp_prepare_transfer(pdev, preq, trbs_per_td); 2191 if (ret) 2192 goto cleanup; 2193 2194 /* 2195 * Set isoc specific data for the first TRB in a TD. 2196 * Prevent HW from getting the TRBs by keeping the cycle state 2197 * inverted in the first TDs isoc TRB. 2198 */ 2199 field = TRB_TYPE(TRB_ISOC) | TRB_TLBPC(last_burst_pkt) | 2200 start_cycle ? 0 : 1 | TRB_SIA | TRB_TBC(burst_count); 2201 2202 /* Fill the rest of the TRB fields, and remaining normal TRBs. */ 2203 for (i = 0; i < trbs_per_td; i++) { 2204 u32 remainder; 2205 2206 /* Calculate TRB length. */ 2207 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr); 2208 if (trb_buff_len > td_remain_len) 2209 trb_buff_len = td_remain_len; 2210 2211 /* Set the TRB length, TD size, & interrupter fields. */ 2212 remainder = cdnsp_td_remainder(pdev, running_total, 2213 trb_buff_len, td_len, preq, 2214 more_trbs_coming); 2215 2216 length_field = TRB_LEN(trb_buff_len) | TRB_INTR_TARGET(0); 2217 2218 /* Only first TRB is isoc, overwrite otherwise. */ 2219 if (i) { 2220 field = TRB_TYPE(TRB_NORMAL) | ep_ring->cycle_state; 2221 length_field |= TRB_TD_SIZE(remainder); 2222 } else { 2223 length_field |= TRB_TD_SIZE_TBC(burst_count); 2224 } 2225 2226 /* Only set interrupt on short packet for OUT EPs. */ 2227 if (usb_endpoint_dir_out(preq->pep->endpoint.desc)) 2228 field |= TRB_ISP; 2229 2230 /* Set the chain bit for all except the last TRB. */ 2231 if (i < trbs_per_td - 1) { 2232 more_trbs_coming = true; 2233 field |= TRB_CHAIN; 2234 } else { 2235 more_trbs_coming = false; 2236 preq->td.last_trb = ep_ring->enqueue; 2237 field |= TRB_IOC; 2238 } 2239 2240 cdnsp_queue_trb(pdev, ep_ring, more_trbs_coming, 2241 lower_32_bits(addr), upper_32_bits(addr), 2242 length_field, field); 2243 2244 running_total += trb_buff_len; 2245 addr += trb_buff_len; 2246 td_remain_len -= trb_buff_len; 2247 } 2248 2249 /* Check TD length */ 2250 if (running_total != td_len) { 2251 dev_err(pdev->dev, "ISOC TD length unmatch\n"); 2252 ret = -EINVAL; 2253 goto cleanup; 2254 } 2255 2256 cdnsp_giveback_first_trb(pdev, preq->pep, preq->request.stream_id, 2257 start_cycle, start_trb); 2258 2259 return 0; 2260 2261 cleanup: 2262 /* Clean up a partially enqueued isoc transfer. */ 2263 list_del_init(&preq->td.td_list); 2264 ep_ring->num_tds--; 2265 2266 /* 2267 * Use the first TD as a temporary variable to turn the TDs we've 2268 * queued into No-ops with a software-owned cycle bit. 2269 * That way the hardware won't accidentally start executing bogus TDs 2270 * when we partially overwrite them. 2271 * td->first_trb and td->start_seg are already set. 2272 */ 2273 preq->td.last_trb = ep_ring->enqueue; 2274 /* Every TRB except the first & last will have its cycle bit flipped. */ 2275 cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true); 2276 2277 /* Reset the ring enqueue back to the first TRB and its cycle bit. */ 2278 ep_ring->enqueue = preq->td.first_trb; 2279 ep_ring->enq_seg = preq->td.start_seg; 2280 ep_ring->cycle_state = start_cycle; 2281 return ret; 2282 } 2283 2284 int cdnsp_queue_isoc_tx_prepare(struct cdnsp_device *pdev, 2285 struct cdnsp_request *preq) 2286 { 2287 struct cdnsp_ring *ep_ring; 2288 u32 ep_state; 2289 int num_trbs; 2290 int ret; 2291 2292 ep_ring = preq->pep->ring; 2293 ep_state = GET_EP_CTX_STATE(preq->pep->out_ctx); 2294 num_trbs = count_isoc_trbs_needed(preq); 2295 2296 /* 2297 * Check the ring to guarantee there is enough room for the whole 2298 * request. Do not insert any td of the USB Request to the ring if the 2299 * check failed. 2300 */ 2301 ret = cdnsp_prepare_ring(pdev, ep_ring, ep_state, num_trbs, GFP_ATOMIC); 2302 if (ret) 2303 return ret; 2304 2305 return cdnsp_queue_isoc_tx(pdev, preq); 2306 } 2307 2308 /**** Command Ring Operations ****/ 2309 /* 2310 * Generic function for queuing a command TRB on the command ring. 2311 * Driver queue only one command to ring in the moment. 2312 */ 2313 static void cdnsp_queue_command(struct cdnsp_device *pdev, 2314 u32 field1, 2315 u32 field2, 2316 u32 field3, 2317 u32 field4) 2318 { 2319 cdnsp_prepare_ring(pdev, pdev->cmd_ring, EP_STATE_RUNNING, 1, 2320 GFP_ATOMIC); 2321 2322 pdev->cmd.command_trb = pdev->cmd_ring->enqueue; 2323 2324 cdnsp_queue_trb(pdev, pdev->cmd_ring, false, field1, field2, 2325 field3, field4 | pdev->cmd_ring->cycle_state); 2326 } 2327 2328 /* Queue a slot enable or disable request on the command ring */ 2329 void cdnsp_queue_slot_control(struct cdnsp_device *pdev, u32 trb_type) 2330 { 2331 cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(trb_type) | 2332 SLOT_ID_FOR_TRB(pdev->slot_id)); 2333 } 2334 2335 /* Queue an address device command TRB */ 2336 void cdnsp_queue_address_device(struct cdnsp_device *pdev, 2337 dma_addr_t in_ctx_ptr, 2338 enum cdnsp_setup_dev setup) 2339 { 2340 cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr), 2341 upper_32_bits(in_ctx_ptr), 0, 2342 TRB_TYPE(TRB_ADDR_DEV) | 2343 SLOT_ID_FOR_TRB(pdev->slot_id) | 2344 (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0)); 2345 } 2346 2347 /* Queue a reset device command TRB */ 2348 void cdnsp_queue_reset_device(struct cdnsp_device *pdev) 2349 { 2350 cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_RESET_DEV) | 2351 SLOT_ID_FOR_TRB(pdev->slot_id)); 2352 } 2353 2354 /* Queue a configure endpoint command TRB */ 2355 void cdnsp_queue_configure_endpoint(struct cdnsp_device *pdev, 2356 dma_addr_t in_ctx_ptr) 2357 { 2358 cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr), 2359 upper_32_bits(in_ctx_ptr), 0, 2360 TRB_TYPE(TRB_CONFIG_EP) | 2361 SLOT_ID_FOR_TRB(pdev->slot_id)); 2362 } 2363 2364 /* 2365 * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop 2366 * activity on an endpoint that is about to be suspended. 2367 */ 2368 void cdnsp_queue_stop_endpoint(struct cdnsp_device *pdev, unsigned int ep_index) 2369 { 2370 cdnsp_queue_command(pdev, 0, 0, 0, SLOT_ID_FOR_TRB(pdev->slot_id) | 2371 EP_ID_FOR_TRB(ep_index) | TRB_TYPE(TRB_STOP_RING)); 2372 } 2373 2374 /* Set Transfer Ring Dequeue Pointer command. */ 2375 void cdnsp_queue_new_dequeue_state(struct cdnsp_device *pdev, 2376 struct cdnsp_ep *pep, 2377 struct cdnsp_dequeue_state *deq_state) 2378 { 2379 u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id); 2380 u32 trb_slot_id = SLOT_ID_FOR_TRB(pdev->slot_id); 2381 u32 type = TRB_TYPE(TRB_SET_DEQ); 2382 u32 trb_sct = 0; 2383 dma_addr_t addr; 2384 2385 addr = cdnsp_trb_virt_to_dma(deq_state->new_deq_seg, 2386 deq_state->new_deq_ptr); 2387 2388 if (deq_state->stream_id) 2389 trb_sct = SCT_FOR_TRB(SCT_PRI_TR); 2390 2391 cdnsp_queue_command(pdev, lower_32_bits(addr) | trb_sct | 2392 deq_state->new_cycle_state, upper_32_bits(addr), 2393 trb_stream_id, trb_slot_id | 2394 EP_ID_FOR_TRB(pep->idx) | type); 2395 } 2396 2397 void cdnsp_queue_reset_ep(struct cdnsp_device *pdev, unsigned int ep_index) 2398 { 2399 return cdnsp_queue_command(pdev, 0, 0, 0, 2400 SLOT_ID_FOR_TRB(pdev->slot_id) | 2401 EP_ID_FOR_TRB(ep_index) | 2402 TRB_TYPE(TRB_RESET_EP)); 2403 } 2404 2405 /* 2406 * Queue a halt endpoint request on the command ring. 2407 */ 2408 void cdnsp_queue_halt_endpoint(struct cdnsp_device *pdev, unsigned int ep_index) 2409 { 2410 cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_HALT_ENDPOINT) | 2411 SLOT_ID_FOR_TRB(pdev->slot_id) | 2412 EP_ID_FOR_TRB(ep_index)); 2413 } 2414 2415 /* 2416 * Queue a flush endpoint request on the command ring. 2417 */ 2418 void cdnsp_queue_flush_endpoint(struct cdnsp_device *pdev, 2419 unsigned int ep_index) 2420 { 2421 cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_FLUSH_ENDPOINT) | 2422 SLOT_ID_FOR_TRB(pdev->slot_id) | 2423 EP_ID_FOR_TRB(ep_index)); 2424 } 2425 2426 void cdnsp_force_header_wakeup(struct cdnsp_device *pdev, int intf_num) 2427 { 2428 u32 lo, mid; 2429 2430 lo = TRB_FH_TO_PACKET_TYPE(TRB_FH_TR_PACKET) | 2431 TRB_FH_TO_DEVICE_ADDRESS(pdev->device_address); 2432 mid = TRB_FH_TR_PACKET_DEV_NOT | 2433 TRB_FH_TO_NOT_TYPE(TRB_FH_TR_PACKET_FUNCTION_WAKE) | 2434 TRB_FH_TO_INTERFACE(intf_num); 2435 2436 cdnsp_queue_command(pdev, lo, mid, 0, 2437 TRB_TYPE(TRB_FORCE_HEADER) | SET_PORT_ID(2)); 2438 } 2439