13d829045SPawel Laszczak // SPDX-License-Identifier: GPL-2.0 23d829045SPawel Laszczak /* 33d829045SPawel Laszczak * Cadence CDNSP DRD Driver. 43d829045SPawel Laszczak * 53d829045SPawel Laszczak * Copyright (C) 2020 Cadence. 63d829045SPawel Laszczak * 73d829045SPawel Laszczak * Author: Pawel Laszczak <pawell@cadence.com> 83d829045SPawel Laszczak * 93d829045SPawel Laszczak * Code based on Linux XHCI driver. 103d829045SPawel Laszczak * Origin: Copyright (C) 2008 Intel Corp 113d829045SPawel Laszczak */ 123d829045SPawel Laszczak 133d829045SPawel Laszczak /* 143d829045SPawel Laszczak * Ring initialization rules: 153d829045SPawel Laszczak * 1. Each segment is initialized to zero, except for link TRBs. 163d829045SPawel Laszczak * 2. Ring cycle state = 0. This represents Producer Cycle State (PCS) or 173d829045SPawel Laszczak * Consumer Cycle State (CCS), depending on ring function. 183d829045SPawel Laszczak * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment. 193d829045SPawel Laszczak * 203d829045SPawel Laszczak * Ring behavior rules: 213d829045SPawel Laszczak * 1. A ring is empty if enqueue == dequeue. This means there will always be at 223d829045SPawel Laszczak * least one free TRB in the ring. This is useful if you want to turn that 233d829045SPawel Laszczak * into a link TRB and expand the ring. 243d829045SPawel Laszczak * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a 253d829045SPawel Laszczak * link TRB, then load the pointer with the address in the link TRB. If the 263d829045SPawel Laszczak * link TRB had its toggle bit set, you may need to update the ring cycle 273d829045SPawel Laszczak * state (see cycle bit rules). You may have to do this multiple times 283d829045SPawel Laszczak * until you reach a non-link TRB. 293d829045SPawel Laszczak * 3. A ring is full if enqueue++ (for the definition of increment above) 303d829045SPawel Laszczak * equals the dequeue pointer. 313d829045SPawel Laszczak * 323d829045SPawel Laszczak * Cycle bit rules: 333d829045SPawel Laszczak * 1. When a consumer increments a dequeue pointer and encounters a toggle bit 343d829045SPawel Laszczak * in a link TRB, it must toggle the ring cycle state. 353d829045SPawel Laszczak * 2. When a producer increments an enqueue pointer and encounters a toggle bit 363d829045SPawel Laszczak * in a link TRB, it must toggle the ring cycle state. 373d829045SPawel Laszczak * 383d829045SPawel Laszczak * Producer rules: 393d829045SPawel Laszczak * 1. Check if ring is full before you enqueue. 403d829045SPawel Laszczak * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing. 413d829045SPawel Laszczak * Update enqueue pointer between each write (which may update the ring 423d829045SPawel Laszczak * cycle state). 433d829045SPawel Laszczak * 3. Notify consumer. If SW is producer, it rings the doorbell for command 443d829045SPawel Laszczak * and endpoint rings. If controller is the producer for the event ring, 453d829045SPawel Laszczak * and it generates an interrupt according to interrupt modulation rules. 463d829045SPawel Laszczak * 473d829045SPawel Laszczak * Consumer rules: 483d829045SPawel Laszczak * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state, 493d829045SPawel Laszczak * the TRB is owned by the consumer. 503d829045SPawel Laszczak * 2. Update dequeue pointer (which may update the ring cycle state) and 513d829045SPawel Laszczak * continue processing TRBs until you reach a TRB which is not owned by you. 523d829045SPawel Laszczak * 3. Notify the producer. SW is the consumer for the event ring, and it 533d829045SPawel Laszczak * updates event ring dequeue pointer. Controller is the consumer for the 543d829045SPawel Laszczak * command and endpoint rings; it generates events on the event ring 553d829045SPawel Laszczak * for these. 563d829045SPawel Laszczak */ 573d829045SPawel Laszczak 583d829045SPawel Laszczak #include <linux/scatterlist.h> 593d829045SPawel Laszczak #include <linux/dma-mapping.h> 603d829045SPawel Laszczak #include <linux/delay.h> 613d829045SPawel Laszczak #include <linux/slab.h> 623d829045SPawel Laszczak #include <linux/irq.h> 633d829045SPawel Laszczak 64118b2a32SPawel Laszczak #include "cdnsp-trace.h" 653d829045SPawel Laszczak #include "cdnsp-gadget.h" 663d829045SPawel Laszczak 673d829045SPawel Laszczak /* 683d829045SPawel Laszczak * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA 693d829045SPawel Laszczak * address of the TRB. 703d829045SPawel Laszczak */ 713d829045SPawel Laszczak dma_addr_t cdnsp_trb_virt_to_dma(struct cdnsp_segment *seg, 723d829045SPawel Laszczak union cdnsp_trb *trb) 733d829045SPawel Laszczak { 743d829045SPawel Laszczak unsigned long segment_offset = trb - seg->trbs; 753d829045SPawel Laszczak 763d829045SPawel Laszczak if (trb < seg->trbs || segment_offset >= TRBS_PER_SEGMENT) 773d829045SPawel Laszczak return 0; 783d829045SPawel Laszczak 793d829045SPawel Laszczak return seg->dma + (segment_offset * sizeof(*trb)); 803d829045SPawel Laszczak } 813d829045SPawel Laszczak 823d829045SPawel Laszczak static bool cdnsp_trb_is_noop(union cdnsp_trb *trb) 833d829045SPawel Laszczak { 843d829045SPawel Laszczak return TRB_TYPE_NOOP_LE32(trb->generic.field[3]); 853d829045SPawel Laszczak } 863d829045SPawel Laszczak 873d829045SPawel Laszczak static bool cdnsp_trb_is_link(union cdnsp_trb *trb) 883d829045SPawel Laszczak { 893d829045SPawel Laszczak return TRB_TYPE_LINK_LE32(trb->link.control); 903d829045SPawel Laszczak } 913d829045SPawel Laszczak 923d829045SPawel Laszczak bool cdnsp_last_trb_on_seg(struct cdnsp_segment *seg, union cdnsp_trb *trb) 933d829045SPawel Laszczak { 943d829045SPawel Laszczak return trb == &seg->trbs[TRBS_PER_SEGMENT - 1]; 953d829045SPawel Laszczak } 963d829045SPawel Laszczak 973d829045SPawel Laszczak bool cdnsp_last_trb_on_ring(struct cdnsp_ring *ring, 983d829045SPawel Laszczak struct cdnsp_segment *seg, 993d829045SPawel Laszczak union cdnsp_trb *trb) 1003d829045SPawel Laszczak { 1013d829045SPawel Laszczak return cdnsp_last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg); 1023d829045SPawel Laszczak } 1033d829045SPawel Laszczak 1043d829045SPawel Laszczak static bool cdnsp_link_trb_toggles_cycle(union cdnsp_trb *trb) 1053d829045SPawel Laszczak { 1063d829045SPawel Laszczak return le32_to_cpu(trb->link.control) & LINK_TOGGLE; 1073d829045SPawel Laszczak } 1083d829045SPawel Laszczak 1093d829045SPawel Laszczak static void cdnsp_trb_to_noop(union cdnsp_trb *trb, u32 noop_type) 1103d829045SPawel Laszczak { 1113d829045SPawel Laszczak if (cdnsp_trb_is_link(trb)) { 1123d829045SPawel Laszczak /* Unchain chained link TRBs. */ 1133d829045SPawel Laszczak trb->link.control &= cpu_to_le32(~TRB_CHAIN); 1143d829045SPawel Laszczak } else { 1153d829045SPawel Laszczak trb->generic.field[0] = 0; 1163d829045SPawel Laszczak trb->generic.field[1] = 0; 1173d829045SPawel Laszczak trb->generic.field[2] = 0; 1183d829045SPawel Laszczak /* Preserve only the cycle bit of this TRB. */ 1193d829045SPawel Laszczak trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE); 1203d829045SPawel Laszczak trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type)); 1213d829045SPawel Laszczak } 1223d829045SPawel Laszczak } 1233d829045SPawel Laszczak 1243d829045SPawel Laszczak /* 1253d829045SPawel Laszczak * Updates trb to point to the next TRB in the ring, and updates seg if the next 1263d829045SPawel Laszczak * TRB is in a new segment. This does not skip over link TRBs, and it does not 1273d829045SPawel Laszczak * effect the ring dequeue or enqueue pointers. 1283d829045SPawel Laszczak */ 1293d829045SPawel Laszczak static void cdnsp_next_trb(struct cdnsp_device *pdev, 1303d829045SPawel Laszczak struct cdnsp_ring *ring, 1313d829045SPawel Laszczak struct cdnsp_segment **seg, 1323d829045SPawel Laszczak union cdnsp_trb **trb) 1333d829045SPawel Laszczak { 1343d829045SPawel Laszczak if (cdnsp_trb_is_link(*trb)) { 1353d829045SPawel Laszczak *seg = (*seg)->next; 1363d829045SPawel Laszczak *trb = ((*seg)->trbs); 1373d829045SPawel Laszczak } else { 1383d829045SPawel Laszczak (*trb)++; 1393d829045SPawel Laszczak } 1403d829045SPawel Laszczak } 1413d829045SPawel Laszczak 1423d829045SPawel Laszczak /* 1433d829045SPawel Laszczak * See Cycle bit rules. SW is the consumer for the event ring only. 1443d829045SPawel Laszczak * Don't make a ring full of link TRBs. That would be dumb and this would loop. 1453d829045SPawel Laszczak */ 1463d829045SPawel Laszczak void cdnsp_inc_deq(struct cdnsp_device *pdev, struct cdnsp_ring *ring) 1473d829045SPawel Laszczak { 1483d829045SPawel Laszczak /* event ring doesn't have link trbs, check for last trb. */ 1493d829045SPawel Laszczak if (ring->type == TYPE_EVENT) { 1503d829045SPawel Laszczak if (!cdnsp_last_trb_on_seg(ring->deq_seg, ring->dequeue)) { 1513d829045SPawel Laszczak ring->dequeue++; 152118b2a32SPawel Laszczak goto out; 1533d829045SPawel Laszczak } 1543d829045SPawel Laszczak 1553d829045SPawel Laszczak if (cdnsp_last_trb_on_ring(ring, ring->deq_seg, ring->dequeue)) 1563d829045SPawel Laszczak ring->cycle_state ^= 1; 1573d829045SPawel Laszczak 1583d829045SPawel Laszczak ring->deq_seg = ring->deq_seg->next; 1593d829045SPawel Laszczak ring->dequeue = ring->deq_seg->trbs; 160118b2a32SPawel Laszczak goto out; 1613d829045SPawel Laszczak } 1623d829045SPawel Laszczak 1633d829045SPawel Laszczak /* All other rings have link trbs. */ 1643d829045SPawel Laszczak if (!cdnsp_trb_is_link(ring->dequeue)) { 1653d829045SPawel Laszczak ring->dequeue++; 1663d829045SPawel Laszczak ring->num_trbs_free++; 1673d829045SPawel Laszczak } 1683d829045SPawel Laszczak while (cdnsp_trb_is_link(ring->dequeue)) { 1693d829045SPawel Laszczak ring->deq_seg = ring->deq_seg->next; 1703d829045SPawel Laszczak ring->dequeue = ring->deq_seg->trbs; 1713d829045SPawel Laszczak } 172118b2a32SPawel Laszczak out: 173118b2a32SPawel Laszczak trace_cdnsp_inc_deq(ring); 1743d829045SPawel Laszczak } 1753d829045SPawel Laszczak 1763d829045SPawel Laszczak /* 1773d829045SPawel Laszczak * See Cycle bit rules. SW is the consumer for the event ring only. 1783d829045SPawel Laszczak * Don't make a ring full of link TRBs. That would be dumb and this would loop. 1793d829045SPawel Laszczak * 1803d829045SPawel Laszczak * If we've just enqueued a TRB that is in the middle of a TD (meaning the 1813d829045SPawel Laszczak * chain bit is set), then set the chain bit in all the following link TRBs. 1823d829045SPawel Laszczak * If we've enqueued the last TRB in a TD, make sure the following link TRBs 1833d829045SPawel Laszczak * have their chain bit cleared (so that each Link TRB is a separate TD). 1843d829045SPawel Laszczak * 1853d829045SPawel Laszczak * @more_trbs_coming: Will you enqueue more TRBs before ringing the doorbell. 1863d829045SPawel Laszczak */ 1873d829045SPawel Laszczak static void cdnsp_inc_enq(struct cdnsp_device *pdev, 1883d829045SPawel Laszczak struct cdnsp_ring *ring, 1893d829045SPawel Laszczak bool more_trbs_coming) 1903d829045SPawel Laszczak { 1913d829045SPawel Laszczak union cdnsp_trb *next; 1923d829045SPawel Laszczak u32 chain; 1933d829045SPawel Laszczak 1943d829045SPawel Laszczak chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN; 1953d829045SPawel Laszczak 1963d829045SPawel Laszczak /* If this is not event ring, there is one less usable TRB. */ 1973d829045SPawel Laszczak if (!cdnsp_trb_is_link(ring->enqueue)) 1983d829045SPawel Laszczak ring->num_trbs_free--; 1993d829045SPawel Laszczak next = ++(ring->enqueue); 2003d829045SPawel Laszczak 2013d829045SPawel Laszczak /* Update the dequeue pointer further if that was a link TRB */ 2023d829045SPawel Laszczak while (cdnsp_trb_is_link(next)) { 2033d829045SPawel Laszczak /* 2043d829045SPawel Laszczak * If the caller doesn't plan on enqueuing more TDs before 2053d829045SPawel Laszczak * ringing the doorbell, then we don't want to give the link TRB 2063d829045SPawel Laszczak * to the hardware just yet. We'll give the link TRB back in 2073d829045SPawel Laszczak * cdnsp_prepare_ring() just before we enqueue the TD at the 2083d829045SPawel Laszczak * top of the ring. 2093d829045SPawel Laszczak */ 2103d829045SPawel Laszczak if (!chain && !more_trbs_coming) 2113d829045SPawel Laszczak break; 2123d829045SPawel Laszczak 2133d829045SPawel Laszczak next->link.control &= cpu_to_le32(~TRB_CHAIN); 2143d829045SPawel Laszczak next->link.control |= cpu_to_le32(chain); 2153d829045SPawel Laszczak 2163d829045SPawel Laszczak /* Give this link TRB to the hardware */ 2173d829045SPawel Laszczak wmb(); 2183d829045SPawel Laszczak next->link.control ^= cpu_to_le32(TRB_CYCLE); 2193d829045SPawel Laszczak 2203d829045SPawel Laszczak /* Toggle the cycle bit after the last ring segment. */ 2213d829045SPawel Laszczak if (cdnsp_link_trb_toggles_cycle(next)) 2223d829045SPawel Laszczak ring->cycle_state ^= 1; 2233d829045SPawel Laszczak 2243d829045SPawel Laszczak ring->enq_seg = ring->enq_seg->next; 2253d829045SPawel Laszczak ring->enqueue = ring->enq_seg->trbs; 2263d829045SPawel Laszczak next = ring->enqueue; 2273d829045SPawel Laszczak } 228118b2a32SPawel Laszczak 229118b2a32SPawel Laszczak trace_cdnsp_inc_enq(ring); 2303d829045SPawel Laszczak } 2313d829045SPawel Laszczak 2323d829045SPawel Laszczak /* 2333d829045SPawel Laszczak * Check to see if there's room to enqueue num_trbs on the ring and make sure 2343d829045SPawel Laszczak * enqueue pointer will not advance into dequeue segment. 2353d829045SPawel Laszczak */ 2363d829045SPawel Laszczak static bool cdnsp_room_on_ring(struct cdnsp_device *pdev, 2373d829045SPawel Laszczak struct cdnsp_ring *ring, 2383d829045SPawel Laszczak unsigned int num_trbs) 2393d829045SPawel Laszczak { 2403d829045SPawel Laszczak int num_trbs_in_deq_seg; 2413d829045SPawel Laszczak 2423d829045SPawel Laszczak if (ring->num_trbs_free < num_trbs) 2433d829045SPawel Laszczak return false; 2443d829045SPawel Laszczak 2453d829045SPawel Laszczak if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) { 2463d829045SPawel Laszczak num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs; 2473d829045SPawel Laszczak 2483d829045SPawel Laszczak if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg) 2493d829045SPawel Laszczak return false; 2503d829045SPawel Laszczak } 2513d829045SPawel Laszczak 2523d829045SPawel Laszczak return true; 2533d829045SPawel Laszczak } 2543d829045SPawel Laszczak 2553d829045SPawel Laszczak /* 2563d829045SPawel Laszczak * Workaround for L1: controller has issue with resuming from L1 after 2573d829045SPawel Laszczak * setting doorbell for endpoint during L1 state. This function forces 2583d829045SPawel Laszczak * resume signal in such case. 2593d829045SPawel Laszczak */ 2603d829045SPawel Laszczak static void cdnsp_force_l0_go(struct cdnsp_device *pdev) 2613d829045SPawel Laszczak { 2623d829045SPawel Laszczak if (pdev->active_port == &pdev->usb2_port && pdev->gadget.lpm_capable) 2633d829045SPawel Laszczak cdnsp_set_link_state(pdev, &pdev->active_port->regs->portsc, XDEV_U0); 2643d829045SPawel Laszczak } 2653d829045SPawel Laszczak 2663d829045SPawel Laszczak /* Ring the doorbell after placing a command on the ring. */ 2673d829045SPawel Laszczak void cdnsp_ring_cmd_db(struct cdnsp_device *pdev) 2683d829045SPawel Laszczak { 2693d829045SPawel Laszczak writel(DB_VALUE_CMD, &pdev->dba->cmd_db); 2703d829045SPawel Laszczak } 2713d829045SPawel Laszczak 2723d829045SPawel Laszczak /* 2733d829045SPawel Laszczak * Ring the doorbell after placing a transfer on the ring. 2743d829045SPawel Laszczak * Returns true if doorbell was set, otherwise false. 2753d829045SPawel Laszczak */ 2763d829045SPawel Laszczak static bool cdnsp_ring_ep_doorbell(struct cdnsp_device *pdev, 2773d829045SPawel Laszczak struct cdnsp_ep *pep, 2783d829045SPawel Laszczak unsigned int stream_id) 2793d829045SPawel Laszczak { 2803d829045SPawel Laszczak __le32 __iomem *reg_addr = &pdev->dba->ep_db; 2813d829045SPawel Laszczak unsigned int ep_state = pep->ep_state; 2823d829045SPawel Laszczak unsigned int db_value; 2833d829045SPawel Laszczak 2843d829045SPawel Laszczak /* 2853d829045SPawel Laszczak * Don't ring the doorbell for this endpoint if endpoint is halted or 2863d829045SPawel Laszczak * disabled. 2873d829045SPawel Laszczak */ 2883d829045SPawel Laszczak if (ep_state & EP_HALTED || !(ep_state & EP_ENABLED)) 2893d829045SPawel Laszczak return false; 2903d829045SPawel Laszczak 2913d829045SPawel Laszczak /* For stream capable endpoints driver can ring doorbell only twice. */ 2923d829045SPawel Laszczak if (pep->ep_state & EP_HAS_STREAMS) { 2933d829045SPawel Laszczak if (pep->stream_info.drbls_count >= 2) 2943d829045SPawel Laszczak return false; 2953d829045SPawel Laszczak 2963d829045SPawel Laszczak pep->stream_info.drbls_count++; 2973d829045SPawel Laszczak } 2983d829045SPawel Laszczak 2993d829045SPawel Laszczak pep->ep_state &= ~EP_STOPPED; 3003d829045SPawel Laszczak 3013d829045SPawel Laszczak if (pep->idx == 0 && pdev->ep0_stage == CDNSP_DATA_STAGE && 3023d829045SPawel Laszczak !pdev->ep0_expect_in) 3033d829045SPawel Laszczak db_value = DB_VALUE_EP0_OUT(pep->idx, stream_id); 3043d829045SPawel Laszczak else 3053d829045SPawel Laszczak db_value = DB_VALUE(pep->idx, stream_id); 3063d829045SPawel Laszczak 307118b2a32SPawel Laszczak trace_cdnsp_tr_drbl(pep, stream_id); 308118b2a32SPawel Laszczak 3093d829045SPawel Laszczak writel(db_value, reg_addr); 3103d829045SPawel Laszczak 3113d829045SPawel Laszczak cdnsp_force_l0_go(pdev); 3123d829045SPawel Laszczak 3133d829045SPawel Laszczak /* Doorbell was set. */ 3143d829045SPawel Laszczak return true; 3153d829045SPawel Laszczak } 3163d829045SPawel Laszczak 3173d829045SPawel Laszczak /* 3183d829045SPawel Laszczak * Get the right ring for the given pep and stream_id. 3193d829045SPawel Laszczak * If the endpoint supports streams, boundary check the USB request's stream ID. 3203d829045SPawel Laszczak * If the endpoint doesn't support streams, return the singular endpoint ring. 3213d829045SPawel Laszczak */ 3223d829045SPawel Laszczak static struct cdnsp_ring *cdnsp_get_transfer_ring(struct cdnsp_device *pdev, 3233d829045SPawel Laszczak struct cdnsp_ep *pep, 3243d829045SPawel Laszczak unsigned int stream_id) 3253d829045SPawel Laszczak { 3263d829045SPawel Laszczak if (!(pep->ep_state & EP_HAS_STREAMS)) 3273d829045SPawel Laszczak return pep->ring; 3283d829045SPawel Laszczak 3293d829045SPawel Laszczak if (stream_id == 0 || stream_id >= pep->stream_info.num_streams) { 3303d829045SPawel Laszczak dev_err(pdev->dev, "ERR: %s ring doesn't exist for SID: %d.\n", 3313d829045SPawel Laszczak pep->name, stream_id); 3323d829045SPawel Laszczak return NULL; 3333d829045SPawel Laszczak } 3343d829045SPawel Laszczak 3353d829045SPawel Laszczak return pep->stream_info.stream_rings[stream_id]; 3363d829045SPawel Laszczak } 3373d829045SPawel Laszczak 3383d829045SPawel Laszczak static struct cdnsp_ring * 3393d829045SPawel Laszczak cdnsp_request_to_transfer_ring(struct cdnsp_device *pdev, 3403d829045SPawel Laszczak struct cdnsp_request *preq) 3413d829045SPawel Laszczak { 3423d829045SPawel Laszczak return cdnsp_get_transfer_ring(pdev, preq->pep, 3433d829045SPawel Laszczak preq->request.stream_id); 3443d829045SPawel Laszczak } 3453d829045SPawel Laszczak 3463d829045SPawel Laszczak /* Ring the doorbell for any rings with pending requests. */ 3473d829045SPawel Laszczak void cdnsp_ring_doorbell_for_active_rings(struct cdnsp_device *pdev, 3483d829045SPawel Laszczak struct cdnsp_ep *pep) 3493d829045SPawel Laszczak { 3503d829045SPawel Laszczak struct cdnsp_stream_info *stream_info; 3513d829045SPawel Laszczak unsigned int stream_id; 3523d829045SPawel Laszczak int ret; 3533d829045SPawel Laszczak 3543d829045SPawel Laszczak if (pep->ep_state & EP_DIS_IN_RROGRESS) 3553d829045SPawel Laszczak return; 3563d829045SPawel Laszczak 3573d829045SPawel Laszczak /* A ring has pending Request if its TD list is not empty. */ 3583d829045SPawel Laszczak if (!(pep->ep_state & EP_HAS_STREAMS) && pep->number) { 3593d829045SPawel Laszczak if (pep->ring && !list_empty(&pep->ring->td_list)) 3603d829045SPawel Laszczak cdnsp_ring_ep_doorbell(pdev, pep, 0); 3613d829045SPawel Laszczak return; 3623d829045SPawel Laszczak } 3633d829045SPawel Laszczak 3643d829045SPawel Laszczak stream_info = &pep->stream_info; 3653d829045SPawel Laszczak 3663d829045SPawel Laszczak for (stream_id = 1; stream_id < stream_info->num_streams; stream_id++) { 3673d829045SPawel Laszczak struct cdnsp_td *td, *td_temp; 3683d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 3693d829045SPawel Laszczak 3703d829045SPawel Laszczak if (stream_info->drbls_count >= 2) 3713d829045SPawel Laszczak return; 3723d829045SPawel Laszczak 3733d829045SPawel Laszczak ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id); 3743d829045SPawel Laszczak if (!ep_ring) 3753d829045SPawel Laszczak continue; 3763d829045SPawel Laszczak 3773d829045SPawel Laszczak if (!ep_ring->stream_active || ep_ring->stream_rejected) 3783d829045SPawel Laszczak continue; 3793d829045SPawel Laszczak 3803d829045SPawel Laszczak list_for_each_entry_safe(td, td_temp, &ep_ring->td_list, 3813d829045SPawel Laszczak td_list) { 3823d829045SPawel Laszczak if (td->drbl) 3833d829045SPawel Laszczak continue; 3843d829045SPawel Laszczak 3853d829045SPawel Laszczak ret = cdnsp_ring_ep_doorbell(pdev, pep, stream_id); 3863d829045SPawel Laszczak if (ret) 3873d829045SPawel Laszczak td->drbl = 1; 3883d829045SPawel Laszczak } 3893d829045SPawel Laszczak } 3903d829045SPawel Laszczak } 3913d829045SPawel Laszczak 3923d829045SPawel Laszczak /* 3933d829045SPawel Laszczak * Get the hw dequeue pointer controller stopped on, either directly from the 3943d829045SPawel Laszczak * endpoint context, or if streams are in use from the stream context. 3953d829045SPawel Laszczak * The returned hw_dequeue contains the lowest four bits with cycle state 3963d829045SPawel Laszczak * and possible stream context type. 3973d829045SPawel Laszczak */ 3983d829045SPawel Laszczak static u64 cdnsp_get_hw_deq(struct cdnsp_device *pdev, 3993d829045SPawel Laszczak unsigned int ep_index, 4003d829045SPawel Laszczak unsigned int stream_id) 4013d829045SPawel Laszczak { 4023d829045SPawel Laszczak struct cdnsp_stream_ctx *st_ctx; 4033d829045SPawel Laszczak struct cdnsp_ep *pep; 4043d829045SPawel Laszczak 4053d829045SPawel Laszczak pep = &pdev->eps[stream_id]; 4063d829045SPawel Laszczak 4073d829045SPawel Laszczak if (pep->ep_state & EP_HAS_STREAMS) { 4083d829045SPawel Laszczak st_ctx = &pep->stream_info.stream_ctx_array[stream_id]; 4093d829045SPawel Laszczak return le64_to_cpu(st_ctx->stream_ring); 4103d829045SPawel Laszczak } 4113d829045SPawel Laszczak 4123d829045SPawel Laszczak return le64_to_cpu(pep->out_ctx->deq); 4133d829045SPawel Laszczak } 4143d829045SPawel Laszczak 4153d829045SPawel Laszczak /* 4163d829045SPawel Laszczak * Move the controller endpoint ring dequeue pointer past cur_td. 4173d829045SPawel Laszczak * Record the new state of the controller endpoint ring dequeue segment, 4183d829045SPawel Laszczak * dequeue pointer, and new consumer cycle state in state. 4193d829045SPawel Laszczak * Update internal representation of the ring's dequeue pointer. 4203d829045SPawel Laszczak * 4213d829045SPawel Laszczak * We do this in three jumps: 4223d829045SPawel Laszczak * - First we update our new ring state to be the same as when the 4233d829045SPawel Laszczak * controller stopped. 4243d829045SPawel Laszczak * - Then we traverse the ring to find the segment that contains 4253d829045SPawel Laszczak * the last TRB in the TD. We toggle the controller new cycle state 4263d829045SPawel Laszczak * when we pass any link TRBs with the toggle cycle bit set. 4273d829045SPawel Laszczak * - Finally we move the dequeue state one TRB further, toggling the cycle bit 4283d829045SPawel Laszczak * if we've moved it past a link TRB with the toggle cycle bit set. 4293d829045SPawel Laszczak */ 4303d829045SPawel Laszczak static void cdnsp_find_new_dequeue_state(struct cdnsp_device *pdev, 4313d829045SPawel Laszczak struct cdnsp_ep *pep, 4323d829045SPawel Laszczak unsigned int stream_id, 4333d829045SPawel Laszczak struct cdnsp_td *cur_td, 4343d829045SPawel Laszczak struct cdnsp_dequeue_state *state) 4353d829045SPawel Laszczak { 4363d829045SPawel Laszczak bool td_last_trb_found = false; 4373d829045SPawel Laszczak struct cdnsp_segment *new_seg; 4383d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 4393d829045SPawel Laszczak union cdnsp_trb *new_deq; 4403d829045SPawel Laszczak bool cycle_found = false; 4413d829045SPawel Laszczak u64 hw_dequeue; 4423d829045SPawel Laszczak 4433d829045SPawel Laszczak ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id); 4443d829045SPawel Laszczak if (!ep_ring) 4453d829045SPawel Laszczak return; 4463d829045SPawel Laszczak 4473d829045SPawel Laszczak /* 4483d829045SPawel Laszczak * Dig out the cycle state saved by the controller during the 4493d829045SPawel Laszczak * stop endpoint command. 4503d829045SPawel Laszczak */ 4513d829045SPawel Laszczak hw_dequeue = cdnsp_get_hw_deq(pdev, pep->idx, stream_id); 4523d829045SPawel Laszczak new_seg = ep_ring->deq_seg; 4533d829045SPawel Laszczak new_deq = ep_ring->dequeue; 4543d829045SPawel Laszczak state->new_cycle_state = hw_dequeue & 0x1; 4553d829045SPawel Laszczak state->stream_id = stream_id; 4563d829045SPawel Laszczak 4573d829045SPawel Laszczak /* 4583d829045SPawel Laszczak * We want to find the pointer, segment and cycle state of the new trb 4593d829045SPawel Laszczak * (the one after current TD's last_trb). We know the cycle state at 4603d829045SPawel Laszczak * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are 4613d829045SPawel Laszczak * found. 4623d829045SPawel Laszczak */ 4633d829045SPawel Laszczak do { 4643d829045SPawel Laszczak if (!cycle_found && cdnsp_trb_virt_to_dma(new_seg, new_deq) 4653d829045SPawel Laszczak == (dma_addr_t)(hw_dequeue & ~0xf)) { 4663d829045SPawel Laszczak cycle_found = true; 4673d829045SPawel Laszczak 4683d829045SPawel Laszczak if (td_last_trb_found) 4693d829045SPawel Laszczak break; 4703d829045SPawel Laszczak } 4713d829045SPawel Laszczak 4723d829045SPawel Laszczak if (new_deq == cur_td->last_trb) 4733d829045SPawel Laszczak td_last_trb_found = true; 4743d829045SPawel Laszczak 4753d829045SPawel Laszczak if (cycle_found && cdnsp_trb_is_link(new_deq) && 4763d829045SPawel Laszczak cdnsp_link_trb_toggles_cycle(new_deq)) 4773d829045SPawel Laszczak state->new_cycle_state ^= 0x1; 4783d829045SPawel Laszczak 4793d829045SPawel Laszczak cdnsp_next_trb(pdev, ep_ring, &new_seg, &new_deq); 4803d829045SPawel Laszczak 4813d829045SPawel Laszczak /* Search wrapped around, bail out. */ 4823d829045SPawel Laszczak if (new_deq == pep->ring->dequeue) { 4833d829045SPawel Laszczak dev_err(pdev->dev, 4843d829045SPawel Laszczak "Error: Failed finding new dequeue state\n"); 4853d829045SPawel Laszczak state->new_deq_seg = NULL; 4863d829045SPawel Laszczak state->new_deq_ptr = NULL; 4873d829045SPawel Laszczak return; 4883d829045SPawel Laszczak } 4893d829045SPawel Laszczak 4903d829045SPawel Laszczak } while (!cycle_found || !td_last_trb_found); 4913d829045SPawel Laszczak 4923d829045SPawel Laszczak state->new_deq_seg = new_seg; 4933d829045SPawel Laszczak state->new_deq_ptr = new_deq; 494118b2a32SPawel Laszczak 495118b2a32SPawel Laszczak trace_cdnsp_new_deq_state(state); 4963d829045SPawel Laszczak } 4973d829045SPawel Laszczak 4983d829045SPawel Laszczak /* 4993d829045SPawel Laszczak * flip_cycle means flip the cycle bit of all but the first and last TRB. 5003d829045SPawel Laszczak * (The last TRB actually points to the ring enqueue pointer, which is not part 5013d829045SPawel Laszczak * of this TD.) This is used to remove partially enqueued isoc TDs from a ring. 5023d829045SPawel Laszczak */ 5033d829045SPawel Laszczak static void cdnsp_td_to_noop(struct cdnsp_device *pdev, 5043d829045SPawel Laszczak struct cdnsp_ring *ep_ring, 5053d829045SPawel Laszczak struct cdnsp_td *td, 5063d829045SPawel Laszczak bool flip_cycle) 5073d829045SPawel Laszczak { 5083d829045SPawel Laszczak struct cdnsp_segment *seg = td->start_seg; 5093d829045SPawel Laszczak union cdnsp_trb *trb = td->first_trb; 5103d829045SPawel Laszczak 5113d829045SPawel Laszczak while (1) { 5123d829045SPawel Laszczak cdnsp_trb_to_noop(trb, TRB_TR_NOOP); 5133d829045SPawel Laszczak 5143d829045SPawel Laszczak /* flip cycle if asked to */ 5153d829045SPawel Laszczak if (flip_cycle && trb != td->first_trb && trb != td->last_trb) 5163d829045SPawel Laszczak trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE); 5173d829045SPawel Laszczak 5183d829045SPawel Laszczak if (trb == td->last_trb) 5193d829045SPawel Laszczak break; 5203d829045SPawel Laszczak 5213d829045SPawel Laszczak cdnsp_next_trb(pdev, ep_ring, &seg, &trb); 5223d829045SPawel Laszczak } 5233d829045SPawel Laszczak } 5243d829045SPawel Laszczak 5253d829045SPawel Laszczak /* 5263d829045SPawel Laszczak * This TD is defined by the TRBs starting at start_trb in start_seg and ending 5273d829045SPawel Laszczak * at end_trb, which may be in another segment. If the suspect DMA address is a 5283d829045SPawel Laszczak * TRB in this TD, this function returns that TRB's segment. Otherwise it 5293d829045SPawel Laszczak * returns 0. 5303d829045SPawel Laszczak */ 5313d829045SPawel Laszczak static struct cdnsp_segment *cdnsp_trb_in_td(struct cdnsp_device *pdev, 5323d829045SPawel Laszczak struct cdnsp_segment *start_seg, 5333d829045SPawel Laszczak union cdnsp_trb *start_trb, 5343d829045SPawel Laszczak union cdnsp_trb *end_trb, 5353d829045SPawel Laszczak dma_addr_t suspect_dma) 5363d829045SPawel Laszczak { 5373d829045SPawel Laszczak struct cdnsp_segment *cur_seg; 5383d829045SPawel Laszczak union cdnsp_trb *temp_trb; 5393d829045SPawel Laszczak dma_addr_t end_seg_dma; 5403d829045SPawel Laszczak dma_addr_t end_trb_dma; 5413d829045SPawel Laszczak dma_addr_t start_dma; 5423d829045SPawel Laszczak 5433d829045SPawel Laszczak start_dma = cdnsp_trb_virt_to_dma(start_seg, start_trb); 5443d829045SPawel Laszczak cur_seg = start_seg; 5453d829045SPawel Laszczak 5463d829045SPawel Laszczak do { 5473d829045SPawel Laszczak if (start_dma == 0) 5483d829045SPawel Laszczak return NULL; 5493d829045SPawel Laszczak 5503d829045SPawel Laszczak temp_trb = &cur_seg->trbs[TRBS_PER_SEGMENT - 1]; 5513d829045SPawel Laszczak /* We may get an event for a Link TRB in the middle of a TD */ 5523d829045SPawel Laszczak end_seg_dma = cdnsp_trb_virt_to_dma(cur_seg, temp_trb); 5533d829045SPawel Laszczak /* If the end TRB isn't in this segment, this is set to 0 */ 5543d829045SPawel Laszczak end_trb_dma = cdnsp_trb_virt_to_dma(cur_seg, end_trb); 5553d829045SPawel Laszczak 556118b2a32SPawel Laszczak trace_cdnsp_looking_trb_in_td(suspect_dma, start_dma, 557118b2a32SPawel Laszczak end_trb_dma, cur_seg->dma, 558118b2a32SPawel Laszczak end_seg_dma); 559118b2a32SPawel Laszczak 5603d829045SPawel Laszczak if (end_trb_dma > 0) { 5613d829045SPawel Laszczak /* 5623d829045SPawel Laszczak * The end TRB is in this segment, so suspect should 5633d829045SPawel Laszczak * be here 5643d829045SPawel Laszczak */ 5653d829045SPawel Laszczak if (start_dma <= end_trb_dma) { 5663d829045SPawel Laszczak if (suspect_dma >= start_dma && 5673d829045SPawel Laszczak suspect_dma <= end_trb_dma) { 5683d829045SPawel Laszczak return cur_seg; 5693d829045SPawel Laszczak } 5703d829045SPawel Laszczak } else { 5713d829045SPawel Laszczak /* 5723d829045SPawel Laszczak * Case for one segment with a 5733d829045SPawel Laszczak * TD wrapped around to the top 5743d829045SPawel Laszczak */ 5753d829045SPawel Laszczak if ((suspect_dma >= start_dma && 5763d829045SPawel Laszczak suspect_dma <= end_seg_dma) || 5773d829045SPawel Laszczak (suspect_dma >= cur_seg->dma && 5783d829045SPawel Laszczak suspect_dma <= end_trb_dma)) { 5793d829045SPawel Laszczak return cur_seg; 5803d829045SPawel Laszczak } 5813d829045SPawel Laszczak } 5823d829045SPawel Laszczak 5833d829045SPawel Laszczak return NULL; 5843d829045SPawel Laszczak } 5853d829045SPawel Laszczak 5863d829045SPawel Laszczak /* Might still be somewhere in this segment */ 5873d829045SPawel Laszczak if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma) 5883d829045SPawel Laszczak return cur_seg; 5893d829045SPawel Laszczak 5903d829045SPawel Laszczak cur_seg = cur_seg->next; 5913d829045SPawel Laszczak start_dma = cdnsp_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]); 5923d829045SPawel Laszczak } while (cur_seg != start_seg); 5933d829045SPawel Laszczak 5943d829045SPawel Laszczak return NULL; 5953d829045SPawel Laszczak } 5963d829045SPawel Laszczak 5973d829045SPawel Laszczak static void cdnsp_unmap_td_bounce_buffer(struct cdnsp_device *pdev, 5983d829045SPawel Laszczak struct cdnsp_ring *ring, 5993d829045SPawel Laszczak struct cdnsp_td *td) 6003d829045SPawel Laszczak { 6013d829045SPawel Laszczak struct cdnsp_segment *seg = td->bounce_seg; 6023d829045SPawel Laszczak struct cdnsp_request *preq; 6033d829045SPawel Laszczak size_t len; 6043d829045SPawel Laszczak 6053d829045SPawel Laszczak if (!seg) 6063d829045SPawel Laszczak return; 6073d829045SPawel Laszczak 6083d829045SPawel Laszczak preq = td->preq; 6093d829045SPawel Laszczak 610118b2a32SPawel Laszczak trace_cdnsp_bounce_unmap(td->preq, seg->bounce_len, seg->bounce_offs, 611118b2a32SPawel Laszczak seg->bounce_dma, 0); 612118b2a32SPawel Laszczak 6133d829045SPawel Laszczak if (!preq->direction) { 6143d829045SPawel Laszczak dma_unmap_single(pdev->dev, seg->bounce_dma, 6153d829045SPawel Laszczak ring->bounce_buf_len, DMA_TO_DEVICE); 6163d829045SPawel Laszczak return; 6173d829045SPawel Laszczak } 6183d829045SPawel Laszczak 6193d829045SPawel Laszczak dma_unmap_single(pdev->dev, seg->bounce_dma, ring->bounce_buf_len, 6203d829045SPawel Laszczak DMA_FROM_DEVICE); 6213d829045SPawel Laszczak 6223d829045SPawel Laszczak /* For in transfers we need to copy the data from bounce to sg */ 6233d829045SPawel Laszczak len = sg_pcopy_from_buffer(preq->request.sg, preq->request.num_sgs, 6243d829045SPawel Laszczak seg->bounce_buf, seg->bounce_len, 6253d829045SPawel Laszczak seg->bounce_offs); 6263d829045SPawel Laszczak if (len != seg->bounce_len) 6273d829045SPawel Laszczak dev_warn(pdev->dev, "WARN Wrong bounce buffer read length: %zu != %d\n", 6283d829045SPawel Laszczak len, seg->bounce_len); 6293d829045SPawel Laszczak 6303d829045SPawel Laszczak seg->bounce_len = 0; 6313d829045SPawel Laszczak seg->bounce_offs = 0; 6323d829045SPawel Laszczak } 6333d829045SPawel Laszczak 6343d829045SPawel Laszczak static int cdnsp_cmd_set_deq(struct cdnsp_device *pdev, 6353d829045SPawel Laszczak struct cdnsp_ep *pep, 6363d829045SPawel Laszczak struct cdnsp_dequeue_state *deq_state) 6373d829045SPawel Laszczak { 6383d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 6393d829045SPawel Laszczak int ret; 6403d829045SPawel Laszczak 6413d829045SPawel Laszczak if (!deq_state->new_deq_ptr || !deq_state->new_deq_seg) { 6423d829045SPawel Laszczak cdnsp_ring_doorbell_for_active_rings(pdev, pep); 6433d829045SPawel Laszczak return 0; 6443d829045SPawel Laszczak } 6453d829045SPawel Laszczak 6463d829045SPawel Laszczak cdnsp_queue_new_dequeue_state(pdev, pep, deq_state); 6473d829045SPawel Laszczak cdnsp_ring_cmd_db(pdev); 6483d829045SPawel Laszczak ret = cdnsp_wait_for_cmd_compl(pdev); 6493d829045SPawel Laszczak 650118b2a32SPawel Laszczak trace_cdnsp_handle_cmd_set_deq(cdnsp_get_slot_ctx(&pdev->out_ctx)); 651118b2a32SPawel Laszczak trace_cdnsp_handle_cmd_set_deq_ep(pep->out_ctx); 652118b2a32SPawel Laszczak 6533d829045SPawel Laszczak /* 6543d829045SPawel Laszczak * Update the ring's dequeue segment and dequeue pointer 6553d829045SPawel Laszczak * to reflect the new position. 6563d829045SPawel Laszczak */ 6573d829045SPawel Laszczak ep_ring = cdnsp_get_transfer_ring(pdev, pep, deq_state->stream_id); 6583d829045SPawel Laszczak 6593d829045SPawel Laszczak if (cdnsp_trb_is_link(ep_ring->dequeue)) { 6603d829045SPawel Laszczak ep_ring->deq_seg = ep_ring->deq_seg->next; 6613d829045SPawel Laszczak ep_ring->dequeue = ep_ring->deq_seg->trbs; 6623d829045SPawel Laszczak } 6633d829045SPawel Laszczak 6643d829045SPawel Laszczak while (ep_ring->dequeue != deq_state->new_deq_ptr) { 6653d829045SPawel Laszczak ep_ring->num_trbs_free++; 6663d829045SPawel Laszczak ep_ring->dequeue++; 6673d829045SPawel Laszczak 6683d829045SPawel Laszczak if (cdnsp_trb_is_link(ep_ring->dequeue)) { 6693d829045SPawel Laszczak if (ep_ring->dequeue == deq_state->new_deq_ptr) 6703d829045SPawel Laszczak break; 6713d829045SPawel Laszczak 6723d829045SPawel Laszczak ep_ring->deq_seg = ep_ring->deq_seg->next; 6733d829045SPawel Laszczak ep_ring->dequeue = ep_ring->deq_seg->trbs; 6743d829045SPawel Laszczak } 6753d829045SPawel Laszczak } 6763d829045SPawel Laszczak 6773d829045SPawel Laszczak /* 6783d829045SPawel Laszczak * Probably there was TIMEOUT during handling Set Dequeue Pointer 6793d829045SPawel Laszczak * command. It's critical error and controller will be stopped. 6803d829045SPawel Laszczak */ 6813d829045SPawel Laszczak if (ret) 6823d829045SPawel Laszczak return -ESHUTDOWN; 6833d829045SPawel Laszczak 6843d829045SPawel Laszczak /* Restart any rings with pending requests */ 6853d829045SPawel Laszczak cdnsp_ring_doorbell_for_active_rings(pdev, pep); 6863d829045SPawel Laszczak 6873d829045SPawel Laszczak return 0; 6883d829045SPawel Laszczak } 6893d829045SPawel Laszczak 6903d829045SPawel Laszczak int cdnsp_remove_request(struct cdnsp_device *pdev, 6913d829045SPawel Laszczak struct cdnsp_request *preq, 6923d829045SPawel Laszczak struct cdnsp_ep *pep) 6933d829045SPawel Laszczak { 6943d829045SPawel Laszczak struct cdnsp_dequeue_state deq_state; 6953d829045SPawel Laszczak struct cdnsp_td *cur_td = NULL; 6963d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 6973d829045SPawel Laszczak struct cdnsp_segment *seg; 6983d829045SPawel Laszczak int status = -ECONNRESET; 6993d829045SPawel Laszczak int ret = 0; 7003d829045SPawel Laszczak u64 hw_deq; 7013d829045SPawel Laszczak 7023d829045SPawel Laszczak memset(&deq_state, 0, sizeof(deq_state)); 7033d829045SPawel Laszczak 704118b2a32SPawel Laszczak trace_cdnsp_remove_request(pep->out_ctx); 705118b2a32SPawel Laszczak trace_cdnsp_remove_request_td(preq); 706118b2a32SPawel Laszczak 7073d829045SPawel Laszczak cur_td = &preq->td; 7083d829045SPawel Laszczak ep_ring = cdnsp_request_to_transfer_ring(pdev, preq); 7093d829045SPawel Laszczak 7103d829045SPawel Laszczak /* 7113d829045SPawel Laszczak * If we stopped on the TD we need to cancel, then we have to 7123d829045SPawel Laszczak * move the controller endpoint ring dequeue pointer past 7133d829045SPawel Laszczak * this TD. 7143d829045SPawel Laszczak */ 7153d829045SPawel Laszczak hw_deq = cdnsp_get_hw_deq(pdev, pep->idx, preq->request.stream_id); 7163d829045SPawel Laszczak hw_deq &= ~0xf; 7173d829045SPawel Laszczak 7183d829045SPawel Laszczak seg = cdnsp_trb_in_td(pdev, cur_td->start_seg, cur_td->first_trb, 7193d829045SPawel Laszczak cur_td->last_trb, hw_deq); 7203d829045SPawel Laszczak 7213d829045SPawel Laszczak if (seg && (pep->ep_state & EP_ENABLED)) 7223d829045SPawel Laszczak cdnsp_find_new_dequeue_state(pdev, pep, preq->request.stream_id, 7233d829045SPawel Laszczak cur_td, &deq_state); 7243d829045SPawel Laszczak else 7253d829045SPawel Laszczak cdnsp_td_to_noop(pdev, ep_ring, cur_td, false); 7263d829045SPawel Laszczak 7273d829045SPawel Laszczak /* 7283d829045SPawel Laszczak * The event handler won't see a completion for this TD anymore, 7293d829045SPawel Laszczak * so remove it from the endpoint ring's TD list. 7303d829045SPawel Laszczak */ 7313d829045SPawel Laszczak list_del_init(&cur_td->td_list); 7323d829045SPawel Laszczak ep_ring->num_tds--; 7333d829045SPawel Laszczak pep->stream_info.td_count--; 7343d829045SPawel Laszczak 7353d829045SPawel Laszczak /* 7363d829045SPawel Laszczak * During disconnecting all endpoint will be disabled so we don't 7373d829045SPawel Laszczak * have to worry about updating dequeue pointer. 7383d829045SPawel Laszczak */ 7393d829045SPawel Laszczak if (pdev->cdnsp_state & CDNSP_STATE_DISCONNECT_PENDING) { 7403d829045SPawel Laszczak status = -ESHUTDOWN; 7413d829045SPawel Laszczak ret = cdnsp_cmd_set_deq(pdev, pep, &deq_state); 7423d829045SPawel Laszczak } 7433d829045SPawel Laszczak 7443d829045SPawel Laszczak cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, cur_td); 7453d829045SPawel Laszczak cdnsp_gadget_giveback(pep, cur_td->preq, status); 7463d829045SPawel Laszczak 7473d829045SPawel Laszczak return ret; 7483d829045SPawel Laszczak } 7493d829045SPawel Laszczak 7503d829045SPawel Laszczak static int cdnsp_update_port_id(struct cdnsp_device *pdev, u32 port_id) 7513d829045SPawel Laszczak { 7523d829045SPawel Laszczak struct cdnsp_port *port = pdev->active_port; 7533d829045SPawel Laszczak u8 old_port = 0; 7543d829045SPawel Laszczak 7553d829045SPawel Laszczak if (port && port->port_num == port_id) 7563d829045SPawel Laszczak return 0; 7573d829045SPawel Laszczak 7583d829045SPawel Laszczak if (port) 7593d829045SPawel Laszczak old_port = port->port_num; 7603d829045SPawel Laszczak 7613d829045SPawel Laszczak if (port_id == pdev->usb2_port.port_num) { 7623d829045SPawel Laszczak port = &pdev->usb2_port; 7633d829045SPawel Laszczak } else if (port_id == pdev->usb3_port.port_num) { 7643d829045SPawel Laszczak port = &pdev->usb3_port; 7653d829045SPawel Laszczak } else { 7663d829045SPawel Laszczak dev_err(pdev->dev, "Port event with invalid port ID %d\n", 7673d829045SPawel Laszczak port_id); 7683d829045SPawel Laszczak return -EINVAL; 7693d829045SPawel Laszczak } 7703d829045SPawel Laszczak 7713d829045SPawel Laszczak if (port_id != old_port) { 7723d829045SPawel Laszczak cdnsp_disable_slot(pdev); 7733d829045SPawel Laszczak pdev->active_port = port; 7743d829045SPawel Laszczak cdnsp_enable_slot(pdev); 7753d829045SPawel Laszczak } 7763d829045SPawel Laszczak 7773d829045SPawel Laszczak if (port_id == pdev->usb2_port.port_num) 7783d829045SPawel Laszczak cdnsp_set_usb2_hardware_lpm(pdev, NULL, 1); 7793d829045SPawel Laszczak else 7803d829045SPawel Laszczak writel(PORT_U1_TIMEOUT(1) | PORT_U2_TIMEOUT(1), 7813d829045SPawel Laszczak &pdev->usb3_port.regs->portpmsc); 7823d829045SPawel Laszczak 7833d829045SPawel Laszczak return 0; 7843d829045SPawel Laszczak } 7853d829045SPawel Laszczak 7863d829045SPawel Laszczak static void cdnsp_handle_port_status(struct cdnsp_device *pdev, 7873d829045SPawel Laszczak union cdnsp_trb *event) 7883d829045SPawel Laszczak { 7893d829045SPawel Laszczak struct cdnsp_port_regs __iomem *port_regs; 7903d829045SPawel Laszczak u32 portsc, cmd_regs; 7913d829045SPawel Laszczak bool port2 = false; 7923d829045SPawel Laszczak u32 link_state; 7933d829045SPawel Laszczak u32 port_id; 7943d829045SPawel Laszczak 7953d829045SPawel Laszczak /* Port status change events always have a successful completion code */ 7963d829045SPawel Laszczak if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) 7973d829045SPawel Laszczak dev_err(pdev->dev, "ERR: incorrect PSC event\n"); 7983d829045SPawel Laszczak 7993d829045SPawel Laszczak port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); 8003d829045SPawel Laszczak 8013d829045SPawel Laszczak if (cdnsp_update_port_id(pdev, port_id)) 8023d829045SPawel Laszczak goto cleanup; 8033d829045SPawel Laszczak 8043d829045SPawel Laszczak port_regs = pdev->active_port->regs; 8053d829045SPawel Laszczak 8063d829045SPawel Laszczak if (port_id == pdev->usb2_port.port_num) 8073d829045SPawel Laszczak port2 = true; 8083d829045SPawel Laszczak 8093d829045SPawel Laszczak new_event: 8103d829045SPawel Laszczak portsc = readl(&port_regs->portsc); 8113d829045SPawel Laszczak writel(cdnsp_port_state_to_neutral(portsc) | 8123d829045SPawel Laszczak (portsc & PORT_CHANGE_BITS), &port_regs->portsc); 8133d829045SPawel Laszczak 814118b2a32SPawel Laszczak trace_cdnsp_handle_port_status(pdev->active_port->port_num, portsc); 815118b2a32SPawel Laszczak 8163d829045SPawel Laszczak pdev->gadget.speed = cdnsp_port_speed(portsc); 8173d829045SPawel Laszczak link_state = portsc & PORT_PLS_MASK; 8183d829045SPawel Laszczak 8193d829045SPawel Laszczak /* Port Link State change detected. */ 8203d829045SPawel Laszczak if ((portsc & PORT_PLC)) { 8213d829045SPawel Laszczak if (!(pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) && 8223d829045SPawel Laszczak link_state == XDEV_RESUME) { 8233d829045SPawel Laszczak cmd_regs = readl(&pdev->op_regs->command); 8243d829045SPawel Laszczak if (!(cmd_regs & CMD_R_S)) 8253d829045SPawel Laszczak goto cleanup; 8263d829045SPawel Laszczak 8273d829045SPawel Laszczak if (DEV_SUPERSPEED_ANY(portsc)) { 8283d829045SPawel Laszczak cdnsp_set_link_state(pdev, &port_regs->portsc, 8293d829045SPawel Laszczak XDEV_U0); 8303d829045SPawel Laszczak 8313d829045SPawel Laszczak cdnsp_resume_gadget(pdev); 8323d829045SPawel Laszczak } 8333d829045SPawel Laszczak } 8343d829045SPawel Laszczak 8353d829045SPawel Laszczak if ((pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) && 8363d829045SPawel Laszczak link_state == XDEV_U0) { 8373d829045SPawel Laszczak pdev->cdnsp_state &= ~CDNSP_WAKEUP_PENDING; 8383d829045SPawel Laszczak 8393d829045SPawel Laszczak cdnsp_force_header_wakeup(pdev, 1); 8403d829045SPawel Laszczak cdnsp_ring_cmd_db(pdev); 8413d829045SPawel Laszczak cdnsp_wait_for_cmd_compl(pdev); 8423d829045SPawel Laszczak } 8433d829045SPawel Laszczak 8443d829045SPawel Laszczak if (link_state == XDEV_U0 && pdev->link_state == XDEV_U3 && 8453d829045SPawel Laszczak !DEV_SUPERSPEED_ANY(portsc)) 8463d829045SPawel Laszczak cdnsp_resume_gadget(pdev); 8473d829045SPawel Laszczak 8483d829045SPawel Laszczak if (link_state == XDEV_U3 && pdev->link_state != XDEV_U3) 8493d829045SPawel Laszczak cdnsp_suspend_gadget(pdev); 8503d829045SPawel Laszczak 8513d829045SPawel Laszczak pdev->link_state = link_state; 8523d829045SPawel Laszczak } 8533d829045SPawel Laszczak 8543d829045SPawel Laszczak if (portsc & PORT_CSC) { 8553d829045SPawel Laszczak /* Detach device. */ 8563d829045SPawel Laszczak if (pdev->gadget.connected && !(portsc & PORT_CONNECT)) 8573d829045SPawel Laszczak cdnsp_disconnect_gadget(pdev); 8583d829045SPawel Laszczak 8593d829045SPawel Laszczak /* Attach device. */ 8603d829045SPawel Laszczak if (portsc & PORT_CONNECT) { 8613d829045SPawel Laszczak if (!port2) 8623d829045SPawel Laszczak cdnsp_irq_reset(pdev); 8633d829045SPawel Laszczak 8643d829045SPawel Laszczak usb_gadget_set_state(&pdev->gadget, USB_STATE_ATTACHED); 8653d829045SPawel Laszczak } 8663d829045SPawel Laszczak } 8673d829045SPawel Laszczak 8683d829045SPawel Laszczak /* Port reset. */ 8693d829045SPawel Laszczak if ((portsc & (PORT_RC | PORT_WRC)) && (portsc & PORT_CONNECT)) { 8703d829045SPawel Laszczak cdnsp_irq_reset(pdev); 8713d829045SPawel Laszczak pdev->u1_allowed = 0; 8723d829045SPawel Laszczak pdev->u2_allowed = 0; 8733d829045SPawel Laszczak pdev->may_wakeup = 0; 8743d829045SPawel Laszczak } 8753d829045SPawel Laszczak 8763d829045SPawel Laszczak if (portsc & PORT_CEC) 8773d829045SPawel Laszczak dev_err(pdev->dev, "Port Over Current detected\n"); 8783d829045SPawel Laszczak 8793d829045SPawel Laszczak if (portsc & PORT_CEC) 8803d829045SPawel Laszczak dev_err(pdev->dev, "Port Configure Error detected\n"); 8813d829045SPawel Laszczak 8823d829045SPawel Laszczak if (readl(&port_regs->portsc) & PORT_CHANGE_BITS) 8833d829045SPawel Laszczak goto new_event; 8843d829045SPawel Laszczak 8853d829045SPawel Laszczak cleanup: 8863d829045SPawel Laszczak cdnsp_inc_deq(pdev, pdev->event_ring); 8873d829045SPawel Laszczak } 8883d829045SPawel Laszczak 8893d829045SPawel Laszczak static void cdnsp_td_cleanup(struct cdnsp_device *pdev, 8903d829045SPawel Laszczak struct cdnsp_td *td, 8913d829045SPawel Laszczak struct cdnsp_ring *ep_ring, 8923d829045SPawel Laszczak int *status) 8933d829045SPawel Laszczak { 8943d829045SPawel Laszczak struct cdnsp_request *preq = td->preq; 8953d829045SPawel Laszczak 8963d829045SPawel Laszczak /* if a bounce buffer was used to align this td then unmap it */ 8973d829045SPawel Laszczak cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, td); 8983d829045SPawel Laszczak 8993d829045SPawel Laszczak /* 9003d829045SPawel Laszczak * If the controller said we transferred more data than the buffer 9013d829045SPawel Laszczak * length, Play it safe and say we didn't transfer anything. 9023d829045SPawel Laszczak */ 9033d829045SPawel Laszczak if (preq->request.actual > preq->request.length) { 9043d829045SPawel Laszczak preq->request.actual = 0; 9053d829045SPawel Laszczak *status = 0; 9063d829045SPawel Laszczak } 9073d829045SPawel Laszczak 9083d829045SPawel Laszczak list_del_init(&td->td_list); 9093d829045SPawel Laszczak ep_ring->num_tds--; 9103d829045SPawel Laszczak preq->pep->stream_info.td_count--; 9113d829045SPawel Laszczak 9123d829045SPawel Laszczak cdnsp_gadget_giveback(preq->pep, preq, *status); 9133d829045SPawel Laszczak } 9143d829045SPawel Laszczak 9153d829045SPawel Laszczak static void cdnsp_finish_td(struct cdnsp_device *pdev, 9163d829045SPawel Laszczak struct cdnsp_td *td, 9173d829045SPawel Laszczak struct cdnsp_transfer_event *event, 9183d829045SPawel Laszczak struct cdnsp_ep *ep, 9193d829045SPawel Laszczak int *status) 9203d829045SPawel Laszczak { 9213d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 9223d829045SPawel Laszczak u32 trb_comp_code; 9233d829045SPawel Laszczak 9243d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); 9253d829045SPawel Laszczak trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 9263d829045SPawel Laszczak 9273d829045SPawel Laszczak if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID || 9283d829045SPawel Laszczak trb_comp_code == COMP_STOPPED || 9293d829045SPawel Laszczak trb_comp_code == COMP_STOPPED_SHORT_PACKET) { 9303d829045SPawel Laszczak /* 9313d829045SPawel Laszczak * The Endpoint Stop Command completion will take care of any 9323d829045SPawel Laszczak * stopped TDs. A stopped TD may be restarted, so don't update 9333d829045SPawel Laszczak * the ring dequeue pointer or take this TD off any lists yet. 9343d829045SPawel Laszczak */ 9353d829045SPawel Laszczak return; 9363d829045SPawel Laszczak } 9373d829045SPawel Laszczak 9383d829045SPawel Laszczak /* Update ring dequeue pointer */ 9393d829045SPawel Laszczak while (ep_ring->dequeue != td->last_trb) 9403d829045SPawel Laszczak cdnsp_inc_deq(pdev, ep_ring); 9413d829045SPawel Laszczak 9423d829045SPawel Laszczak cdnsp_inc_deq(pdev, ep_ring); 9433d829045SPawel Laszczak 9443d829045SPawel Laszczak cdnsp_td_cleanup(pdev, td, ep_ring, status); 9453d829045SPawel Laszczak } 9463d829045SPawel Laszczak 9473d829045SPawel Laszczak /* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */ 9483d829045SPawel Laszczak static int cdnsp_sum_trb_lengths(struct cdnsp_device *pdev, 9493d829045SPawel Laszczak struct cdnsp_ring *ring, 9503d829045SPawel Laszczak union cdnsp_trb *stop_trb) 9513d829045SPawel Laszczak { 9523d829045SPawel Laszczak struct cdnsp_segment *seg = ring->deq_seg; 9533d829045SPawel Laszczak union cdnsp_trb *trb = ring->dequeue; 9543d829045SPawel Laszczak u32 sum; 9553d829045SPawel Laszczak 9563d829045SPawel Laszczak for (sum = 0; trb != stop_trb; cdnsp_next_trb(pdev, ring, &seg, &trb)) { 9573d829045SPawel Laszczak if (!cdnsp_trb_is_noop(trb) && !cdnsp_trb_is_link(trb)) 9583d829045SPawel Laszczak sum += TRB_LEN(le32_to_cpu(trb->generic.field[2])); 9593d829045SPawel Laszczak } 9603d829045SPawel Laszczak return sum; 9613d829045SPawel Laszczak } 9623d829045SPawel Laszczak 9633d829045SPawel Laszczak static int cdnsp_giveback_first_trb(struct cdnsp_device *pdev, 9643d829045SPawel Laszczak struct cdnsp_ep *pep, 9653d829045SPawel Laszczak unsigned int stream_id, 9663d829045SPawel Laszczak int start_cycle, 9673d829045SPawel Laszczak struct cdnsp_generic_trb *start_trb) 9683d829045SPawel Laszczak { 9693d829045SPawel Laszczak /* 9703d829045SPawel Laszczak * Pass all the TRBs to the hardware at once and make sure this write 9713d829045SPawel Laszczak * isn't reordered. 9723d829045SPawel Laszczak */ 9733d829045SPawel Laszczak wmb(); 9743d829045SPawel Laszczak 9753d829045SPawel Laszczak if (start_cycle) 9763d829045SPawel Laszczak start_trb->field[3] |= cpu_to_le32(start_cycle); 9773d829045SPawel Laszczak else 9783d829045SPawel Laszczak start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); 9793d829045SPawel Laszczak 9803d829045SPawel Laszczak if ((pep->ep_state & EP_HAS_STREAMS) && 981118b2a32SPawel Laszczak !pep->stream_info.first_prime_det) { 982118b2a32SPawel Laszczak trace_cdnsp_wait_for_prime(pep, stream_id); 9833d829045SPawel Laszczak return 0; 984118b2a32SPawel Laszczak } 9853d829045SPawel Laszczak 9863d829045SPawel Laszczak return cdnsp_ring_ep_doorbell(pdev, pep, stream_id); 9873d829045SPawel Laszczak } 9883d829045SPawel Laszczak 9893d829045SPawel Laszczak /* 9903d829045SPawel Laszczak * Process control tds, update USB request status and actual_length. 9913d829045SPawel Laszczak */ 9923d829045SPawel Laszczak static void cdnsp_process_ctrl_td(struct cdnsp_device *pdev, 9933d829045SPawel Laszczak struct cdnsp_td *td, 9943d829045SPawel Laszczak union cdnsp_trb *event_trb, 9953d829045SPawel Laszczak struct cdnsp_transfer_event *event, 9963d829045SPawel Laszczak struct cdnsp_ep *pep, 9973d829045SPawel Laszczak int *status) 9983d829045SPawel Laszczak { 9993d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 10003d829045SPawel Laszczak u32 remaining; 10013d829045SPawel Laszczak u32 trb_type; 10023d829045SPawel Laszczak 10033d829045SPawel Laszczak trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event_trb->generic.field[3])); 10043d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 10053d829045SPawel Laszczak remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 10063d829045SPawel Laszczak 10073d829045SPawel Laszczak /* 10083d829045SPawel Laszczak * if on data stage then update the actual_length of the USB 10093d829045SPawel Laszczak * request and flag it as set, so it won't be overwritten in the event 10103d829045SPawel Laszczak * for the last TRB. 10113d829045SPawel Laszczak */ 10123d829045SPawel Laszczak if (trb_type == TRB_DATA) { 10133d829045SPawel Laszczak td->request_length_set = true; 10143d829045SPawel Laszczak td->preq->request.actual = td->preq->request.length - remaining; 10153d829045SPawel Laszczak } 10163d829045SPawel Laszczak 10173d829045SPawel Laszczak /* at status stage */ 10183d829045SPawel Laszczak if (!td->request_length_set) 10193d829045SPawel Laszczak td->preq->request.actual = td->preq->request.length; 10203d829045SPawel Laszczak 10213d829045SPawel Laszczak if (pdev->ep0_stage == CDNSP_DATA_STAGE && pep->number == 0 && 10223d829045SPawel Laszczak pdev->three_stage_setup) { 10233d829045SPawel Laszczak td = list_entry(ep_ring->td_list.next, struct cdnsp_td, 10243d829045SPawel Laszczak td_list); 10253d829045SPawel Laszczak pdev->ep0_stage = CDNSP_STATUS_STAGE; 10263d829045SPawel Laszczak 10273d829045SPawel Laszczak cdnsp_giveback_first_trb(pdev, pep, 0, ep_ring->cycle_state, 10283d829045SPawel Laszczak &td->last_trb->generic); 10293d829045SPawel Laszczak return; 10303d829045SPawel Laszczak } 10313d829045SPawel Laszczak 103299ea221fSPawel Laszczak *status = 0; 103399ea221fSPawel Laszczak 10343d829045SPawel Laszczak cdnsp_finish_td(pdev, td, event, pep, status); 10353d829045SPawel Laszczak } 10363d829045SPawel Laszczak 10373d829045SPawel Laszczak /* 10383d829045SPawel Laszczak * Process isochronous tds, update usb request status and actual_length. 10393d829045SPawel Laszczak */ 10403d829045SPawel Laszczak static void cdnsp_process_isoc_td(struct cdnsp_device *pdev, 10413d829045SPawel Laszczak struct cdnsp_td *td, 10423d829045SPawel Laszczak union cdnsp_trb *ep_trb, 10433d829045SPawel Laszczak struct cdnsp_transfer_event *event, 10443d829045SPawel Laszczak struct cdnsp_ep *pep, 10453d829045SPawel Laszczak int status) 10463d829045SPawel Laszczak { 10473d829045SPawel Laszczak struct cdnsp_request *preq = td->preq; 10483d829045SPawel Laszczak u32 remaining, requested, ep_trb_len; 10493d829045SPawel Laszczak bool sum_trbs_for_length = false; 10503d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 10513d829045SPawel Laszczak u32 trb_comp_code; 10523d829045SPawel Laszczak u32 td_length; 10533d829045SPawel Laszczak 10543d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 10553d829045SPawel Laszczak trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 10563d829045SPawel Laszczak remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 10573d829045SPawel Laszczak ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); 10583d829045SPawel Laszczak 10593d829045SPawel Laszczak requested = preq->request.length; 10603d829045SPawel Laszczak 10613d829045SPawel Laszczak /* handle completion code */ 10623d829045SPawel Laszczak switch (trb_comp_code) { 10633d829045SPawel Laszczak case COMP_SUCCESS: 10643d829045SPawel Laszczak preq->request.status = 0; 10653d829045SPawel Laszczak break; 10663d829045SPawel Laszczak case COMP_SHORT_PACKET: 10673d829045SPawel Laszczak preq->request.status = 0; 10683d829045SPawel Laszczak sum_trbs_for_length = true; 10693d829045SPawel Laszczak break; 10703d829045SPawel Laszczak case COMP_ISOCH_BUFFER_OVERRUN: 10713d829045SPawel Laszczak case COMP_BABBLE_DETECTED_ERROR: 10723d829045SPawel Laszczak preq->request.status = -EOVERFLOW; 10733d829045SPawel Laszczak break; 10743d829045SPawel Laszczak case COMP_STOPPED: 10753d829045SPawel Laszczak sum_trbs_for_length = true; 10763d829045SPawel Laszczak break; 10773d829045SPawel Laszczak case COMP_STOPPED_SHORT_PACKET: 10783d829045SPawel Laszczak /* field normally containing residue now contains transferred */ 10793d829045SPawel Laszczak preq->request.status = 0; 10803d829045SPawel Laszczak requested = remaining; 10813d829045SPawel Laszczak break; 10823d829045SPawel Laszczak case COMP_STOPPED_LENGTH_INVALID: 10833d829045SPawel Laszczak requested = 0; 10843d829045SPawel Laszczak remaining = 0; 10853d829045SPawel Laszczak break; 10863d829045SPawel Laszczak default: 10873d829045SPawel Laszczak sum_trbs_for_length = true; 10883d829045SPawel Laszczak preq->request.status = -1; 10893d829045SPawel Laszczak break; 10903d829045SPawel Laszczak } 10913d829045SPawel Laszczak 10923d829045SPawel Laszczak if (sum_trbs_for_length) { 10933d829045SPawel Laszczak td_length = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb); 10943d829045SPawel Laszczak td_length += ep_trb_len - remaining; 10953d829045SPawel Laszczak } else { 10963d829045SPawel Laszczak td_length = requested; 10973d829045SPawel Laszczak } 10983d829045SPawel Laszczak 10993d829045SPawel Laszczak td->preq->request.actual += td_length; 11003d829045SPawel Laszczak 11013d829045SPawel Laszczak cdnsp_finish_td(pdev, td, event, pep, &status); 11023d829045SPawel Laszczak } 11033d829045SPawel Laszczak 11043d829045SPawel Laszczak static void cdnsp_skip_isoc_td(struct cdnsp_device *pdev, 11053d829045SPawel Laszczak struct cdnsp_td *td, 11063d829045SPawel Laszczak struct cdnsp_transfer_event *event, 11073d829045SPawel Laszczak struct cdnsp_ep *pep, 11083d829045SPawel Laszczak int status) 11093d829045SPawel Laszczak { 11103d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 11113d829045SPawel Laszczak 11123d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 11133d829045SPawel Laszczak td->preq->request.status = -EXDEV; 11143d829045SPawel Laszczak td->preq->request.actual = 0; 11153d829045SPawel Laszczak 11163d829045SPawel Laszczak /* Update ring dequeue pointer */ 11173d829045SPawel Laszczak while (ep_ring->dequeue != td->last_trb) 11183d829045SPawel Laszczak cdnsp_inc_deq(pdev, ep_ring); 11193d829045SPawel Laszczak 11203d829045SPawel Laszczak cdnsp_inc_deq(pdev, ep_ring); 11213d829045SPawel Laszczak 11223d829045SPawel Laszczak cdnsp_td_cleanup(pdev, td, ep_ring, &status); 11233d829045SPawel Laszczak } 11243d829045SPawel Laszczak 11253d829045SPawel Laszczak /* 11263d829045SPawel Laszczak * Process bulk and interrupt tds, update usb request status and actual_length. 11273d829045SPawel Laszczak */ 11283d829045SPawel Laszczak static void cdnsp_process_bulk_intr_td(struct cdnsp_device *pdev, 11293d829045SPawel Laszczak struct cdnsp_td *td, 11303d829045SPawel Laszczak union cdnsp_trb *ep_trb, 11313d829045SPawel Laszczak struct cdnsp_transfer_event *event, 11323d829045SPawel Laszczak struct cdnsp_ep *ep, 11333d829045SPawel Laszczak int *status) 11343d829045SPawel Laszczak { 11353d829045SPawel Laszczak u32 remaining, requested, ep_trb_len; 11363d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 11373d829045SPawel Laszczak u32 trb_comp_code; 11383d829045SPawel Laszczak 11393d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); 11403d829045SPawel Laszczak trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 11413d829045SPawel Laszczak remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 11423d829045SPawel Laszczak ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); 11433d829045SPawel Laszczak requested = td->preq->request.length; 11443d829045SPawel Laszczak 11453d829045SPawel Laszczak switch (trb_comp_code) { 11463d829045SPawel Laszczak case COMP_SUCCESS: 11473d829045SPawel Laszczak case COMP_SHORT_PACKET: 11483d829045SPawel Laszczak *status = 0; 11493d829045SPawel Laszczak break; 11503d829045SPawel Laszczak case COMP_STOPPED_SHORT_PACKET: 11513d829045SPawel Laszczak td->preq->request.actual = remaining; 11523d829045SPawel Laszczak goto finish_td; 11533d829045SPawel Laszczak case COMP_STOPPED_LENGTH_INVALID: 11543d829045SPawel Laszczak /* Stopped on ep trb with invalid length, exclude it. */ 11553d829045SPawel Laszczak ep_trb_len = 0; 11563d829045SPawel Laszczak remaining = 0; 11573d829045SPawel Laszczak break; 11583d829045SPawel Laszczak } 11593d829045SPawel Laszczak 11603d829045SPawel Laszczak if (ep_trb == td->last_trb) 11613d829045SPawel Laszczak ep_trb_len = requested - remaining; 11623d829045SPawel Laszczak else 11633d829045SPawel Laszczak ep_trb_len = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb) + 11643d829045SPawel Laszczak ep_trb_len - remaining; 11653d829045SPawel Laszczak td->preq->request.actual = ep_trb_len; 11663d829045SPawel Laszczak 11673d829045SPawel Laszczak finish_td: 11683d829045SPawel Laszczak ep->stream_info.drbls_count--; 11693d829045SPawel Laszczak 11703d829045SPawel Laszczak cdnsp_finish_td(pdev, td, event, ep, status); 11713d829045SPawel Laszczak } 11723d829045SPawel Laszczak 11733d829045SPawel Laszczak static void cdnsp_handle_tx_nrdy(struct cdnsp_device *pdev, 11743d829045SPawel Laszczak struct cdnsp_transfer_event *event) 11753d829045SPawel Laszczak { 11763d829045SPawel Laszczak struct cdnsp_generic_trb *generic; 11773d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 11783d829045SPawel Laszczak struct cdnsp_ep *pep; 11793d829045SPawel Laszczak int cur_stream; 11803d829045SPawel Laszczak int ep_index; 11813d829045SPawel Laszczak int host_sid; 11823d829045SPawel Laszczak int dev_sid; 11833d829045SPawel Laszczak 11843d829045SPawel Laszczak generic = (struct cdnsp_generic_trb *)event; 11853d829045SPawel Laszczak ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; 11863d829045SPawel Laszczak dev_sid = TRB_TO_DEV_STREAM(le32_to_cpu(generic->field[0])); 11873d829045SPawel Laszczak host_sid = TRB_TO_HOST_STREAM(le32_to_cpu(generic->field[2])); 11883d829045SPawel Laszczak 11893d829045SPawel Laszczak pep = &pdev->eps[ep_index]; 11903d829045SPawel Laszczak 11913d829045SPawel Laszczak if (!(pep->ep_state & EP_HAS_STREAMS)) 11923d829045SPawel Laszczak return; 11933d829045SPawel Laszczak 11943d829045SPawel Laszczak if (host_sid == STREAM_PRIME_ACK) { 11953d829045SPawel Laszczak pep->stream_info.first_prime_det = 1; 11963d829045SPawel Laszczak for (cur_stream = 1; cur_stream < pep->stream_info.num_streams; 11973d829045SPawel Laszczak cur_stream++) { 11983d829045SPawel Laszczak ep_ring = pep->stream_info.stream_rings[cur_stream]; 11993d829045SPawel Laszczak ep_ring->stream_active = 1; 12003d829045SPawel Laszczak ep_ring->stream_rejected = 0; 12013d829045SPawel Laszczak } 12023d829045SPawel Laszczak } 12033d829045SPawel Laszczak 12043d829045SPawel Laszczak if (host_sid == STREAM_REJECTED) { 12053d829045SPawel Laszczak struct cdnsp_td *td, *td_temp; 12063d829045SPawel Laszczak 12073d829045SPawel Laszczak pep->stream_info.drbls_count--; 12083d829045SPawel Laszczak ep_ring = pep->stream_info.stream_rings[dev_sid]; 12093d829045SPawel Laszczak ep_ring->stream_active = 0; 12103d829045SPawel Laszczak ep_ring->stream_rejected = 1; 12113d829045SPawel Laszczak 12123d829045SPawel Laszczak list_for_each_entry_safe(td, td_temp, &ep_ring->td_list, 12133d829045SPawel Laszczak td_list) { 12143d829045SPawel Laszczak td->drbl = 0; 12153d829045SPawel Laszczak } 12163d829045SPawel Laszczak } 12173d829045SPawel Laszczak 12183d829045SPawel Laszczak cdnsp_ring_doorbell_for_active_rings(pdev, pep); 12193d829045SPawel Laszczak } 12203d829045SPawel Laszczak 12213d829045SPawel Laszczak /* 12223d829045SPawel Laszczak * If this function returns an error condition, it means it got a Transfer 12233d829045SPawel Laszczak * event with a corrupted TRB DMA address or endpoint is disabled. 12243d829045SPawel Laszczak */ 12253d829045SPawel Laszczak static int cdnsp_handle_tx_event(struct cdnsp_device *pdev, 12263d829045SPawel Laszczak struct cdnsp_transfer_event *event) 12273d829045SPawel Laszczak { 12283d829045SPawel Laszczak const struct usb_endpoint_descriptor *desc; 12293d829045SPawel Laszczak bool handling_skipped_tds = false; 12303d829045SPawel Laszczak struct cdnsp_segment *ep_seg; 12313d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 12323d829045SPawel Laszczak int status = -EINPROGRESS; 12333d829045SPawel Laszczak union cdnsp_trb *ep_trb; 12343d829045SPawel Laszczak dma_addr_t ep_trb_dma; 12353d829045SPawel Laszczak struct cdnsp_ep *pep; 12363d829045SPawel Laszczak struct cdnsp_td *td; 12373d829045SPawel Laszczak u32 trb_comp_code; 12383d829045SPawel Laszczak int invalidate; 12393d829045SPawel Laszczak int ep_index; 12403d829045SPawel Laszczak 12413d829045SPawel Laszczak invalidate = le32_to_cpu(event->flags) & TRB_EVENT_INVALIDATE; 12423d829045SPawel Laszczak ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; 12433d829045SPawel Laszczak trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 12443d829045SPawel Laszczak ep_trb_dma = le64_to_cpu(event->buffer); 12453d829045SPawel Laszczak 12463d829045SPawel Laszczak pep = &pdev->eps[ep_index]; 12473d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 12483d829045SPawel Laszczak 12493d829045SPawel Laszczak /* 12503d829045SPawel Laszczak * If device is disconnect then all requests will be dequeued 12513d829045SPawel Laszczak * by upper layers as part of disconnect sequence. 12523d829045SPawel Laszczak * We don't want handle such event to avoid racing. 12533d829045SPawel Laszczak */ 12543d829045SPawel Laszczak if (invalidate || !pdev->gadget.connected) 12553d829045SPawel Laszczak goto cleanup; 12563d829045SPawel Laszczak 1257118b2a32SPawel Laszczak if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_DISABLED) { 1258118b2a32SPawel Laszczak trace_cdnsp_ep_disabled(pep->out_ctx); 12593d829045SPawel Laszczak goto err_out; 1260118b2a32SPawel Laszczak } 12613d829045SPawel Laszczak 12623d829045SPawel Laszczak /* Some transfer events don't always point to a trb*/ 12633d829045SPawel Laszczak if (!ep_ring) { 12643d829045SPawel Laszczak switch (trb_comp_code) { 12653d829045SPawel Laszczak case COMP_INVALID_STREAM_TYPE_ERROR: 12663d829045SPawel Laszczak case COMP_INVALID_STREAM_ID_ERROR: 12673d829045SPawel Laszczak case COMP_RING_UNDERRUN: 12683d829045SPawel Laszczak case COMP_RING_OVERRUN: 12693d829045SPawel Laszczak goto cleanup; 12703d829045SPawel Laszczak default: 12713d829045SPawel Laszczak dev_err(pdev->dev, "ERROR: %s event for unknown ring\n", 12723d829045SPawel Laszczak pep->name); 12733d829045SPawel Laszczak goto err_out; 12743d829045SPawel Laszczak } 12753d829045SPawel Laszczak } 12763d829045SPawel Laszczak 12773d829045SPawel Laszczak /* Look for some error cases that need special treatment. */ 12783d829045SPawel Laszczak switch (trb_comp_code) { 12793d829045SPawel Laszczak case COMP_BABBLE_DETECTED_ERROR: 12803d829045SPawel Laszczak status = -EOVERFLOW; 12813d829045SPawel Laszczak break; 12823d829045SPawel Laszczak case COMP_RING_UNDERRUN: 12833d829045SPawel Laszczak case COMP_RING_OVERRUN: 12843d829045SPawel Laszczak /* 12853d829045SPawel Laszczak * When the Isoch ring is empty, the controller will generate 12863d829045SPawel Laszczak * a Ring Overrun Event for IN Isoch endpoint or Ring 12873d829045SPawel Laszczak * Underrun Event for OUT Isoch endpoint. 12883d829045SPawel Laszczak */ 12893d829045SPawel Laszczak goto cleanup; 12903d829045SPawel Laszczak case COMP_MISSED_SERVICE_ERROR: 12913d829045SPawel Laszczak /* 12923d829045SPawel Laszczak * When encounter missed service error, one or more isoc tds 12933d829045SPawel Laszczak * may be missed by controller. 12943d829045SPawel Laszczak * Set skip flag of the ep_ring; Complete the missed tds as 12953d829045SPawel Laszczak * short transfer when process the ep_ring next time. 12963d829045SPawel Laszczak */ 12973d829045SPawel Laszczak pep->skip = true; 12983d829045SPawel Laszczak break; 12993d829045SPawel Laszczak } 13003d829045SPawel Laszczak 13013d829045SPawel Laszczak do { 13023d829045SPawel Laszczak /* 13033d829045SPawel Laszczak * This TRB should be in the TD at the head of this ring's TD 13043d829045SPawel Laszczak * list. 13053d829045SPawel Laszczak */ 13063d829045SPawel Laszczak if (list_empty(&ep_ring->td_list)) { 1307118b2a32SPawel Laszczak /* 1308118b2a32SPawel Laszczak * Don't print warnings if it's due to a stopped 1309118b2a32SPawel Laszczak * endpoint generating an extra completion event, or 1310118b2a32SPawel Laszczak * a event for the last TRB of a short TD we already 1311118b2a32SPawel Laszczak * got a short event for. 1312118b2a32SPawel Laszczak * The short TD is already removed from the TD list. 1313118b2a32SPawel Laszczak */ 1314118b2a32SPawel Laszczak if (!(trb_comp_code == COMP_STOPPED || 1315118b2a32SPawel Laszczak trb_comp_code == COMP_STOPPED_LENGTH_INVALID || 1316118b2a32SPawel Laszczak ep_ring->last_td_was_short)) 1317118b2a32SPawel Laszczak trace_cdnsp_trb_without_td(ep_ring, 1318118b2a32SPawel Laszczak (struct cdnsp_generic_trb *)event); 1319118b2a32SPawel Laszczak 1320118b2a32SPawel Laszczak if (pep->skip) { 13213d829045SPawel Laszczak pep->skip = false; 1322118b2a32SPawel Laszczak trace_cdnsp_ep_list_empty_with_skip(pep, 0); 1323118b2a32SPawel Laszczak } 13243d829045SPawel Laszczak 13253d829045SPawel Laszczak goto cleanup; 13263d829045SPawel Laszczak } 13273d829045SPawel Laszczak 13283d829045SPawel Laszczak td = list_entry(ep_ring->td_list.next, struct cdnsp_td, 13293d829045SPawel Laszczak td_list); 13303d829045SPawel Laszczak 13313d829045SPawel Laszczak /* Is this a TRB in the currently executing TD? */ 13323d829045SPawel Laszczak ep_seg = cdnsp_trb_in_td(pdev, ep_ring->deq_seg, 13333d829045SPawel Laszczak ep_ring->dequeue, td->last_trb, 13343d829045SPawel Laszczak ep_trb_dma); 13353d829045SPawel Laszczak 13363d829045SPawel Laszczak /* 13373d829045SPawel Laszczak * Skip the Force Stopped Event. The event_trb(ep_trb_dma) 13383d829045SPawel Laszczak * of FSE is not in the current TD pointed by ep_ring->dequeue 13393d829045SPawel Laszczak * because that the hardware dequeue pointer still at the 13403d829045SPawel Laszczak * previous TRB of the current TD. The previous TRB maybe a 13413d829045SPawel Laszczak * Link TD or the last TRB of the previous TD. The command 13423d829045SPawel Laszczak * completion handle will take care the rest. 13433d829045SPawel Laszczak */ 13443d829045SPawel Laszczak if (!ep_seg && (trb_comp_code == COMP_STOPPED || 13453d829045SPawel Laszczak trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) { 13463d829045SPawel Laszczak pep->skip = false; 13473d829045SPawel Laszczak goto cleanup; 13483d829045SPawel Laszczak } 13493d829045SPawel Laszczak 13503d829045SPawel Laszczak desc = td->preq->pep->endpoint.desc; 13513d829045SPawel Laszczak if (!ep_seg) { 13523d829045SPawel Laszczak if (!pep->skip || !usb_endpoint_xfer_isoc(desc)) { 13533d829045SPawel Laszczak /* Something is busted, give up! */ 13543d829045SPawel Laszczak dev_err(pdev->dev, 13553d829045SPawel Laszczak "ERROR Transfer event TRB DMA ptr not " 13563d829045SPawel Laszczak "part of current TD ep_index %d " 13573d829045SPawel Laszczak "comp_code %u\n", ep_index, 13583d829045SPawel Laszczak trb_comp_code); 13593d829045SPawel Laszczak return -EINVAL; 13603d829045SPawel Laszczak } 13613d829045SPawel Laszczak 13623d829045SPawel Laszczak cdnsp_skip_isoc_td(pdev, td, event, pep, status); 13633d829045SPawel Laszczak goto cleanup; 13643d829045SPawel Laszczak } 13653d829045SPawel Laszczak 13663d829045SPawel Laszczak if (trb_comp_code == COMP_SHORT_PACKET) 13673d829045SPawel Laszczak ep_ring->last_td_was_short = true; 13683d829045SPawel Laszczak else 13693d829045SPawel Laszczak ep_ring->last_td_was_short = false; 13703d829045SPawel Laszczak 13713d829045SPawel Laszczak if (pep->skip) { 13723d829045SPawel Laszczak pep->skip = false; 13733d829045SPawel Laszczak cdnsp_skip_isoc_td(pdev, td, event, pep, status); 13743d829045SPawel Laszczak goto cleanup; 13753d829045SPawel Laszczak } 13763d829045SPawel Laszczak 13773d829045SPawel Laszczak ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) 13783d829045SPawel Laszczak / sizeof(*ep_trb)]; 13793d829045SPawel Laszczak 1380118b2a32SPawel Laszczak trace_cdnsp_handle_transfer(ep_ring, 1381118b2a32SPawel Laszczak (struct cdnsp_generic_trb *)ep_trb); 1382118b2a32SPawel Laszczak 13833d829045SPawel Laszczak if (cdnsp_trb_is_noop(ep_trb)) 13843d829045SPawel Laszczak goto cleanup; 13853d829045SPawel Laszczak 13863d829045SPawel Laszczak if (usb_endpoint_xfer_control(desc)) 13873d829045SPawel Laszczak cdnsp_process_ctrl_td(pdev, td, ep_trb, event, pep, 13883d829045SPawel Laszczak &status); 13893d829045SPawel Laszczak else if (usb_endpoint_xfer_isoc(desc)) 13903d829045SPawel Laszczak cdnsp_process_isoc_td(pdev, td, ep_trb, event, pep, 13913d829045SPawel Laszczak status); 13923d829045SPawel Laszczak else 13933d829045SPawel Laszczak cdnsp_process_bulk_intr_td(pdev, td, ep_trb, event, pep, 13943d829045SPawel Laszczak &status); 13953d829045SPawel Laszczak cleanup: 13963d829045SPawel Laszczak handling_skipped_tds = pep->skip; 13973d829045SPawel Laszczak 13983d829045SPawel Laszczak /* 13993d829045SPawel Laszczak * Do not update event ring dequeue pointer if we're in a loop 14003d829045SPawel Laszczak * processing missed tds. 14013d829045SPawel Laszczak */ 14023d829045SPawel Laszczak if (!handling_skipped_tds) 14033d829045SPawel Laszczak cdnsp_inc_deq(pdev, pdev->event_ring); 14043d829045SPawel Laszczak 14053d829045SPawel Laszczak /* 14063d829045SPawel Laszczak * If ep->skip is set, it means there are missed tds on the 14073d829045SPawel Laszczak * endpoint ring need to take care of. 14083d829045SPawel Laszczak * Process them as short transfer until reach the td pointed by 14093d829045SPawel Laszczak * the event. 14103d829045SPawel Laszczak */ 14113d829045SPawel Laszczak } while (handling_skipped_tds); 14123d829045SPawel Laszczak return 0; 14133d829045SPawel Laszczak 14143d829045SPawel Laszczak err_out: 14153d829045SPawel Laszczak dev_err(pdev->dev, "@%016llx %08x %08x %08x %08x\n", 14163d829045SPawel Laszczak (unsigned long long) 14173d829045SPawel Laszczak cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg, 14183d829045SPawel Laszczak pdev->event_ring->dequeue), 14193d829045SPawel Laszczak lower_32_bits(le64_to_cpu(event->buffer)), 14203d829045SPawel Laszczak upper_32_bits(le64_to_cpu(event->buffer)), 14213d829045SPawel Laszczak le32_to_cpu(event->transfer_len), 14223d829045SPawel Laszczak le32_to_cpu(event->flags)); 14233d829045SPawel Laszczak return -EINVAL; 14243d829045SPawel Laszczak } 14253d829045SPawel Laszczak 14263d829045SPawel Laszczak /* 14273d829045SPawel Laszczak * This function handles all events on the event ring. 14283d829045SPawel Laszczak * Returns true for "possibly more events to process" (caller should call 14293d829045SPawel Laszczak * again), otherwise false if done. 14303d829045SPawel Laszczak */ 14313d829045SPawel Laszczak static bool cdnsp_handle_event(struct cdnsp_device *pdev) 14323d829045SPawel Laszczak { 14333d829045SPawel Laszczak unsigned int comp_code; 14343d829045SPawel Laszczak union cdnsp_trb *event; 14353d829045SPawel Laszczak bool update_ptrs = true; 143616e36101SPawel Laszczak u32 cycle_bit; 14373d829045SPawel Laszczak int ret = 0; 14383d829045SPawel Laszczak u32 flags; 14393d829045SPawel Laszczak 14403d829045SPawel Laszczak event = pdev->event_ring->dequeue; 14413d829045SPawel Laszczak flags = le32_to_cpu(event->event_cmd.flags); 14423d829045SPawel Laszczak cycle_bit = (flags & TRB_CYCLE); 14433d829045SPawel Laszczak 14443d829045SPawel Laszczak /* Does the controller or driver own the TRB? */ 14453d829045SPawel Laszczak if (cycle_bit != pdev->event_ring->cycle_state) 14463d829045SPawel Laszczak return false; 14473d829045SPawel Laszczak 1448118b2a32SPawel Laszczak trace_cdnsp_handle_event(pdev->event_ring, &event->generic); 1449118b2a32SPawel Laszczak 14503d829045SPawel Laszczak /* 14513d829045SPawel Laszczak * Barrier between reading the TRB_CYCLE (valid) flag above and any 14523d829045SPawel Laszczak * reads of the event's flags/data below. 14533d829045SPawel Laszczak */ 14543d829045SPawel Laszczak rmb(); 14553d829045SPawel Laszczak 14563d829045SPawel Laszczak switch (flags & TRB_TYPE_BITMASK) { 14573d829045SPawel Laszczak case TRB_TYPE(TRB_COMPLETION): 14583d829045SPawel Laszczak /* 14593d829045SPawel Laszczak * Command can't be handled in interrupt context so just 14603d829045SPawel Laszczak * increment command ring dequeue pointer. 14613d829045SPawel Laszczak */ 14623d829045SPawel Laszczak cdnsp_inc_deq(pdev, pdev->cmd_ring); 14633d829045SPawel Laszczak break; 14643d829045SPawel Laszczak case TRB_TYPE(TRB_PORT_STATUS): 14653d829045SPawel Laszczak cdnsp_handle_port_status(pdev, event); 14663d829045SPawel Laszczak update_ptrs = false; 14673d829045SPawel Laszczak break; 14683d829045SPawel Laszczak case TRB_TYPE(TRB_TRANSFER): 14693d829045SPawel Laszczak ret = cdnsp_handle_tx_event(pdev, &event->trans_event); 14703d829045SPawel Laszczak if (ret >= 0) 14713d829045SPawel Laszczak update_ptrs = false; 14723d829045SPawel Laszczak break; 14733d829045SPawel Laszczak case TRB_TYPE(TRB_SETUP): 14743d829045SPawel Laszczak pdev->ep0_stage = CDNSP_SETUP_STAGE; 14753d829045SPawel Laszczak pdev->setup_id = TRB_SETUPID_TO_TYPE(flags); 14763d829045SPawel Laszczak pdev->setup_speed = TRB_SETUP_SPEEDID(flags); 14773d829045SPawel Laszczak pdev->setup = *((struct usb_ctrlrequest *) 14783d829045SPawel Laszczak &event->trans_event.buffer); 14793d829045SPawel Laszczak 14803d829045SPawel Laszczak cdnsp_setup_analyze(pdev); 14813d829045SPawel Laszczak break; 14823d829045SPawel Laszczak case TRB_TYPE(TRB_ENDPOINT_NRDY): 14833d829045SPawel Laszczak cdnsp_handle_tx_nrdy(pdev, &event->trans_event); 14843d829045SPawel Laszczak break; 14853d829045SPawel Laszczak case TRB_TYPE(TRB_HC_EVENT): { 14863d829045SPawel Laszczak comp_code = GET_COMP_CODE(le32_to_cpu(event->generic.field[2])); 14873d829045SPawel Laszczak 14883d829045SPawel Laszczak switch (comp_code) { 14893d829045SPawel Laszczak case COMP_EVENT_RING_FULL_ERROR: 14903d829045SPawel Laszczak dev_err(pdev->dev, "Event Ring Full\n"); 14913d829045SPawel Laszczak break; 14923d829045SPawel Laszczak default: 14933d829045SPawel Laszczak dev_err(pdev->dev, "Controller error code 0x%02x\n", 14943d829045SPawel Laszczak comp_code); 14953d829045SPawel Laszczak } 14963d829045SPawel Laszczak 14973d829045SPawel Laszczak break; 14983d829045SPawel Laszczak } 14993d829045SPawel Laszczak case TRB_TYPE(TRB_MFINDEX_WRAP): 15003d829045SPawel Laszczak case TRB_TYPE(TRB_DRB_OVERFLOW): 15013d829045SPawel Laszczak break; 15023d829045SPawel Laszczak default: 15033d829045SPawel Laszczak dev_warn(pdev->dev, "ERROR unknown event type %ld\n", 15043d829045SPawel Laszczak TRB_FIELD_TO_TYPE(flags)); 15053d829045SPawel Laszczak } 15063d829045SPawel Laszczak 15073d829045SPawel Laszczak if (update_ptrs) 15083d829045SPawel Laszczak /* Update SW event ring dequeue pointer. */ 15093d829045SPawel Laszczak cdnsp_inc_deq(pdev, pdev->event_ring); 15103d829045SPawel Laszczak 15113d829045SPawel Laszczak /* 15123d829045SPawel Laszczak * Caller will call us again to check if there are more items 15133d829045SPawel Laszczak * on the event ring. 15143d829045SPawel Laszczak */ 15153d829045SPawel Laszczak return true; 15163d829045SPawel Laszczak } 15173d829045SPawel Laszczak 15183d829045SPawel Laszczak irqreturn_t cdnsp_thread_irq_handler(int irq, void *data) 15193d829045SPawel Laszczak { 15203d829045SPawel Laszczak struct cdnsp_device *pdev = (struct cdnsp_device *)data; 15213d829045SPawel Laszczak union cdnsp_trb *event_ring_deq; 1522a9aecef1SPawel Laszczak unsigned long flags; 15233d829045SPawel Laszczak int counter = 0; 15243d829045SPawel Laszczak 1525a9aecef1SPawel Laszczak spin_lock_irqsave(&pdev->lock, flags); 15263d829045SPawel Laszczak 15273d829045SPawel Laszczak if (pdev->cdnsp_state & (CDNSP_STATE_HALTED | CDNSP_STATE_DYING)) { 152816f00d96SPawel Laszczak /* 152916f00d96SPawel Laszczak * While removing or stopping driver there may still be deferred 153016f00d96SPawel Laszczak * not handled interrupt which should not be treated as error. 153116f00d96SPawel Laszczak * Driver should simply ignore it. 153216f00d96SPawel Laszczak */ 153316f00d96SPawel Laszczak if (pdev->gadget_driver) 15343d829045SPawel Laszczak cdnsp_died(pdev); 153516f00d96SPawel Laszczak 1536a9aecef1SPawel Laszczak spin_unlock_irqrestore(&pdev->lock, flags); 15373d829045SPawel Laszczak return IRQ_HANDLED; 15383d829045SPawel Laszczak } 15393d829045SPawel Laszczak 15403d829045SPawel Laszczak event_ring_deq = pdev->event_ring->dequeue; 15413d829045SPawel Laszczak 15423d829045SPawel Laszczak while (cdnsp_handle_event(pdev)) { 15433d829045SPawel Laszczak if (++counter >= TRBS_PER_EV_DEQ_UPDATE) { 15443d829045SPawel Laszczak cdnsp_update_erst_dequeue(pdev, event_ring_deq, 0); 15453d829045SPawel Laszczak event_ring_deq = pdev->event_ring->dequeue; 15463d829045SPawel Laszczak counter = 0; 15473d829045SPawel Laszczak } 15483d829045SPawel Laszczak } 15493d829045SPawel Laszczak 15503d829045SPawel Laszczak cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1); 15513d829045SPawel Laszczak 1552a9aecef1SPawel Laszczak spin_unlock_irqrestore(&pdev->lock, flags); 15533d829045SPawel Laszczak 15543d829045SPawel Laszczak return IRQ_HANDLED; 15553d829045SPawel Laszczak } 15563d829045SPawel Laszczak 15573d829045SPawel Laszczak irqreturn_t cdnsp_irq_handler(int irq, void *priv) 15583d829045SPawel Laszczak { 15593d829045SPawel Laszczak struct cdnsp_device *pdev = (struct cdnsp_device *)priv; 15603d829045SPawel Laszczak u32 irq_pending; 15613d829045SPawel Laszczak u32 status; 15623d829045SPawel Laszczak 15633d829045SPawel Laszczak status = readl(&pdev->op_regs->status); 15643d829045SPawel Laszczak 15653d829045SPawel Laszczak if (status == ~(u32)0) { 15663d829045SPawel Laszczak cdnsp_died(pdev); 15673d829045SPawel Laszczak return IRQ_HANDLED; 15683d829045SPawel Laszczak } 15693d829045SPawel Laszczak 15703d829045SPawel Laszczak if (!(status & STS_EINT)) 15713d829045SPawel Laszczak return IRQ_NONE; 15723d829045SPawel Laszczak 15733d829045SPawel Laszczak writel(status | STS_EINT, &pdev->op_regs->status); 15743d829045SPawel Laszczak irq_pending = readl(&pdev->ir_set->irq_pending); 15753d829045SPawel Laszczak irq_pending |= IMAN_IP; 15763d829045SPawel Laszczak writel(irq_pending, &pdev->ir_set->irq_pending); 15773d829045SPawel Laszczak 15783d829045SPawel Laszczak if (status & STS_FATAL) { 15793d829045SPawel Laszczak cdnsp_died(pdev); 15803d829045SPawel Laszczak return IRQ_HANDLED; 15813d829045SPawel Laszczak } 15823d829045SPawel Laszczak 15833d829045SPawel Laszczak return IRQ_WAKE_THREAD; 15843d829045SPawel Laszczak } 15853d829045SPawel Laszczak 15863d829045SPawel Laszczak /* 15873d829045SPawel Laszczak * Generic function for queuing a TRB on a ring. 15883d829045SPawel Laszczak * The caller must have checked to make sure there's room on the ring. 15893d829045SPawel Laszczak * 15903d829045SPawel Laszczak * @more_trbs_coming: Will you enqueue more TRBs before setting doorbell? 15913d829045SPawel Laszczak */ 15923d829045SPawel Laszczak static void cdnsp_queue_trb(struct cdnsp_device *pdev, struct cdnsp_ring *ring, 15933d829045SPawel Laszczak bool more_trbs_coming, u32 field1, u32 field2, 15943d829045SPawel Laszczak u32 field3, u32 field4) 15953d829045SPawel Laszczak { 15963d829045SPawel Laszczak struct cdnsp_generic_trb *trb; 15973d829045SPawel Laszczak 15983d829045SPawel Laszczak trb = &ring->enqueue->generic; 15993d829045SPawel Laszczak 16003d829045SPawel Laszczak trb->field[0] = cpu_to_le32(field1); 16013d829045SPawel Laszczak trb->field[1] = cpu_to_le32(field2); 16023d829045SPawel Laszczak trb->field[2] = cpu_to_le32(field3); 16033d829045SPawel Laszczak trb->field[3] = cpu_to_le32(field4); 16043d829045SPawel Laszczak 1605118b2a32SPawel Laszczak trace_cdnsp_queue_trb(ring, trb); 16063d829045SPawel Laszczak cdnsp_inc_enq(pdev, ring, more_trbs_coming); 16073d829045SPawel Laszczak } 16083d829045SPawel Laszczak 16093d829045SPawel Laszczak /* 16103d829045SPawel Laszczak * Does various checks on the endpoint ring, and makes it ready to 16113d829045SPawel Laszczak * queue num_trbs. 16123d829045SPawel Laszczak */ 16133d829045SPawel Laszczak static int cdnsp_prepare_ring(struct cdnsp_device *pdev, 16143d829045SPawel Laszczak struct cdnsp_ring *ep_ring, 16153d829045SPawel Laszczak u32 ep_state, unsigned 16163d829045SPawel Laszczak int num_trbs, 16173d829045SPawel Laszczak gfp_t mem_flags) 16183d829045SPawel Laszczak { 16193d829045SPawel Laszczak unsigned int num_trbs_needed; 16203d829045SPawel Laszczak 16213d829045SPawel Laszczak /* Make sure the endpoint has been added to controller schedule. */ 16223d829045SPawel Laszczak switch (ep_state) { 16233d829045SPawel Laszczak case EP_STATE_STOPPED: 16243d829045SPawel Laszczak case EP_STATE_RUNNING: 16253d829045SPawel Laszczak case EP_STATE_HALTED: 16263d829045SPawel Laszczak break; 16273d829045SPawel Laszczak default: 16283d829045SPawel Laszczak dev_err(pdev->dev, "ERROR: incorrect endpoint state\n"); 16293d829045SPawel Laszczak return -EINVAL; 16303d829045SPawel Laszczak } 16313d829045SPawel Laszczak 16323d829045SPawel Laszczak while (1) { 16333d829045SPawel Laszczak if (cdnsp_room_on_ring(pdev, ep_ring, num_trbs)) 16343d829045SPawel Laszczak break; 16353d829045SPawel Laszczak 1636118b2a32SPawel Laszczak trace_cdnsp_no_room_on_ring("try ring expansion"); 1637118b2a32SPawel Laszczak 16383d829045SPawel Laszczak num_trbs_needed = num_trbs - ep_ring->num_trbs_free; 16393d829045SPawel Laszczak if (cdnsp_ring_expansion(pdev, ep_ring, num_trbs_needed, 16403d829045SPawel Laszczak mem_flags)) { 16413d829045SPawel Laszczak dev_err(pdev->dev, "Ring expansion failed\n"); 16423d829045SPawel Laszczak return -ENOMEM; 16433d829045SPawel Laszczak } 16443d829045SPawel Laszczak } 16453d829045SPawel Laszczak 16463d829045SPawel Laszczak while (cdnsp_trb_is_link(ep_ring->enqueue)) { 16473d829045SPawel Laszczak ep_ring->enqueue->link.control |= cpu_to_le32(TRB_CHAIN); 16483d829045SPawel Laszczak /* The cycle bit must be set as the last operation. */ 16493d829045SPawel Laszczak wmb(); 16503d829045SPawel Laszczak ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE); 16513d829045SPawel Laszczak 16523d829045SPawel Laszczak /* Toggle the cycle bit after the last ring segment. */ 16533d829045SPawel Laszczak if (cdnsp_link_trb_toggles_cycle(ep_ring->enqueue)) 16543d829045SPawel Laszczak ep_ring->cycle_state ^= 1; 16553d829045SPawel Laszczak ep_ring->enq_seg = ep_ring->enq_seg->next; 16563d829045SPawel Laszczak ep_ring->enqueue = ep_ring->enq_seg->trbs; 16573d829045SPawel Laszczak } 16583d829045SPawel Laszczak return 0; 16593d829045SPawel Laszczak } 16603d829045SPawel Laszczak 16613d829045SPawel Laszczak static int cdnsp_prepare_transfer(struct cdnsp_device *pdev, 16623d829045SPawel Laszczak struct cdnsp_request *preq, 16633d829045SPawel Laszczak unsigned int num_trbs) 16643d829045SPawel Laszczak { 16653d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 16663d829045SPawel Laszczak int ret; 16673d829045SPawel Laszczak 16683d829045SPawel Laszczak ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep, 16693d829045SPawel Laszczak preq->request.stream_id); 16703d829045SPawel Laszczak if (!ep_ring) 16713d829045SPawel Laszczak return -EINVAL; 16723d829045SPawel Laszczak 16733d829045SPawel Laszczak ret = cdnsp_prepare_ring(pdev, ep_ring, 16743d829045SPawel Laszczak GET_EP_CTX_STATE(preq->pep->out_ctx), 16753d829045SPawel Laszczak num_trbs, GFP_ATOMIC); 16763d829045SPawel Laszczak if (ret) 16773d829045SPawel Laszczak return ret; 16783d829045SPawel Laszczak 16793d829045SPawel Laszczak INIT_LIST_HEAD(&preq->td.td_list); 16803d829045SPawel Laszczak preq->td.preq = preq; 16813d829045SPawel Laszczak 16823d829045SPawel Laszczak /* Add this TD to the tail of the endpoint ring's TD list. */ 16833d829045SPawel Laszczak list_add_tail(&preq->td.td_list, &ep_ring->td_list); 16843d829045SPawel Laszczak ep_ring->num_tds++; 16853d829045SPawel Laszczak preq->pep->stream_info.td_count++; 16863d829045SPawel Laszczak 16873d829045SPawel Laszczak preq->td.start_seg = ep_ring->enq_seg; 16883d829045SPawel Laszczak preq->td.first_trb = ep_ring->enqueue; 16893d829045SPawel Laszczak 16903d829045SPawel Laszczak return 0; 16913d829045SPawel Laszczak } 16923d829045SPawel Laszczak 16933d829045SPawel Laszczak static unsigned int cdnsp_count_trbs(u64 addr, u64 len) 16943d829045SPawel Laszczak { 16953d829045SPawel Laszczak unsigned int num_trbs; 16963d829045SPawel Laszczak 16973d829045SPawel Laszczak num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)), 16983d829045SPawel Laszczak TRB_MAX_BUFF_SIZE); 16993d829045SPawel Laszczak if (num_trbs == 0) 17003d829045SPawel Laszczak num_trbs++; 17013d829045SPawel Laszczak 17023d829045SPawel Laszczak return num_trbs; 17033d829045SPawel Laszczak } 17043d829045SPawel Laszczak 17053d829045SPawel Laszczak static unsigned int count_trbs_needed(struct cdnsp_request *preq) 17063d829045SPawel Laszczak { 17073d829045SPawel Laszczak return cdnsp_count_trbs(preq->request.dma, preq->request.length); 17083d829045SPawel Laszczak } 17093d829045SPawel Laszczak 17103d829045SPawel Laszczak static unsigned int count_sg_trbs_needed(struct cdnsp_request *preq) 17113d829045SPawel Laszczak { 17123d829045SPawel Laszczak unsigned int i, len, full_len, num_trbs = 0; 17133d829045SPawel Laszczak struct scatterlist *sg; 17143d829045SPawel Laszczak 17153d829045SPawel Laszczak full_len = preq->request.length; 17163d829045SPawel Laszczak 17173d829045SPawel Laszczak for_each_sg(preq->request.sg, sg, preq->request.num_sgs, i) { 17183d829045SPawel Laszczak len = sg_dma_len(sg); 17193d829045SPawel Laszczak num_trbs += cdnsp_count_trbs(sg_dma_address(sg), len); 17203d829045SPawel Laszczak len = min(len, full_len); 17213d829045SPawel Laszczak full_len -= len; 17223d829045SPawel Laszczak if (full_len == 0) 17233d829045SPawel Laszczak break; 17243d829045SPawel Laszczak } 17253d829045SPawel Laszczak 17263d829045SPawel Laszczak return num_trbs; 17273d829045SPawel Laszczak } 17283d829045SPawel Laszczak 17293d829045SPawel Laszczak static unsigned int count_isoc_trbs_needed(struct cdnsp_request *preq) 17303d829045SPawel Laszczak { 17313d829045SPawel Laszczak return cdnsp_count_trbs(preq->request.dma, preq->request.length); 17323d829045SPawel Laszczak } 17333d829045SPawel Laszczak 17343d829045SPawel Laszczak static void cdnsp_check_trb_math(struct cdnsp_request *preq, int running_total) 17353d829045SPawel Laszczak { 17363d829045SPawel Laszczak if (running_total != preq->request.length) 17373d829045SPawel Laszczak dev_err(preq->pep->pdev->dev, 17383d829045SPawel Laszczak "%s - Miscalculated tx length, " 17393d829045SPawel Laszczak "queued %#x, asked for %#x (%d)\n", 17403d829045SPawel Laszczak preq->pep->name, running_total, 17413d829045SPawel Laszczak preq->request.length, preq->request.actual); 17423d829045SPawel Laszczak } 17433d829045SPawel Laszczak 17443d829045SPawel Laszczak /* 17453d829045SPawel Laszczak * TD size is the number of max packet sized packets remaining in the TD 17463d829045SPawel Laszczak * (*not* including this TRB). 17473d829045SPawel Laszczak * 17483d829045SPawel Laszczak * Total TD packet count = total_packet_count = 17493d829045SPawel Laszczak * DIV_ROUND_UP(TD size in bytes / wMaxPacketSize) 17503d829045SPawel Laszczak * 17513d829045SPawel Laszczak * Packets transferred up to and including this TRB = packets_transferred = 17523d829045SPawel Laszczak * rounddown(total bytes transferred including this TRB / wMaxPacketSize) 17533d829045SPawel Laszczak * 17543d829045SPawel Laszczak * TD size = total_packet_count - packets_transferred 17553d829045SPawel Laszczak * 17563d829045SPawel Laszczak * It must fit in bits 21:17, so it can't be bigger than 31. 17573d829045SPawel Laszczak * This is taken care of in the TRB_TD_SIZE() macro 17583d829045SPawel Laszczak * 17593d829045SPawel Laszczak * The last TRB in a TD must have the TD size set to zero. 17603d829045SPawel Laszczak */ 17613d829045SPawel Laszczak static u32 cdnsp_td_remainder(struct cdnsp_device *pdev, 17623d829045SPawel Laszczak int transferred, 17633d829045SPawel Laszczak int trb_buff_len, 17643d829045SPawel Laszczak unsigned int td_total_len, 17653d829045SPawel Laszczak struct cdnsp_request *preq, 17667a21b27aSPawel Laszczak bool more_trbs_coming, 17677a21b27aSPawel Laszczak bool zlp) 17683d829045SPawel Laszczak { 17693d829045SPawel Laszczak u32 maxp, total_packet_count; 17703d829045SPawel Laszczak 17717a21b27aSPawel Laszczak /* Before ZLP driver needs set TD_SIZE = 1. */ 17727a21b27aSPawel Laszczak if (zlp) 17737a21b27aSPawel Laszczak return 1; 17747a21b27aSPawel Laszczak 17753d829045SPawel Laszczak /* One TRB with a zero-length data packet. */ 17763d829045SPawel Laszczak if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) || 17773d829045SPawel Laszczak trb_buff_len == td_total_len) 17783d829045SPawel Laszczak return 0; 17793d829045SPawel Laszczak 17803d829045SPawel Laszczak maxp = usb_endpoint_maxp(preq->pep->endpoint.desc); 17813d829045SPawel Laszczak total_packet_count = DIV_ROUND_UP(td_total_len, maxp); 17823d829045SPawel Laszczak 17833d829045SPawel Laszczak /* Queuing functions don't count the current TRB into transferred. */ 17843d829045SPawel Laszczak return (total_packet_count - ((transferred + trb_buff_len) / maxp)); 17853d829045SPawel Laszczak } 17863d829045SPawel Laszczak 17873d829045SPawel Laszczak static int cdnsp_align_td(struct cdnsp_device *pdev, 17883d829045SPawel Laszczak struct cdnsp_request *preq, u32 enqd_len, 17893d829045SPawel Laszczak u32 *trb_buff_len, struct cdnsp_segment *seg) 17903d829045SPawel Laszczak { 17913d829045SPawel Laszczak struct device *dev = pdev->dev; 17923d829045SPawel Laszczak unsigned int unalign; 17933d829045SPawel Laszczak unsigned int max_pkt; 17943d829045SPawel Laszczak u32 new_buff_len; 17953d829045SPawel Laszczak 17963d829045SPawel Laszczak max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc); 17973d829045SPawel Laszczak unalign = (enqd_len + *trb_buff_len) % max_pkt; 17983d829045SPawel Laszczak 17993d829045SPawel Laszczak /* We got lucky, last normal TRB data on segment is packet aligned. */ 18003d829045SPawel Laszczak if (unalign == 0) 18013d829045SPawel Laszczak return 0; 18023d829045SPawel Laszczak 18033d829045SPawel Laszczak /* Is the last nornal TRB alignable by splitting it. */ 18043d829045SPawel Laszczak if (*trb_buff_len > unalign) { 18053d829045SPawel Laszczak *trb_buff_len -= unalign; 1806118b2a32SPawel Laszczak trace_cdnsp_bounce_align_td_split(preq, *trb_buff_len, 1807118b2a32SPawel Laszczak enqd_len, 0, unalign); 18083d829045SPawel Laszczak return 0; 18093d829045SPawel Laszczak } 18103d829045SPawel Laszczak 18113d829045SPawel Laszczak /* 18123d829045SPawel Laszczak * We want enqd_len + trb_buff_len to sum up to a number aligned to 18133d829045SPawel Laszczak * number which is divisible by the endpoint's wMaxPacketSize. IOW: 18143d829045SPawel Laszczak * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0. 18153d829045SPawel Laszczak */ 18163d829045SPawel Laszczak new_buff_len = max_pkt - (enqd_len % max_pkt); 18173d829045SPawel Laszczak 18183d829045SPawel Laszczak if (new_buff_len > (preq->request.length - enqd_len)) 18193d829045SPawel Laszczak new_buff_len = (preq->request.length - enqd_len); 18203d829045SPawel Laszczak 18213d829045SPawel Laszczak /* Create a max max_pkt sized bounce buffer pointed to by last trb. */ 18223d829045SPawel Laszczak if (preq->direction) { 18233d829045SPawel Laszczak sg_pcopy_to_buffer(preq->request.sg, 18243d829045SPawel Laszczak preq->request.num_mapped_sgs, 18253d829045SPawel Laszczak seg->bounce_buf, new_buff_len, enqd_len); 18263d829045SPawel Laszczak seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, 18273d829045SPawel Laszczak max_pkt, DMA_TO_DEVICE); 18283d829045SPawel Laszczak } else { 18293d829045SPawel Laszczak seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, 18303d829045SPawel Laszczak max_pkt, DMA_FROM_DEVICE); 18313d829045SPawel Laszczak } 18323d829045SPawel Laszczak 18333d829045SPawel Laszczak if (dma_mapping_error(dev, seg->bounce_dma)) { 18343d829045SPawel Laszczak /* Try without aligning.*/ 18353d829045SPawel Laszczak dev_warn(pdev->dev, 18363d829045SPawel Laszczak "Failed mapping bounce buffer, not aligning\n"); 18373d829045SPawel Laszczak return 0; 18383d829045SPawel Laszczak } 18393d829045SPawel Laszczak 18403d829045SPawel Laszczak *trb_buff_len = new_buff_len; 18413d829045SPawel Laszczak seg->bounce_len = new_buff_len; 18423d829045SPawel Laszczak seg->bounce_offs = enqd_len; 18433d829045SPawel Laszczak 1844118b2a32SPawel Laszczak trace_cdnsp_bounce_map(preq, new_buff_len, enqd_len, seg->bounce_dma, 1845118b2a32SPawel Laszczak unalign); 1846118b2a32SPawel Laszczak 18473d829045SPawel Laszczak /* 18483d829045SPawel Laszczak * Bounce buffer successful aligned and seg->bounce_dma will be used 18493d829045SPawel Laszczak * in transfer TRB as new transfer buffer address. 18503d829045SPawel Laszczak */ 18513d829045SPawel Laszczak return 1; 18523d829045SPawel Laszczak } 18533d829045SPawel Laszczak 18543d829045SPawel Laszczak int cdnsp_queue_bulk_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq) 18553d829045SPawel Laszczak { 18563d829045SPawel Laszczak unsigned int enqd_len, block_len, trb_buff_len, full_len; 18573d829045SPawel Laszczak unsigned int start_cycle, num_sgs = 0; 18583d829045SPawel Laszczak struct cdnsp_generic_trb *start_trb; 18593d829045SPawel Laszczak u32 field, length_field, remainder; 18603d829045SPawel Laszczak struct scatterlist *sg = NULL; 18613d829045SPawel Laszczak bool more_trbs_coming = true; 18623d829045SPawel Laszczak bool need_zero_pkt = false; 18633d829045SPawel Laszczak bool zero_len_trb = false; 18643d829045SPawel Laszczak struct cdnsp_ring *ring; 18653d829045SPawel Laszczak bool first_trb = true; 18663d829045SPawel Laszczak unsigned int num_trbs; 18673d829045SPawel Laszczak struct cdnsp_ep *pep; 18683d829045SPawel Laszczak u64 addr, send_addr; 18693d829045SPawel Laszczak int sent_len, ret; 18703d829045SPawel Laszczak 18713d829045SPawel Laszczak ring = cdnsp_request_to_transfer_ring(pdev, preq); 18723d829045SPawel Laszczak if (!ring) 18733d829045SPawel Laszczak return -EINVAL; 18743d829045SPawel Laszczak 18753d829045SPawel Laszczak full_len = preq->request.length; 18763d829045SPawel Laszczak 18773d829045SPawel Laszczak if (preq->request.num_sgs) { 18783d829045SPawel Laszczak num_sgs = preq->request.num_sgs; 18793d829045SPawel Laszczak sg = preq->request.sg; 18803d829045SPawel Laszczak addr = (u64)sg_dma_address(sg); 18813d829045SPawel Laszczak block_len = sg_dma_len(sg); 18823d829045SPawel Laszczak num_trbs = count_sg_trbs_needed(preq); 18833d829045SPawel Laszczak } else { 18843d829045SPawel Laszczak num_trbs = count_trbs_needed(preq); 18853d829045SPawel Laszczak addr = (u64)preq->request.dma; 18863d829045SPawel Laszczak block_len = full_len; 18873d829045SPawel Laszczak } 18883d829045SPawel Laszczak 18893d829045SPawel Laszczak pep = preq->pep; 18903d829045SPawel Laszczak 18913d829045SPawel Laszczak /* Deal with request.zero - need one more td/trb. */ 18923d829045SPawel Laszczak if (preq->request.zero && preq->request.length && 18933d829045SPawel Laszczak IS_ALIGNED(full_len, usb_endpoint_maxp(pep->endpoint.desc))) { 18943d829045SPawel Laszczak need_zero_pkt = true; 18953d829045SPawel Laszczak num_trbs++; 18963d829045SPawel Laszczak } 18973d829045SPawel Laszczak 18983d829045SPawel Laszczak ret = cdnsp_prepare_transfer(pdev, preq, num_trbs); 18993d829045SPawel Laszczak if (ret) 19003d829045SPawel Laszczak return ret; 19013d829045SPawel Laszczak 19023d829045SPawel Laszczak /* 19033d829045SPawel Laszczak * Don't give the first TRB to the hardware (by toggling the cycle bit) 19043d829045SPawel Laszczak * until we've finished creating all the other TRBs. The ring's cycle 19053d829045SPawel Laszczak * state may change as we enqueue the other TRBs, so save it too. 19063d829045SPawel Laszczak */ 19073d829045SPawel Laszczak start_trb = &ring->enqueue->generic; 19083d829045SPawel Laszczak start_cycle = ring->cycle_state; 19093d829045SPawel Laszczak send_addr = addr; 19103d829045SPawel Laszczak 19113d829045SPawel Laszczak /* Queue the TRBs, even if they are zero-length */ 19123d829045SPawel Laszczak for (enqd_len = 0; zero_len_trb || first_trb || enqd_len < full_len; 19133d829045SPawel Laszczak enqd_len += trb_buff_len) { 19143d829045SPawel Laszczak field = TRB_TYPE(TRB_NORMAL); 19153d829045SPawel Laszczak 19163d829045SPawel Laszczak /* TRB buffer should not cross 64KB boundaries */ 19173d829045SPawel Laszczak trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr); 19183d829045SPawel Laszczak trb_buff_len = min(trb_buff_len, block_len); 19193d829045SPawel Laszczak if (enqd_len + trb_buff_len > full_len) 19203d829045SPawel Laszczak trb_buff_len = full_len - enqd_len; 19213d829045SPawel Laszczak 19223d829045SPawel Laszczak /* Don't change the cycle bit of the first TRB until later */ 19233d829045SPawel Laszczak if (first_trb) { 19243d829045SPawel Laszczak first_trb = false; 19253d829045SPawel Laszczak if (start_cycle == 0) 19263d829045SPawel Laszczak field |= TRB_CYCLE; 19273d829045SPawel Laszczak } else { 19283d829045SPawel Laszczak field |= ring->cycle_state; 19293d829045SPawel Laszczak } 19303d829045SPawel Laszczak 19313d829045SPawel Laszczak /* 19323d829045SPawel Laszczak * Chain all the TRBs together; clear the chain bit in the last 19333d829045SPawel Laszczak * TRB to indicate it's the last TRB in the chain. 19343d829045SPawel Laszczak */ 19353d829045SPawel Laszczak if (enqd_len + trb_buff_len < full_len || need_zero_pkt) { 19363d829045SPawel Laszczak field |= TRB_CHAIN; 19373d829045SPawel Laszczak if (cdnsp_trb_is_link(ring->enqueue + 1)) { 19383d829045SPawel Laszczak if (cdnsp_align_td(pdev, preq, enqd_len, 19393d829045SPawel Laszczak &trb_buff_len, 19403d829045SPawel Laszczak ring->enq_seg)) { 19413d829045SPawel Laszczak send_addr = ring->enq_seg->bounce_dma; 19423d829045SPawel Laszczak /* Assuming TD won't span 2 segs */ 19433d829045SPawel Laszczak preq->td.bounce_seg = ring->enq_seg; 19443d829045SPawel Laszczak } 19453d829045SPawel Laszczak } 19463d829045SPawel Laszczak } 19473d829045SPawel Laszczak 19483d829045SPawel Laszczak if (enqd_len + trb_buff_len >= full_len) { 19495c7578c3SJing Leng if (need_zero_pkt && !zero_len_trb) { 19505c7578c3SJing Leng zero_len_trb = true; 19515c7578c3SJing Leng } else { 19525c7578c3SJing Leng zero_len_trb = false; 19533d829045SPawel Laszczak field &= ~TRB_CHAIN; 19543d829045SPawel Laszczak field |= TRB_IOC; 19553d829045SPawel Laszczak more_trbs_coming = false; 19565c7578c3SJing Leng need_zero_pkt = false; 19573d829045SPawel Laszczak preq->td.last_trb = ring->enqueue; 19583d829045SPawel Laszczak } 19595c7578c3SJing Leng } 19603d829045SPawel Laszczak 19613d829045SPawel Laszczak /* Only set interrupt on short packet for OUT endpoints. */ 19623d829045SPawel Laszczak if (!preq->direction) 19633d829045SPawel Laszczak field |= TRB_ISP; 19643d829045SPawel Laszczak 19653d829045SPawel Laszczak /* Set the TRB length, TD size, and interrupter fields. */ 19663d829045SPawel Laszczak remainder = cdnsp_td_remainder(pdev, enqd_len, trb_buff_len, 19673d829045SPawel Laszczak full_len, preq, 19687a21b27aSPawel Laszczak more_trbs_coming, 19697a21b27aSPawel Laszczak zero_len_trb); 19703d829045SPawel Laszczak 19713d829045SPawel Laszczak length_field = TRB_LEN(trb_buff_len) | TRB_TD_SIZE(remainder) | 19723d829045SPawel Laszczak TRB_INTR_TARGET(0); 19733d829045SPawel Laszczak 19745c7578c3SJing Leng cdnsp_queue_trb(pdev, ring, more_trbs_coming, 19753d829045SPawel Laszczak lower_32_bits(send_addr), 19763d829045SPawel Laszczak upper_32_bits(send_addr), 19773d829045SPawel Laszczak length_field, 19783d829045SPawel Laszczak field); 19793d829045SPawel Laszczak 19803d829045SPawel Laszczak addr += trb_buff_len; 19813d829045SPawel Laszczak sent_len = trb_buff_len; 19823d829045SPawel Laszczak while (sg && sent_len >= block_len) { 19833d829045SPawel Laszczak /* New sg entry */ 19843d829045SPawel Laszczak --num_sgs; 19853d829045SPawel Laszczak sent_len -= block_len; 19863d829045SPawel Laszczak if (num_sgs != 0) { 19873d829045SPawel Laszczak sg = sg_next(sg); 19883d829045SPawel Laszczak block_len = sg_dma_len(sg); 19893d829045SPawel Laszczak addr = (u64)sg_dma_address(sg); 19903d829045SPawel Laszczak addr += sent_len; 19913d829045SPawel Laszczak } 19923d829045SPawel Laszczak } 19933d829045SPawel Laszczak block_len -= sent_len; 19943d829045SPawel Laszczak send_addr = addr; 19953d829045SPawel Laszczak } 19963d829045SPawel Laszczak 19973d829045SPawel Laszczak cdnsp_check_trb_math(preq, enqd_len); 19983d829045SPawel Laszczak ret = cdnsp_giveback_first_trb(pdev, pep, preq->request.stream_id, 19993d829045SPawel Laszczak start_cycle, start_trb); 20003d829045SPawel Laszczak 20013d829045SPawel Laszczak if (ret) 20023d829045SPawel Laszczak preq->td.drbl = 1; 20033d829045SPawel Laszczak 20043d829045SPawel Laszczak return 0; 20053d829045SPawel Laszczak } 20063d829045SPawel Laszczak 20073d829045SPawel Laszczak int cdnsp_queue_ctrl_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq) 20083d829045SPawel Laszczak { 2009*ae423ef5SPawel Laszczak u32 field, length_field, zlp = 0; 20103d829045SPawel Laszczak struct cdnsp_ep *pep = preq->pep; 20113d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 20123d829045SPawel Laszczak int num_trbs; 2013*ae423ef5SPawel Laszczak u32 maxp; 20143d829045SPawel Laszczak int ret; 20153d829045SPawel Laszczak 20163d829045SPawel Laszczak ep_ring = cdnsp_request_to_transfer_ring(pdev, preq); 20173d829045SPawel Laszczak if (!ep_ring) 20183d829045SPawel Laszczak return -EINVAL; 20193d829045SPawel Laszczak 20203d829045SPawel Laszczak /* 1 TRB for data, 1 for status */ 20213d829045SPawel Laszczak num_trbs = (pdev->three_stage_setup) ? 2 : 1; 20223d829045SPawel Laszczak 2023*ae423ef5SPawel Laszczak maxp = usb_endpoint_maxp(pep->endpoint.desc); 2024*ae423ef5SPawel Laszczak 2025*ae423ef5SPawel Laszczak if (preq->request.zero && preq->request.length && 2026*ae423ef5SPawel Laszczak (preq->request.length % maxp == 0)) { 2027*ae423ef5SPawel Laszczak num_trbs++; 2028*ae423ef5SPawel Laszczak zlp = 1; 2029*ae423ef5SPawel Laszczak } 2030*ae423ef5SPawel Laszczak 20313d829045SPawel Laszczak ret = cdnsp_prepare_transfer(pdev, preq, num_trbs); 20323d829045SPawel Laszczak if (ret) 20333d829045SPawel Laszczak return ret; 20343d829045SPawel Laszczak 20353d829045SPawel Laszczak /* If there's data, queue data TRBs */ 20363d829045SPawel Laszczak if (preq->request.length > 0) { 2037*ae423ef5SPawel Laszczak field = TRB_TYPE(TRB_DATA); 20383d829045SPawel Laszczak 2039*ae423ef5SPawel Laszczak if (zlp) 2040*ae423ef5SPawel Laszczak field |= TRB_CHAIN; 2041*ae423ef5SPawel Laszczak else 2042*ae423ef5SPawel Laszczak field |= TRB_IOC | (pdev->ep0_expect_in ? 0 : TRB_ISP); 20433d829045SPawel Laszczak 20443d829045SPawel Laszczak if (pdev->ep0_expect_in) 20453d829045SPawel Laszczak field |= TRB_DIR_IN; 20463d829045SPawel Laszczak 2047*ae423ef5SPawel Laszczak length_field = TRB_LEN(preq->request.length) | 2048*ae423ef5SPawel Laszczak TRB_TD_SIZE(zlp) | TRB_INTR_TARGET(0); 2049*ae423ef5SPawel Laszczak 20503d829045SPawel Laszczak cdnsp_queue_trb(pdev, ep_ring, true, 20513d829045SPawel Laszczak lower_32_bits(preq->request.dma), 20523d829045SPawel Laszczak upper_32_bits(preq->request.dma), length_field, 20533d829045SPawel Laszczak field | ep_ring->cycle_state | 20543d829045SPawel Laszczak TRB_SETUPID(pdev->setup_id) | 20553d829045SPawel Laszczak pdev->setup_speed); 20563d829045SPawel Laszczak 2057*ae423ef5SPawel Laszczak if (zlp) { 2058*ae423ef5SPawel Laszczak field = TRB_TYPE(TRB_NORMAL) | TRB_IOC; 2059*ae423ef5SPawel Laszczak 2060*ae423ef5SPawel Laszczak if (!pdev->ep0_expect_in) 2061*ae423ef5SPawel Laszczak field = TRB_ISP; 2062*ae423ef5SPawel Laszczak 2063*ae423ef5SPawel Laszczak cdnsp_queue_trb(pdev, ep_ring, true, 2064*ae423ef5SPawel Laszczak lower_32_bits(preq->request.dma), 2065*ae423ef5SPawel Laszczak upper_32_bits(preq->request.dma), 0, 2066*ae423ef5SPawel Laszczak field | ep_ring->cycle_state | 2067*ae423ef5SPawel Laszczak TRB_SETUPID(pdev->setup_id) | 2068*ae423ef5SPawel Laszczak pdev->setup_speed); 2069*ae423ef5SPawel Laszczak } 2070*ae423ef5SPawel Laszczak 20713d829045SPawel Laszczak pdev->ep0_stage = CDNSP_DATA_STAGE; 20723d829045SPawel Laszczak } 20733d829045SPawel Laszczak 20743d829045SPawel Laszczak /* Save the DMA address of the last TRB in the TD. */ 20753d829045SPawel Laszczak preq->td.last_trb = ep_ring->enqueue; 20763d829045SPawel Laszczak 20773d829045SPawel Laszczak /* Queue status TRB. */ 20783d829045SPawel Laszczak if (preq->request.length == 0) 20793d829045SPawel Laszczak field = ep_ring->cycle_state; 20803d829045SPawel Laszczak else 20813d829045SPawel Laszczak field = (ep_ring->cycle_state ^ 1); 20823d829045SPawel Laszczak 20833d829045SPawel Laszczak if (preq->request.length > 0 && pdev->ep0_expect_in) 20843d829045SPawel Laszczak field |= TRB_DIR_IN; 20853d829045SPawel Laszczak 20863d829045SPawel Laszczak if (pep->ep_state & EP0_HALTED_STATUS) { 20873d829045SPawel Laszczak pep->ep_state &= ~EP0_HALTED_STATUS; 20883d829045SPawel Laszczak field |= TRB_SETUPSTAT(TRB_SETUPSTAT_STALL); 20893d829045SPawel Laszczak } else { 20903d829045SPawel Laszczak field |= TRB_SETUPSTAT(TRB_SETUPSTAT_ACK); 20913d829045SPawel Laszczak } 20923d829045SPawel Laszczak 20933d829045SPawel Laszczak cdnsp_queue_trb(pdev, ep_ring, false, 0, 0, TRB_INTR_TARGET(0), 20943d829045SPawel Laszczak field | TRB_IOC | TRB_SETUPID(pdev->setup_id) | 20953d829045SPawel Laszczak TRB_TYPE(TRB_STATUS) | pdev->setup_speed); 20963d829045SPawel Laszczak 20973d829045SPawel Laszczak cdnsp_ring_ep_doorbell(pdev, pep, preq->request.stream_id); 20983d829045SPawel Laszczak 20993d829045SPawel Laszczak return 0; 21003d829045SPawel Laszczak } 21013d829045SPawel Laszczak 21023d829045SPawel Laszczak int cdnsp_cmd_stop_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep) 21033d829045SPawel Laszczak { 21043d829045SPawel Laszczak u32 ep_state = GET_EP_CTX_STATE(pep->out_ctx); 21053d829045SPawel Laszczak int ret = 0; 21063d829045SPawel Laszczak 2107b25264f2SPawel Laszczak if (ep_state == EP_STATE_STOPPED || ep_state == EP_STATE_DISABLED || 2108b25264f2SPawel Laszczak ep_state == EP_STATE_HALTED) { 2109118b2a32SPawel Laszczak trace_cdnsp_ep_stopped_or_disabled(pep->out_ctx); 21103d829045SPawel Laszczak goto ep_stopped; 2111118b2a32SPawel Laszczak } 21123d829045SPawel Laszczak 21133d829045SPawel Laszczak cdnsp_queue_stop_endpoint(pdev, pep->idx); 21143d829045SPawel Laszczak cdnsp_ring_cmd_db(pdev); 21153d829045SPawel Laszczak ret = cdnsp_wait_for_cmd_compl(pdev); 21163d829045SPawel Laszczak 2117118b2a32SPawel Laszczak trace_cdnsp_handle_cmd_stop_ep(pep->out_ctx); 2118118b2a32SPawel Laszczak 21193d829045SPawel Laszczak ep_stopped: 21203d829045SPawel Laszczak pep->ep_state |= EP_STOPPED; 21213d829045SPawel Laszczak return ret; 21223d829045SPawel Laszczak } 21233d829045SPawel Laszczak 21243d829045SPawel Laszczak int cdnsp_cmd_flush_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep) 21253d829045SPawel Laszczak { 21263d829045SPawel Laszczak int ret; 21273d829045SPawel Laszczak 21283d829045SPawel Laszczak cdnsp_queue_flush_endpoint(pdev, pep->idx); 21293d829045SPawel Laszczak cdnsp_ring_cmd_db(pdev); 21303d829045SPawel Laszczak ret = cdnsp_wait_for_cmd_compl(pdev); 21313d829045SPawel Laszczak 2132118b2a32SPawel Laszczak trace_cdnsp_handle_cmd_flush_ep(pep->out_ctx); 2133118b2a32SPawel Laszczak 21343d829045SPawel Laszczak return ret; 21353d829045SPawel Laszczak } 21363d829045SPawel Laszczak 21373d829045SPawel Laszczak /* 21383d829045SPawel Laszczak * The transfer burst count field of the isochronous TRB defines the number of 21393d829045SPawel Laszczak * bursts that are required to move all packets in this TD. Only SuperSpeed 21403d829045SPawel Laszczak * devices can burst up to bMaxBurst number of packets per service interval. 21413d829045SPawel Laszczak * This field is zero based, meaning a value of zero in the field means one 21423d829045SPawel Laszczak * burst. Basically, for everything but SuperSpeed devices, this field will be 21433d829045SPawel Laszczak * zero. 21443d829045SPawel Laszczak */ 21453d829045SPawel Laszczak static unsigned int cdnsp_get_burst_count(struct cdnsp_device *pdev, 21463d829045SPawel Laszczak struct cdnsp_request *preq, 21473d829045SPawel Laszczak unsigned int total_packet_count) 21483d829045SPawel Laszczak { 21493d829045SPawel Laszczak unsigned int max_burst; 21503d829045SPawel Laszczak 21513d829045SPawel Laszczak if (pdev->gadget.speed < USB_SPEED_SUPER) 21523d829045SPawel Laszczak return 0; 21533d829045SPawel Laszczak 21543d829045SPawel Laszczak max_burst = preq->pep->endpoint.comp_desc->bMaxBurst; 21553d829045SPawel Laszczak return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1; 21563d829045SPawel Laszczak } 21573d829045SPawel Laszczak 21583d829045SPawel Laszczak /* 21593d829045SPawel Laszczak * Returns the number of packets in the last "burst" of packets. This field is 21603d829045SPawel Laszczak * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so 21613d829045SPawel Laszczak * the last burst packet count is equal to the total number of packets in the 21623d829045SPawel Laszczak * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst 21633d829045SPawel Laszczak * must contain (bMaxBurst + 1) number of packets, but the last burst can 21643d829045SPawel Laszczak * contain 1 to (bMaxBurst + 1) packets. 21653d829045SPawel Laszczak */ 21663d829045SPawel Laszczak static unsigned int 21673d829045SPawel Laszczak cdnsp_get_last_burst_packet_count(struct cdnsp_device *pdev, 21683d829045SPawel Laszczak struct cdnsp_request *preq, 21693d829045SPawel Laszczak unsigned int total_packet_count) 21703d829045SPawel Laszczak { 21713d829045SPawel Laszczak unsigned int max_burst; 21723d829045SPawel Laszczak unsigned int residue; 21733d829045SPawel Laszczak 21743d829045SPawel Laszczak if (pdev->gadget.speed >= USB_SPEED_SUPER) { 21753d829045SPawel Laszczak /* bMaxBurst is zero based: 0 means 1 packet per burst. */ 21763d829045SPawel Laszczak max_burst = preq->pep->endpoint.comp_desc->bMaxBurst; 21773d829045SPawel Laszczak residue = total_packet_count % (max_burst + 1); 21783d829045SPawel Laszczak 21793d829045SPawel Laszczak /* 21803d829045SPawel Laszczak * If residue is zero, the last burst contains (max_burst + 1) 21813d829045SPawel Laszczak * number of packets, but the TLBPC field is zero-based. 21823d829045SPawel Laszczak */ 21833d829045SPawel Laszczak if (residue == 0) 21843d829045SPawel Laszczak return max_burst; 21853d829045SPawel Laszczak 21863d829045SPawel Laszczak return residue - 1; 21873d829045SPawel Laszczak } 21883d829045SPawel Laszczak if (total_packet_count == 0) 21893d829045SPawel Laszczak return 0; 21903d829045SPawel Laszczak 21913d829045SPawel Laszczak return total_packet_count - 1; 21923d829045SPawel Laszczak } 21933d829045SPawel Laszczak 21943d829045SPawel Laszczak /* Queue function isoc transfer */ 21953d829045SPawel Laszczak static int cdnsp_queue_isoc_tx(struct cdnsp_device *pdev, 21963d829045SPawel Laszczak struct cdnsp_request *preq) 21973d829045SPawel Laszczak { 21983d829045SPawel Laszczak int trb_buff_len, td_len, td_remain_len, ret; 21993d829045SPawel Laszczak unsigned int burst_count, last_burst_pkt; 22003d829045SPawel Laszczak unsigned int total_pkt_count, max_pkt; 22013d829045SPawel Laszczak struct cdnsp_generic_trb *start_trb; 22023d829045SPawel Laszczak bool more_trbs_coming = true; 22033d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 22043d829045SPawel Laszczak int running_total = 0; 22053d829045SPawel Laszczak u32 field, length_field; 22063d829045SPawel Laszczak int start_cycle; 22073d829045SPawel Laszczak int trbs_per_td; 22083d829045SPawel Laszczak u64 addr; 22093d829045SPawel Laszczak int i; 22103d829045SPawel Laszczak 22113d829045SPawel Laszczak ep_ring = preq->pep->ring; 22123d829045SPawel Laszczak start_trb = &ep_ring->enqueue->generic; 22133d829045SPawel Laszczak start_cycle = ep_ring->cycle_state; 22143d829045SPawel Laszczak td_len = preq->request.length; 22153d829045SPawel Laszczak addr = (u64)preq->request.dma; 22163d829045SPawel Laszczak td_remain_len = td_len; 22173d829045SPawel Laszczak 22183d829045SPawel Laszczak max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc); 22193d829045SPawel Laszczak total_pkt_count = DIV_ROUND_UP(td_len, max_pkt); 22203d829045SPawel Laszczak 22213d829045SPawel Laszczak /* A zero-length transfer still involves at least one packet. */ 22223d829045SPawel Laszczak if (total_pkt_count == 0) 22233d829045SPawel Laszczak total_pkt_count++; 22243d829045SPawel Laszczak 22253d829045SPawel Laszczak burst_count = cdnsp_get_burst_count(pdev, preq, total_pkt_count); 22263d829045SPawel Laszczak last_burst_pkt = cdnsp_get_last_burst_packet_count(pdev, preq, 22273d829045SPawel Laszczak total_pkt_count); 22283d829045SPawel Laszczak trbs_per_td = count_isoc_trbs_needed(preq); 22293d829045SPawel Laszczak 22303d829045SPawel Laszczak ret = cdnsp_prepare_transfer(pdev, preq, trbs_per_td); 22313d829045SPawel Laszczak if (ret) 22323d829045SPawel Laszczak goto cleanup; 22333d829045SPawel Laszczak 22343d829045SPawel Laszczak /* 22353d829045SPawel Laszczak * Set isoc specific data for the first TRB in a TD. 22363d829045SPawel Laszczak * Prevent HW from getting the TRBs by keeping the cycle state 22373d829045SPawel Laszczak * inverted in the first TDs isoc TRB. 22383d829045SPawel Laszczak */ 22393d829045SPawel Laszczak field = TRB_TYPE(TRB_ISOC) | TRB_TLBPC(last_burst_pkt) | 2240d0dcd90bSPawel Laszczak TRB_SIA | TRB_TBC(burst_count); 2241d0dcd90bSPawel Laszczak 2242d0dcd90bSPawel Laszczak if (!start_cycle) 2243d0dcd90bSPawel Laszczak field |= TRB_CYCLE; 22443d829045SPawel Laszczak 22453d829045SPawel Laszczak /* Fill the rest of the TRB fields, and remaining normal TRBs. */ 22463d829045SPawel Laszczak for (i = 0; i < trbs_per_td; i++) { 22473d829045SPawel Laszczak u32 remainder; 22483d829045SPawel Laszczak 22493d829045SPawel Laszczak /* Calculate TRB length. */ 22503d829045SPawel Laszczak trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr); 22513d829045SPawel Laszczak if (trb_buff_len > td_remain_len) 22523d829045SPawel Laszczak trb_buff_len = td_remain_len; 22533d829045SPawel Laszczak 22543d829045SPawel Laszczak /* Set the TRB length, TD size, & interrupter fields. */ 22553d829045SPawel Laszczak remainder = cdnsp_td_remainder(pdev, running_total, 22563d829045SPawel Laszczak trb_buff_len, td_len, preq, 22577a21b27aSPawel Laszczak more_trbs_coming, 0); 22583d829045SPawel Laszczak 22593d829045SPawel Laszczak length_field = TRB_LEN(trb_buff_len) | TRB_INTR_TARGET(0); 22603d829045SPawel Laszczak 22613d829045SPawel Laszczak /* Only first TRB is isoc, overwrite otherwise. */ 22623d829045SPawel Laszczak if (i) { 22633d829045SPawel Laszczak field = TRB_TYPE(TRB_NORMAL) | ep_ring->cycle_state; 22643d829045SPawel Laszczak length_field |= TRB_TD_SIZE(remainder); 22653d829045SPawel Laszczak } else { 22663d829045SPawel Laszczak length_field |= TRB_TD_SIZE_TBC(burst_count); 22673d829045SPawel Laszczak } 22683d829045SPawel Laszczak 22693d829045SPawel Laszczak /* Only set interrupt on short packet for OUT EPs. */ 22703d829045SPawel Laszczak if (usb_endpoint_dir_out(preq->pep->endpoint.desc)) 22713d829045SPawel Laszczak field |= TRB_ISP; 22723d829045SPawel Laszczak 22733d829045SPawel Laszczak /* Set the chain bit for all except the last TRB. */ 22743d829045SPawel Laszczak if (i < trbs_per_td - 1) { 22753d829045SPawel Laszczak more_trbs_coming = true; 22763d829045SPawel Laszczak field |= TRB_CHAIN; 22773d829045SPawel Laszczak } else { 22783d829045SPawel Laszczak more_trbs_coming = false; 22793d829045SPawel Laszczak preq->td.last_trb = ep_ring->enqueue; 22803d829045SPawel Laszczak field |= TRB_IOC; 22813d829045SPawel Laszczak } 22823d829045SPawel Laszczak 22833d829045SPawel Laszczak cdnsp_queue_trb(pdev, ep_ring, more_trbs_coming, 22843d829045SPawel Laszczak lower_32_bits(addr), upper_32_bits(addr), 22853d829045SPawel Laszczak length_field, field); 22863d829045SPawel Laszczak 22873d829045SPawel Laszczak running_total += trb_buff_len; 22883d829045SPawel Laszczak addr += trb_buff_len; 22893d829045SPawel Laszczak td_remain_len -= trb_buff_len; 22903d829045SPawel Laszczak } 22913d829045SPawel Laszczak 22923d829045SPawel Laszczak /* Check TD length */ 22933d829045SPawel Laszczak if (running_total != td_len) { 22943d829045SPawel Laszczak dev_err(pdev->dev, "ISOC TD length unmatch\n"); 22953d829045SPawel Laszczak ret = -EINVAL; 22963d829045SPawel Laszczak goto cleanup; 22973d829045SPawel Laszczak } 22983d829045SPawel Laszczak 22993d829045SPawel Laszczak cdnsp_giveback_first_trb(pdev, preq->pep, preq->request.stream_id, 23003d829045SPawel Laszczak start_cycle, start_trb); 23013d829045SPawel Laszczak 23023d829045SPawel Laszczak return 0; 23033d829045SPawel Laszczak 23043d829045SPawel Laszczak cleanup: 23053d829045SPawel Laszczak /* Clean up a partially enqueued isoc transfer. */ 23063d829045SPawel Laszczak list_del_init(&preq->td.td_list); 23073d829045SPawel Laszczak ep_ring->num_tds--; 23083d829045SPawel Laszczak 23093d829045SPawel Laszczak /* 23103d829045SPawel Laszczak * Use the first TD as a temporary variable to turn the TDs we've 23113d829045SPawel Laszczak * queued into No-ops with a software-owned cycle bit. 23123d829045SPawel Laszczak * That way the hardware won't accidentally start executing bogus TDs 23133d829045SPawel Laszczak * when we partially overwrite them. 23143d829045SPawel Laszczak * td->first_trb and td->start_seg are already set. 23153d829045SPawel Laszczak */ 23163d829045SPawel Laszczak preq->td.last_trb = ep_ring->enqueue; 23173d829045SPawel Laszczak /* Every TRB except the first & last will have its cycle bit flipped. */ 23183d829045SPawel Laszczak cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true); 23193d829045SPawel Laszczak 23203d829045SPawel Laszczak /* Reset the ring enqueue back to the first TRB and its cycle bit. */ 23213d829045SPawel Laszczak ep_ring->enqueue = preq->td.first_trb; 23223d829045SPawel Laszczak ep_ring->enq_seg = preq->td.start_seg; 23233d829045SPawel Laszczak ep_ring->cycle_state = start_cycle; 23243d829045SPawel Laszczak return ret; 23253d829045SPawel Laszczak } 23263d829045SPawel Laszczak 23273d829045SPawel Laszczak int cdnsp_queue_isoc_tx_prepare(struct cdnsp_device *pdev, 23283d829045SPawel Laszczak struct cdnsp_request *preq) 23293d829045SPawel Laszczak { 23303d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 23313d829045SPawel Laszczak u32 ep_state; 23323d829045SPawel Laszczak int num_trbs; 23333d829045SPawel Laszczak int ret; 23343d829045SPawel Laszczak 23353d829045SPawel Laszczak ep_ring = preq->pep->ring; 23363d829045SPawel Laszczak ep_state = GET_EP_CTX_STATE(preq->pep->out_ctx); 23373d829045SPawel Laszczak num_trbs = count_isoc_trbs_needed(preq); 23383d829045SPawel Laszczak 23393d829045SPawel Laszczak /* 23403d829045SPawel Laszczak * Check the ring to guarantee there is enough room for the whole 23413d829045SPawel Laszczak * request. Do not insert any td of the USB Request to the ring if the 23423d829045SPawel Laszczak * check failed. 23433d829045SPawel Laszczak */ 23443d829045SPawel Laszczak ret = cdnsp_prepare_ring(pdev, ep_ring, ep_state, num_trbs, GFP_ATOMIC); 23453d829045SPawel Laszczak if (ret) 23463d829045SPawel Laszczak return ret; 23473d829045SPawel Laszczak 23483d829045SPawel Laszczak return cdnsp_queue_isoc_tx(pdev, preq); 23493d829045SPawel Laszczak } 23503d829045SPawel Laszczak 23513d829045SPawel Laszczak /**** Command Ring Operations ****/ 23523d829045SPawel Laszczak /* 23533d829045SPawel Laszczak * Generic function for queuing a command TRB on the command ring. 23543d829045SPawel Laszczak * Driver queue only one command to ring in the moment. 23553d829045SPawel Laszczak */ 23563d829045SPawel Laszczak static void cdnsp_queue_command(struct cdnsp_device *pdev, 23573d829045SPawel Laszczak u32 field1, 23583d829045SPawel Laszczak u32 field2, 23593d829045SPawel Laszczak u32 field3, 23603d829045SPawel Laszczak u32 field4) 23613d829045SPawel Laszczak { 23623d829045SPawel Laszczak cdnsp_prepare_ring(pdev, pdev->cmd_ring, EP_STATE_RUNNING, 1, 23633d829045SPawel Laszczak GFP_ATOMIC); 23643d829045SPawel Laszczak 23653d829045SPawel Laszczak pdev->cmd.command_trb = pdev->cmd_ring->enqueue; 23663d829045SPawel Laszczak 23673d829045SPawel Laszczak cdnsp_queue_trb(pdev, pdev->cmd_ring, false, field1, field2, 23683d829045SPawel Laszczak field3, field4 | pdev->cmd_ring->cycle_state); 23693d829045SPawel Laszczak } 23703d829045SPawel Laszczak 23713d829045SPawel Laszczak /* Queue a slot enable or disable request on the command ring */ 23723d829045SPawel Laszczak void cdnsp_queue_slot_control(struct cdnsp_device *pdev, u32 trb_type) 23733d829045SPawel Laszczak { 23743d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(trb_type) | 23753d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id)); 23763d829045SPawel Laszczak } 23773d829045SPawel Laszczak 23783d829045SPawel Laszczak /* Queue an address device command TRB */ 23793d829045SPawel Laszczak void cdnsp_queue_address_device(struct cdnsp_device *pdev, 23803d829045SPawel Laszczak dma_addr_t in_ctx_ptr, 23813d829045SPawel Laszczak enum cdnsp_setup_dev setup) 23823d829045SPawel Laszczak { 23833d829045SPawel Laszczak cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr), 23843d829045SPawel Laszczak upper_32_bits(in_ctx_ptr), 0, 23853d829045SPawel Laszczak TRB_TYPE(TRB_ADDR_DEV) | 23863d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id) | 23873d829045SPawel Laszczak (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0)); 23883d829045SPawel Laszczak } 23893d829045SPawel Laszczak 23903d829045SPawel Laszczak /* Queue a reset device command TRB */ 23913d829045SPawel Laszczak void cdnsp_queue_reset_device(struct cdnsp_device *pdev) 23923d829045SPawel Laszczak { 23933d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_RESET_DEV) | 23943d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id)); 23953d829045SPawel Laszczak } 23963d829045SPawel Laszczak 23973d829045SPawel Laszczak /* Queue a configure endpoint command TRB */ 23983d829045SPawel Laszczak void cdnsp_queue_configure_endpoint(struct cdnsp_device *pdev, 23993d829045SPawel Laszczak dma_addr_t in_ctx_ptr) 24003d829045SPawel Laszczak { 24013d829045SPawel Laszczak cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr), 24023d829045SPawel Laszczak upper_32_bits(in_ctx_ptr), 0, 24033d829045SPawel Laszczak TRB_TYPE(TRB_CONFIG_EP) | 24043d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id)); 24053d829045SPawel Laszczak } 24063d829045SPawel Laszczak 24073d829045SPawel Laszczak /* 24083d829045SPawel Laszczak * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop 24093d829045SPawel Laszczak * activity on an endpoint that is about to be suspended. 24103d829045SPawel Laszczak */ 24113d829045SPawel Laszczak void cdnsp_queue_stop_endpoint(struct cdnsp_device *pdev, unsigned int ep_index) 24123d829045SPawel Laszczak { 24133d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, SLOT_ID_FOR_TRB(pdev->slot_id) | 24143d829045SPawel Laszczak EP_ID_FOR_TRB(ep_index) | TRB_TYPE(TRB_STOP_RING)); 24153d829045SPawel Laszczak } 24163d829045SPawel Laszczak 24173d829045SPawel Laszczak /* Set Transfer Ring Dequeue Pointer command. */ 24183d829045SPawel Laszczak void cdnsp_queue_new_dequeue_state(struct cdnsp_device *pdev, 24193d829045SPawel Laszczak struct cdnsp_ep *pep, 24203d829045SPawel Laszczak struct cdnsp_dequeue_state *deq_state) 24213d829045SPawel Laszczak { 24223d829045SPawel Laszczak u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id); 24233d829045SPawel Laszczak u32 trb_slot_id = SLOT_ID_FOR_TRB(pdev->slot_id); 24243d829045SPawel Laszczak u32 type = TRB_TYPE(TRB_SET_DEQ); 24253d829045SPawel Laszczak u32 trb_sct = 0; 24263d829045SPawel Laszczak dma_addr_t addr; 24273d829045SPawel Laszczak 24283d829045SPawel Laszczak addr = cdnsp_trb_virt_to_dma(deq_state->new_deq_seg, 24293d829045SPawel Laszczak deq_state->new_deq_ptr); 24303d829045SPawel Laszczak 24313d829045SPawel Laszczak if (deq_state->stream_id) 24323d829045SPawel Laszczak trb_sct = SCT_FOR_TRB(SCT_PRI_TR); 24333d829045SPawel Laszczak 24343d829045SPawel Laszczak cdnsp_queue_command(pdev, lower_32_bits(addr) | trb_sct | 24353d829045SPawel Laszczak deq_state->new_cycle_state, upper_32_bits(addr), 24363d829045SPawel Laszczak trb_stream_id, trb_slot_id | 24373d829045SPawel Laszczak EP_ID_FOR_TRB(pep->idx) | type); 24383d829045SPawel Laszczak } 24393d829045SPawel Laszczak 24403d829045SPawel Laszczak void cdnsp_queue_reset_ep(struct cdnsp_device *pdev, unsigned int ep_index) 24413d829045SPawel Laszczak { 24423d829045SPawel Laszczak return cdnsp_queue_command(pdev, 0, 0, 0, 24433d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id) | 24443d829045SPawel Laszczak EP_ID_FOR_TRB(ep_index) | 24453d829045SPawel Laszczak TRB_TYPE(TRB_RESET_EP)); 24463d829045SPawel Laszczak } 24473d829045SPawel Laszczak 24483d829045SPawel Laszczak /* 24493d829045SPawel Laszczak * Queue a halt endpoint request on the command ring. 24503d829045SPawel Laszczak */ 24513d829045SPawel Laszczak void cdnsp_queue_halt_endpoint(struct cdnsp_device *pdev, unsigned int ep_index) 24523d829045SPawel Laszczak { 24533d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_HALT_ENDPOINT) | 24543d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id) | 24553d829045SPawel Laszczak EP_ID_FOR_TRB(ep_index)); 24563d829045SPawel Laszczak } 24573d829045SPawel Laszczak 24583d829045SPawel Laszczak /* 24593d829045SPawel Laszczak * Queue a flush endpoint request on the command ring. 24603d829045SPawel Laszczak */ 24613d829045SPawel Laszczak void cdnsp_queue_flush_endpoint(struct cdnsp_device *pdev, 24623d829045SPawel Laszczak unsigned int ep_index) 24633d829045SPawel Laszczak { 24643d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_FLUSH_ENDPOINT) | 24653d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id) | 24663d829045SPawel Laszczak EP_ID_FOR_TRB(ep_index)); 24673d829045SPawel Laszczak } 24683d829045SPawel Laszczak 24693d829045SPawel Laszczak void cdnsp_force_header_wakeup(struct cdnsp_device *pdev, int intf_num) 24703d829045SPawel Laszczak { 24713d829045SPawel Laszczak u32 lo, mid; 24723d829045SPawel Laszczak 24733d829045SPawel Laszczak lo = TRB_FH_TO_PACKET_TYPE(TRB_FH_TR_PACKET) | 24743d829045SPawel Laszczak TRB_FH_TO_DEVICE_ADDRESS(pdev->device_address); 24753d829045SPawel Laszczak mid = TRB_FH_TR_PACKET_DEV_NOT | 24763d829045SPawel Laszczak TRB_FH_TO_NOT_TYPE(TRB_FH_TR_PACKET_FUNCTION_WAKE) | 24773d829045SPawel Laszczak TRB_FH_TO_INTERFACE(intf_num); 24783d829045SPawel Laszczak 24793d829045SPawel Laszczak cdnsp_queue_command(pdev, lo, mid, 0, 24803d829045SPawel Laszczak TRB_TYPE(TRB_FORCE_HEADER) | SET_PORT_ID(2)); 24813d829045SPawel Laszczak } 2482