13d829045SPawel Laszczak // SPDX-License-Identifier: GPL-2.0 23d829045SPawel Laszczak /* 33d829045SPawel Laszczak * Cadence CDNSP DRD Driver. 43d829045SPawel Laszczak * 53d829045SPawel Laszczak * Copyright (C) 2020 Cadence. 63d829045SPawel Laszczak * 73d829045SPawel Laszczak * Author: Pawel Laszczak <pawell@cadence.com> 83d829045SPawel Laszczak * 93d829045SPawel Laszczak * Code based on Linux XHCI driver. 103d829045SPawel Laszczak * Origin: Copyright (C) 2008 Intel Corp 113d829045SPawel Laszczak */ 123d829045SPawel Laszczak 133d829045SPawel Laszczak /* 143d829045SPawel Laszczak * Ring initialization rules: 153d829045SPawel Laszczak * 1. Each segment is initialized to zero, except for link TRBs. 163d829045SPawel Laszczak * 2. Ring cycle state = 0. This represents Producer Cycle State (PCS) or 173d829045SPawel Laszczak * Consumer Cycle State (CCS), depending on ring function. 183d829045SPawel Laszczak * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment. 193d829045SPawel Laszczak * 203d829045SPawel Laszczak * Ring behavior rules: 213d829045SPawel Laszczak * 1. A ring is empty if enqueue == dequeue. This means there will always be at 223d829045SPawel Laszczak * least one free TRB in the ring. This is useful if you want to turn that 233d829045SPawel Laszczak * into a link TRB and expand the ring. 243d829045SPawel Laszczak * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a 253d829045SPawel Laszczak * link TRB, then load the pointer with the address in the link TRB. If the 263d829045SPawel Laszczak * link TRB had its toggle bit set, you may need to update the ring cycle 273d829045SPawel Laszczak * state (see cycle bit rules). You may have to do this multiple times 283d829045SPawel Laszczak * until you reach a non-link TRB. 293d829045SPawel Laszczak * 3. A ring is full if enqueue++ (for the definition of increment above) 303d829045SPawel Laszczak * equals the dequeue pointer. 313d829045SPawel Laszczak * 323d829045SPawel Laszczak * Cycle bit rules: 333d829045SPawel Laszczak * 1. When a consumer increments a dequeue pointer and encounters a toggle bit 343d829045SPawel Laszczak * in a link TRB, it must toggle the ring cycle state. 353d829045SPawel Laszczak * 2. When a producer increments an enqueue pointer and encounters a toggle bit 363d829045SPawel Laszczak * in a link TRB, it must toggle the ring cycle state. 373d829045SPawel Laszczak * 383d829045SPawel Laszczak * Producer rules: 393d829045SPawel Laszczak * 1. Check if ring is full before you enqueue. 403d829045SPawel Laszczak * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing. 413d829045SPawel Laszczak * Update enqueue pointer between each write (which may update the ring 423d829045SPawel Laszczak * cycle state). 433d829045SPawel Laszczak * 3. Notify consumer. If SW is producer, it rings the doorbell for command 443d829045SPawel Laszczak * and endpoint rings. If controller is the producer for the event ring, 453d829045SPawel Laszczak * and it generates an interrupt according to interrupt modulation rules. 463d829045SPawel Laszczak * 473d829045SPawel Laszczak * Consumer rules: 483d829045SPawel Laszczak * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state, 493d829045SPawel Laszczak * the TRB is owned by the consumer. 503d829045SPawel Laszczak * 2. Update dequeue pointer (which may update the ring cycle state) and 513d829045SPawel Laszczak * continue processing TRBs until you reach a TRB which is not owned by you. 523d829045SPawel Laszczak * 3. Notify the producer. SW is the consumer for the event ring, and it 533d829045SPawel Laszczak * updates event ring dequeue pointer. Controller is the consumer for the 543d829045SPawel Laszczak * command and endpoint rings; it generates events on the event ring 553d829045SPawel Laszczak * for these. 563d829045SPawel Laszczak */ 573d829045SPawel Laszczak 583d829045SPawel Laszczak #include <linux/scatterlist.h> 593d829045SPawel Laszczak #include <linux/dma-mapping.h> 603d829045SPawel Laszczak #include <linux/delay.h> 613d829045SPawel Laszczak #include <linux/slab.h> 623d829045SPawel Laszczak #include <linux/irq.h> 633d829045SPawel Laszczak 64*118b2a32SPawel Laszczak #include "cdnsp-trace.h" 653d829045SPawel Laszczak #include "cdnsp-gadget.h" 663d829045SPawel Laszczak 673d829045SPawel Laszczak /* 683d829045SPawel Laszczak * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA 693d829045SPawel Laszczak * address of the TRB. 703d829045SPawel Laszczak */ 713d829045SPawel Laszczak dma_addr_t cdnsp_trb_virt_to_dma(struct cdnsp_segment *seg, 723d829045SPawel Laszczak union cdnsp_trb *trb) 733d829045SPawel Laszczak { 743d829045SPawel Laszczak unsigned long segment_offset = trb - seg->trbs; 753d829045SPawel Laszczak 763d829045SPawel Laszczak if (trb < seg->trbs || segment_offset >= TRBS_PER_SEGMENT) 773d829045SPawel Laszczak return 0; 783d829045SPawel Laszczak 793d829045SPawel Laszczak return seg->dma + (segment_offset * sizeof(*trb)); 803d829045SPawel Laszczak } 813d829045SPawel Laszczak 823d829045SPawel Laszczak static bool cdnsp_trb_is_noop(union cdnsp_trb *trb) 833d829045SPawel Laszczak { 843d829045SPawel Laszczak return TRB_TYPE_NOOP_LE32(trb->generic.field[3]); 853d829045SPawel Laszczak } 863d829045SPawel Laszczak 873d829045SPawel Laszczak static bool cdnsp_trb_is_link(union cdnsp_trb *trb) 883d829045SPawel Laszczak { 893d829045SPawel Laszczak return TRB_TYPE_LINK_LE32(trb->link.control); 903d829045SPawel Laszczak } 913d829045SPawel Laszczak 923d829045SPawel Laszczak bool cdnsp_last_trb_on_seg(struct cdnsp_segment *seg, union cdnsp_trb *trb) 933d829045SPawel Laszczak { 943d829045SPawel Laszczak return trb == &seg->trbs[TRBS_PER_SEGMENT - 1]; 953d829045SPawel Laszczak } 963d829045SPawel Laszczak 973d829045SPawel Laszczak bool cdnsp_last_trb_on_ring(struct cdnsp_ring *ring, 983d829045SPawel Laszczak struct cdnsp_segment *seg, 993d829045SPawel Laszczak union cdnsp_trb *trb) 1003d829045SPawel Laszczak { 1013d829045SPawel Laszczak return cdnsp_last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg); 1023d829045SPawel Laszczak } 1033d829045SPawel Laszczak 1043d829045SPawel Laszczak static bool cdnsp_link_trb_toggles_cycle(union cdnsp_trb *trb) 1053d829045SPawel Laszczak { 1063d829045SPawel Laszczak return le32_to_cpu(trb->link.control) & LINK_TOGGLE; 1073d829045SPawel Laszczak } 1083d829045SPawel Laszczak 1093d829045SPawel Laszczak static void cdnsp_trb_to_noop(union cdnsp_trb *trb, u32 noop_type) 1103d829045SPawel Laszczak { 1113d829045SPawel Laszczak if (cdnsp_trb_is_link(trb)) { 1123d829045SPawel Laszczak /* Unchain chained link TRBs. */ 1133d829045SPawel Laszczak trb->link.control &= cpu_to_le32(~TRB_CHAIN); 1143d829045SPawel Laszczak } else { 1153d829045SPawel Laszczak trb->generic.field[0] = 0; 1163d829045SPawel Laszczak trb->generic.field[1] = 0; 1173d829045SPawel Laszczak trb->generic.field[2] = 0; 1183d829045SPawel Laszczak /* Preserve only the cycle bit of this TRB. */ 1193d829045SPawel Laszczak trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE); 1203d829045SPawel Laszczak trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type)); 1213d829045SPawel Laszczak } 1223d829045SPawel Laszczak } 1233d829045SPawel Laszczak 1243d829045SPawel Laszczak /* 1253d829045SPawel Laszczak * Updates trb to point to the next TRB in the ring, and updates seg if the next 1263d829045SPawel Laszczak * TRB is in a new segment. This does not skip over link TRBs, and it does not 1273d829045SPawel Laszczak * effect the ring dequeue or enqueue pointers. 1283d829045SPawel Laszczak */ 1293d829045SPawel Laszczak static void cdnsp_next_trb(struct cdnsp_device *pdev, 1303d829045SPawel Laszczak struct cdnsp_ring *ring, 1313d829045SPawel Laszczak struct cdnsp_segment **seg, 1323d829045SPawel Laszczak union cdnsp_trb **trb) 1333d829045SPawel Laszczak { 1343d829045SPawel Laszczak if (cdnsp_trb_is_link(*trb)) { 1353d829045SPawel Laszczak *seg = (*seg)->next; 1363d829045SPawel Laszczak *trb = ((*seg)->trbs); 1373d829045SPawel Laszczak } else { 1383d829045SPawel Laszczak (*trb)++; 1393d829045SPawel Laszczak } 1403d829045SPawel Laszczak } 1413d829045SPawel Laszczak 1423d829045SPawel Laszczak /* 1433d829045SPawel Laszczak * See Cycle bit rules. SW is the consumer for the event ring only. 1443d829045SPawel Laszczak * Don't make a ring full of link TRBs. That would be dumb and this would loop. 1453d829045SPawel Laszczak */ 1463d829045SPawel Laszczak void cdnsp_inc_deq(struct cdnsp_device *pdev, struct cdnsp_ring *ring) 1473d829045SPawel Laszczak { 1483d829045SPawel Laszczak /* event ring doesn't have link trbs, check for last trb. */ 1493d829045SPawel Laszczak if (ring->type == TYPE_EVENT) { 1503d829045SPawel Laszczak if (!cdnsp_last_trb_on_seg(ring->deq_seg, ring->dequeue)) { 1513d829045SPawel Laszczak ring->dequeue++; 152*118b2a32SPawel Laszczak goto out; 1533d829045SPawel Laszczak } 1543d829045SPawel Laszczak 1553d829045SPawel Laszczak if (cdnsp_last_trb_on_ring(ring, ring->deq_seg, ring->dequeue)) 1563d829045SPawel Laszczak ring->cycle_state ^= 1; 1573d829045SPawel Laszczak 1583d829045SPawel Laszczak ring->deq_seg = ring->deq_seg->next; 1593d829045SPawel Laszczak ring->dequeue = ring->deq_seg->trbs; 160*118b2a32SPawel Laszczak goto out; 1613d829045SPawel Laszczak } 1623d829045SPawel Laszczak 1633d829045SPawel Laszczak /* All other rings have link trbs. */ 1643d829045SPawel Laszczak if (!cdnsp_trb_is_link(ring->dequeue)) { 1653d829045SPawel Laszczak ring->dequeue++; 1663d829045SPawel Laszczak ring->num_trbs_free++; 1673d829045SPawel Laszczak } 1683d829045SPawel Laszczak while (cdnsp_trb_is_link(ring->dequeue)) { 1693d829045SPawel Laszczak ring->deq_seg = ring->deq_seg->next; 1703d829045SPawel Laszczak ring->dequeue = ring->deq_seg->trbs; 1713d829045SPawel Laszczak } 172*118b2a32SPawel Laszczak out: 173*118b2a32SPawel Laszczak trace_cdnsp_inc_deq(ring); 1743d829045SPawel Laszczak } 1753d829045SPawel Laszczak 1763d829045SPawel Laszczak /* 1773d829045SPawel Laszczak * See Cycle bit rules. SW is the consumer for the event ring only. 1783d829045SPawel Laszczak * Don't make a ring full of link TRBs. That would be dumb and this would loop. 1793d829045SPawel Laszczak * 1803d829045SPawel Laszczak * If we've just enqueued a TRB that is in the middle of a TD (meaning the 1813d829045SPawel Laszczak * chain bit is set), then set the chain bit in all the following link TRBs. 1823d829045SPawel Laszczak * If we've enqueued the last TRB in a TD, make sure the following link TRBs 1833d829045SPawel Laszczak * have their chain bit cleared (so that each Link TRB is a separate TD). 1843d829045SPawel Laszczak * 1853d829045SPawel Laszczak * @more_trbs_coming: Will you enqueue more TRBs before ringing the doorbell. 1863d829045SPawel Laszczak */ 1873d829045SPawel Laszczak static void cdnsp_inc_enq(struct cdnsp_device *pdev, 1883d829045SPawel Laszczak struct cdnsp_ring *ring, 1893d829045SPawel Laszczak bool more_trbs_coming) 1903d829045SPawel Laszczak { 1913d829045SPawel Laszczak union cdnsp_trb *next; 1923d829045SPawel Laszczak u32 chain; 1933d829045SPawel Laszczak 1943d829045SPawel Laszczak chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN; 1953d829045SPawel Laszczak 1963d829045SPawel Laszczak /* If this is not event ring, there is one less usable TRB. */ 1973d829045SPawel Laszczak if (!cdnsp_trb_is_link(ring->enqueue)) 1983d829045SPawel Laszczak ring->num_trbs_free--; 1993d829045SPawel Laszczak next = ++(ring->enqueue); 2003d829045SPawel Laszczak 2013d829045SPawel Laszczak /* Update the dequeue pointer further if that was a link TRB */ 2023d829045SPawel Laszczak while (cdnsp_trb_is_link(next)) { 2033d829045SPawel Laszczak /* 2043d829045SPawel Laszczak * If the caller doesn't plan on enqueuing more TDs before 2053d829045SPawel Laszczak * ringing the doorbell, then we don't want to give the link TRB 2063d829045SPawel Laszczak * to the hardware just yet. We'll give the link TRB back in 2073d829045SPawel Laszczak * cdnsp_prepare_ring() just before we enqueue the TD at the 2083d829045SPawel Laszczak * top of the ring. 2093d829045SPawel Laszczak */ 2103d829045SPawel Laszczak if (!chain && !more_trbs_coming) 2113d829045SPawel Laszczak break; 2123d829045SPawel Laszczak 2133d829045SPawel Laszczak next->link.control &= cpu_to_le32(~TRB_CHAIN); 2143d829045SPawel Laszczak next->link.control |= cpu_to_le32(chain); 2153d829045SPawel Laszczak 2163d829045SPawel Laszczak /* Give this link TRB to the hardware */ 2173d829045SPawel Laszczak wmb(); 2183d829045SPawel Laszczak next->link.control ^= cpu_to_le32(TRB_CYCLE); 2193d829045SPawel Laszczak 2203d829045SPawel Laszczak /* Toggle the cycle bit after the last ring segment. */ 2213d829045SPawel Laszczak if (cdnsp_link_trb_toggles_cycle(next)) 2223d829045SPawel Laszczak ring->cycle_state ^= 1; 2233d829045SPawel Laszczak 2243d829045SPawel Laszczak ring->enq_seg = ring->enq_seg->next; 2253d829045SPawel Laszczak ring->enqueue = ring->enq_seg->trbs; 2263d829045SPawel Laszczak next = ring->enqueue; 2273d829045SPawel Laszczak } 228*118b2a32SPawel Laszczak 229*118b2a32SPawel Laszczak trace_cdnsp_inc_enq(ring); 2303d829045SPawel Laszczak } 2313d829045SPawel Laszczak 2323d829045SPawel Laszczak /* 2333d829045SPawel Laszczak * Check to see if there's room to enqueue num_trbs on the ring and make sure 2343d829045SPawel Laszczak * enqueue pointer will not advance into dequeue segment. 2353d829045SPawel Laszczak */ 2363d829045SPawel Laszczak static bool cdnsp_room_on_ring(struct cdnsp_device *pdev, 2373d829045SPawel Laszczak struct cdnsp_ring *ring, 2383d829045SPawel Laszczak unsigned int num_trbs) 2393d829045SPawel Laszczak { 2403d829045SPawel Laszczak int num_trbs_in_deq_seg; 2413d829045SPawel Laszczak 2423d829045SPawel Laszczak if (ring->num_trbs_free < num_trbs) 2433d829045SPawel Laszczak return false; 2443d829045SPawel Laszczak 2453d829045SPawel Laszczak if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) { 2463d829045SPawel Laszczak num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs; 2473d829045SPawel Laszczak 2483d829045SPawel Laszczak if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg) 2493d829045SPawel Laszczak return false; 2503d829045SPawel Laszczak } 2513d829045SPawel Laszczak 2523d829045SPawel Laszczak return true; 2533d829045SPawel Laszczak } 2543d829045SPawel Laszczak 2553d829045SPawel Laszczak /* 2563d829045SPawel Laszczak * Workaround for L1: controller has issue with resuming from L1 after 2573d829045SPawel Laszczak * setting doorbell for endpoint during L1 state. This function forces 2583d829045SPawel Laszczak * resume signal in such case. 2593d829045SPawel Laszczak */ 2603d829045SPawel Laszczak static void cdnsp_force_l0_go(struct cdnsp_device *pdev) 2613d829045SPawel Laszczak { 2623d829045SPawel Laszczak if (pdev->active_port == &pdev->usb2_port && pdev->gadget.lpm_capable) 2633d829045SPawel Laszczak cdnsp_set_link_state(pdev, &pdev->active_port->regs->portsc, XDEV_U0); 2643d829045SPawel Laszczak } 2653d829045SPawel Laszczak 2663d829045SPawel Laszczak /* Ring the doorbell after placing a command on the ring. */ 2673d829045SPawel Laszczak void cdnsp_ring_cmd_db(struct cdnsp_device *pdev) 2683d829045SPawel Laszczak { 269*118b2a32SPawel Laszczak trace_cdnsp_cmd_drbl("Ding Dong"); 2703d829045SPawel Laszczak writel(DB_VALUE_CMD, &pdev->dba->cmd_db); 2713d829045SPawel Laszczak } 2723d829045SPawel Laszczak 2733d829045SPawel Laszczak /* 2743d829045SPawel Laszczak * Ring the doorbell after placing a transfer on the ring. 2753d829045SPawel Laszczak * Returns true if doorbell was set, otherwise false. 2763d829045SPawel Laszczak */ 2773d829045SPawel Laszczak static bool cdnsp_ring_ep_doorbell(struct cdnsp_device *pdev, 2783d829045SPawel Laszczak struct cdnsp_ep *pep, 2793d829045SPawel Laszczak unsigned int stream_id) 2803d829045SPawel Laszczak { 2813d829045SPawel Laszczak __le32 __iomem *reg_addr = &pdev->dba->ep_db; 2823d829045SPawel Laszczak unsigned int ep_state = pep->ep_state; 2833d829045SPawel Laszczak unsigned int db_value; 2843d829045SPawel Laszczak 2853d829045SPawel Laszczak /* 2863d829045SPawel Laszczak * Don't ring the doorbell for this endpoint if endpoint is halted or 2873d829045SPawel Laszczak * disabled. 2883d829045SPawel Laszczak */ 2893d829045SPawel Laszczak if (ep_state & EP_HALTED || !(ep_state & EP_ENABLED)) 2903d829045SPawel Laszczak return false; 2913d829045SPawel Laszczak 2923d829045SPawel Laszczak /* For stream capable endpoints driver can ring doorbell only twice. */ 2933d829045SPawel Laszczak if (pep->ep_state & EP_HAS_STREAMS) { 2943d829045SPawel Laszczak if (pep->stream_info.drbls_count >= 2) 2953d829045SPawel Laszczak return false; 2963d829045SPawel Laszczak 2973d829045SPawel Laszczak pep->stream_info.drbls_count++; 2983d829045SPawel Laszczak } 2993d829045SPawel Laszczak 3003d829045SPawel Laszczak pep->ep_state &= ~EP_STOPPED; 3013d829045SPawel Laszczak 3023d829045SPawel Laszczak if (pep->idx == 0 && pdev->ep0_stage == CDNSP_DATA_STAGE && 3033d829045SPawel Laszczak !pdev->ep0_expect_in) 3043d829045SPawel Laszczak db_value = DB_VALUE_EP0_OUT(pep->idx, stream_id); 3053d829045SPawel Laszczak else 3063d829045SPawel Laszczak db_value = DB_VALUE(pep->idx, stream_id); 3073d829045SPawel Laszczak 308*118b2a32SPawel Laszczak trace_cdnsp_tr_drbl(pep, stream_id); 309*118b2a32SPawel Laszczak 3103d829045SPawel Laszczak writel(db_value, reg_addr); 3113d829045SPawel Laszczak 3123d829045SPawel Laszczak cdnsp_force_l0_go(pdev); 3133d829045SPawel Laszczak 3143d829045SPawel Laszczak /* Doorbell was set. */ 3153d829045SPawel Laszczak return true; 3163d829045SPawel Laszczak } 3173d829045SPawel Laszczak 3183d829045SPawel Laszczak /* 3193d829045SPawel Laszczak * Get the right ring for the given pep and stream_id. 3203d829045SPawel Laszczak * If the endpoint supports streams, boundary check the USB request's stream ID. 3213d829045SPawel Laszczak * If the endpoint doesn't support streams, return the singular endpoint ring. 3223d829045SPawel Laszczak */ 3233d829045SPawel Laszczak static struct cdnsp_ring *cdnsp_get_transfer_ring(struct cdnsp_device *pdev, 3243d829045SPawel Laszczak struct cdnsp_ep *pep, 3253d829045SPawel Laszczak unsigned int stream_id) 3263d829045SPawel Laszczak { 3273d829045SPawel Laszczak if (!(pep->ep_state & EP_HAS_STREAMS)) 3283d829045SPawel Laszczak return pep->ring; 3293d829045SPawel Laszczak 3303d829045SPawel Laszczak if (stream_id == 0 || stream_id >= pep->stream_info.num_streams) { 3313d829045SPawel Laszczak dev_err(pdev->dev, "ERR: %s ring doesn't exist for SID: %d.\n", 3323d829045SPawel Laszczak pep->name, stream_id); 3333d829045SPawel Laszczak return NULL; 3343d829045SPawel Laszczak } 3353d829045SPawel Laszczak 3363d829045SPawel Laszczak return pep->stream_info.stream_rings[stream_id]; 3373d829045SPawel Laszczak } 3383d829045SPawel Laszczak 3393d829045SPawel Laszczak static struct cdnsp_ring * 3403d829045SPawel Laszczak cdnsp_request_to_transfer_ring(struct cdnsp_device *pdev, 3413d829045SPawel Laszczak struct cdnsp_request *preq) 3423d829045SPawel Laszczak { 3433d829045SPawel Laszczak return cdnsp_get_transfer_ring(pdev, preq->pep, 3443d829045SPawel Laszczak preq->request.stream_id); 3453d829045SPawel Laszczak } 3463d829045SPawel Laszczak 3473d829045SPawel Laszczak /* Ring the doorbell for any rings with pending requests. */ 3483d829045SPawel Laszczak void cdnsp_ring_doorbell_for_active_rings(struct cdnsp_device *pdev, 3493d829045SPawel Laszczak struct cdnsp_ep *pep) 3503d829045SPawel Laszczak { 3513d829045SPawel Laszczak struct cdnsp_stream_info *stream_info; 3523d829045SPawel Laszczak unsigned int stream_id; 3533d829045SPawel Laszczak int ret; 3543d829045SPawel Laszczak 3553d829045SPawel Laszczak if (pep->ep_state & EP_DIS_IN_RROGRESS) 3563d829045SPawel Laszczak return; 3573d829045SPawel Laszczak 3583d829045SPawel Laszczak /* A ring has pending Request if its TD list is not empty. */ 3593d829045SPawel Laszczak if (!(pep->ep_state & EP_HAS_STREAMS) && pep->number) { 3603d829045SPawel Laszczak if (pep->ring && !list_empty(&pep->ring->td_list)) 3613d829045SPawel Laszczak cdnsp_ring_ep_doorbell(pdev, pep, 0); 3623d829045SPawel Laszczak return; 3633d829045SPawel Laszczak } 3643d829045SPawel Laszczak 3653d829045SPawel Laszczak stream_info = &pep->stream_info; 3663d829045SPawel Laszczak 3673d829045SPawel Laszczak for (stream_id = 1; stream_id < stream_info->num_streams; stream_id++) { 3683d829045SPawel Laszczak struct cdnsp_td *td, *td_temp; 3693d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 3703d829045SPawel Laszczak 3713d829045SPawel Laszczak if (stream_info->drbls_count >= 2) 3723d829045SPawel Laszczak return; 3733d829045SPawel Laszczak 3743d829045SPawel Laszczak ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id); 3753d829045SPawel Laszczak if (!ep_ring) 3763d829045SPawel Laszczak continue; 3773d829045SPawel Laszczak 3783d829045SPawel Laszczak if (!ep_ring->stream_active || ep_ring->stream_rejected) 3793d829045SPawel Laszczak continue; 3803d829045SPawel Laszczak 3813d829045SPawel Laszczak list_for_each_entry_safe(td, td_temp, &ep_ring->td_list, 3823d829045SPawel Laszczak td_list) { 3833d829045SPawel Laszczak if (td->drbl) 3843d829045SPawel Laszczak continue; 3853d829045SPawel Laszczak 3863d829045SPawel Laszczak ret = cdnsp_ring_ep_doorbell(pdev, pep, stream_id); 3873d829045SPawel Laszczak if (ret) 3883d829045SPawel Laszczak td->drbl = 1; 3893d829045SPawel Laszczak } 3903d829045SPawel Laszczak } 3913d829045SPawel Laszczak } 3923d829045SPawel Laszczak 3933d829045SPawel Laszczak /* 3943d829045SPawel Laszczak * Get the hw dequeue pointer controller stopped on, either directly from the 3953d829045SPawel Laszczak * endpoint context, or if streams are in use from the stream context. 3963d829045SPawel Laszczak * The returned hw_dequeue contains the lowest four bits with cycle state 3973d829045SPawel Laszczak * and possible stream context type. 3983d829045SPawel Laszczak */ 3993d829045SPawel Laszczak static u64 cdnsp_get_hw_deq(struct cdnsp_device *pdev, 4003d829045SPawel Laszczak unsigned int ep_index, 4013d829045SPawel Laszczak unsigned int stream_id) 4023d829045SPawel Laszczak { 4033d829045SPawel Laszczak struct cdnsp_stream_ctx *st_ctx; 4043d829045SPawel Laszczak struct cdnsp_ep *pep; 4053d829045SPawel Laszczak 4063d829045SPawel Laszczak pep = &pdev->eps[stream_id]; 4073d829045SPawel Laszczak 4083d829045SPawel Laszczak if (pep->ep_state & EP_HAS_STREAMS) { 4093d829045SPawel Laszczak st_ctx = &pep->stream_info.stream_ctx_array[stream_id]; 4103d829045SPawel Laszczak return le64_to_cpu(st_ctx->stream_ring); 4113d829045SPawel Laszczak } 4123d829045SPawel Laszczak 4133d829045SPawel Laszczak return le64_to_cpu(pep->out_ctx->deq); 4143d829045SPawel Laszczak } 4153d829045SPawel Laszczak 4163d829045SPawel Laszczak /* 4173d829045SPawel Laszczak * Move the controller endpoint ring dequeue pointer past cur_td. 4183d829045SPawel Laszczak * Record the new state of the controller endpoint ring dequeue segment, 4193d829045SPawel Laszczak * dequeue pointer, and new consumer cycle state in state. 4203d829045SPawel Laszczak * Update internal representation of the ring's dequeue pointer. 4213d829045SPawel Laszczak * 4223d829045SPawel Laszczak * We do this in three jumps: 4233d829045SPawel Laszczak * - First we update our new ring state to be the same as when the 4243d829045SPawel Laszczak * controller stopped. 4253d829045SPawel Laszczak * - Then we traverse the ring to find the segment that contains 4263d829045SPawel Laszczak * the last TRB in the TD. We toggle the controller new cycle state 4273d829045SPawel Laszczak * when we pass any link TRBs with the toggle cycle bit set. 4283d829045SPawel Laszczak * - Finally we move the dequeue state one TRB further, toggling the cycle bit 4293d829045SPawel Laszczak * if we've moved it past a link TRB with the toggle cycle bit set. 4303d829045SPawel Laszczak */ 4313d829045SPawel Laszczak static void cdnsp_find_new_dequeue_state(struct cdnsp_device *pdev, 4323d829045SPawel Laszczak struct cdnsp_ep *pep, 4333d829045SPawel Laszczak unsigned int stream_id, 4343d829045SPawel Laszczak struct cdnsp_td *cur_td, 4353d829045SPawel Laszczak struct cdnsp_dequeue_state *state) 4363d829045SPawel Laszczak { 4373d829045SPawel Laszczak bool td_last_trb_found = false; 4383d829045SPawel Laszczak struct cdnsp_segment *new_seg; 4393d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 4403d829045SPawel Laszczak union cdnsp_trb *new_deq; 4413d829045SPawel Laszczak bool cycle_found = false; 4423d829045SPawel Laszczak u64 hw_dequeue; 4433d829045SPawel Laszczak 4443d829045SPawel Laszczak ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id); 4453d829045SPawel Laszczak if (!ep_ring) 4463d829045SPawel Laszczak return; 4473d829045SPawel Laszczak 4483d829045SPawel Laszczak /* 4493d829045SPawel Laszczak * Dig out the cycle state saved by the controller during the 4503d829045SPawel Laszczak * stop endpoint command. 4513d829045SPawel Laszczak */ 4523d829045SPawel Laszczak hw_dequeue = cdnsp_get_hw_deq(pdev, pep->idx, stream_id); 4533d829045SPawel Laszczak new_seg = ep_ring->deq_seg; 4543d829045SPawel Laszczak new_deq = ep_ring->dequeue; 4553d829045SPawel Laszczak state->new_cycle_state = hw_dequeue & 0x1; 4563d829045SPawel Laszczak state->stream_id = stream_id; 4573d829045SPawel Laszczak 4583d829045SPawel Laszczak /* 4593d829045SPawel Laszczak * We want to find the pointer, segment and cycle state of the new trb 4603d829045SPawel Laszczak * (the one after current TD's last_trb). We know the cycle state at 4613d829045SPawel Laszczak * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are 4623d829045SPawel Laszczak * found. 4633d829045SPawel Laszczak */ 4643d829045SPawel Laszczak do { 4653d829045SPawel Laszczak if (!cycle_found && cdnsp_trb_virt_to_dma(new_seg, new_deq) 4663d829045SPawel Laszczak == (dma_addr_t)(hw_dequeue & ~0xf)) { 4673d829045SPawel Laszczak cycle_found = true; 4683d829045SPawel Laszczak 4693d829045SPawel Laszczak if (td_last_trb_found) 4703d829045SPawel Laszczak break; 4713d829045SPawel Laszczak } 4723d829045SPawel Laszczak 4733d829045SPawel Laszczak if (new_deq == cur_td->last_trb) 4743d829045SPawel Laszczak td_last_trb_found = true; 4753d829045SPawel Laszczak 4763d829045SPawel Laszczak if (cycle_found && cdnsp_trb_is_link(new_deq) && 4773d829045SPawel Laszczak cdnsp_link_trb_toggles_cycle(new_deq)) 4783d829045SPawel Laszczak state->new_cycle_state ^= 0x1; 4793d829045SPawel Laszczak 4803d829045SPawel Laszczak cdnsp_next_trb(pdev, ep_ring, &new_seg, &new_deq); 4813d829045SPawel Laszczak 4823d829045SPawel Laszczak /* Search wrapped around, bail out. */ 4833d829045SPawel Laszczak if (new_deq == pep->ring->dequeue) { 4843d829045SPawel Laszczak dev_err(pdev->dev, 4853d829045SPawel Laszczak "Error: Failed finding new dequeue state\n"); 4863d829045SPawel Laszczak state->new_deq_seg = NULL; 4873d829045SPawel Laszczak state->new_deq_ptr = NULL; 4883d829045SPawel Laszczak return; 4893d829045SPawel Laszczak } 4903d829045SPawel Laszczak 4913d829045SPawel Laszczak } while (!cycle_found || !td_last_trb_found); 4923d829045SPawel Laszczak 4933d829045SPawel Laszczak state->new_deq_seg = new_seg; 4943d829045SPawel Laszczak state->new_deq_ptr = new_deq; 495*118b2a32SPawel Laszczak 496*118b2a32SPawel Laszczak trace_cdnsp_new_deq_state(state); 4973d829045SPawel Laszczak } 4983d829045SPawel Laszczak 4993d829045SPawel Laszczak /* 5003d829045SPawel Laszczak * flip_cycle means flip the cycle bit of all but the first and last TRB. 5013d829045SPawel Laszczak * (The last TRB actually points to the ring enqueue pointer, which is not part 5023d829045SPawel Laszczak * of this TD.) This is used to remove partially enqueued isoc TDs from a ring. 5033d829045SPawel Laszczak */ 5043d829045SPawel Laszczak static void cdnsp_td_to_noop(struct cdnsp_device *pdev, 5053d829045SPawel Laszczak struct cdnsp_ring *ep_ring, 5063d829045SPawel Laszczak struct cdnsp_td *td, 5073d829045SPawel Laszczak bool flip_cycle) 5083d829045SPawel Laszczak { 5093d829045SPawel Laszczak struct cdnsp_segment *seg = td->start_seg; 5103d829045SPawel Laszczak union cdnsp_trb *trb = td->first_trb; 5113d829045SPawel Laszczak 5123d829045SPawel Laszczak while (1) { 5133d829045SPawel Laszczak cdnsp_trb_to_noop(trb, TRB_TR_NOOP); 5143d829045SPawel Laszczak 5153d829045SPawel Laszczak /* flip cycle if asked to */ 5163d829045SPawel Laszczak if (flip_cycle && trb != td->first_trb && trb != td->last_trb) 5173d829045SPawel Laszczak trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE); 5183d829045SPawel Laszczak 5193d829045SPawel Laszczak if (trb == td->last_trb) 5203d829045SPawel Laszczak break; 5213d829045SPawel Laszczak 5223d829045SPawel Laszczak cdnsp_next_trb(pdev, ep_ring, &seg, &trb); 5233d829045SPawel Laszczak } 5243d829045SPawel Laszczak } 5253d829045SPawel Laszczak 5263d829045SPawel Laszczak /* 5273d829045SPawel Laszczak * This TD is defined by the TRBs starting at start_trb in start_seg and ending 5283d829045SPawel Laszczak * at end_trb, which may be in another segment. If the suspect DMA address is a 5293d829045SPawel Laszczak * TRB in this TD, this function returns that TRB's segment. Otherwise it 5303d829045SPawel Laszczak * returns 0. 5313d829045SPawel Laszczak */ 5323d829045SPawel Laszczak static struct cdnsp_segment *cdnsp_trb_in_td(struct cdnsp_device *pdev, 5333d829045SPawel Laszczak struct cdnsp_segment *start_seg, 5343d829045SPawel Laszczak union cdnsp_trb *start_trb, 5353d829045SPawel Laszczak union cdnsp_trb *end_trb, 5363d829045SPawel Laszczak dma_addr_t suspect_dma) 5373d829045SPawel Laszczak { 5383d829045SPawel Laszczak struct cdnsp_segment *cur_seg; 5393d829045SPawel Laszczak union cdnsp_trb *temp_trb; 5403d829045SPawel Laszczak dma_addr_t end_seg_dma; 5413d829045SPawel Laszczak dma_addr_t end_trb_dma; 5423d829045SPawel Laszczak dma_addr_t start_dma; 5433d829045SPawel Laszczak 5443d829045SPawel Laszczak start_dma = cdnsp_trb_virt_to_dma(start_seg, start_trb); 5453d829045SPawel Laszczak cur_seg = start_seg; 5463d829045SPawel Laszczak 5473d829045SPawel Laszczak do { 5483d829045SPawel Laszczak if (start_dma == 0) 5493d829045SPawel Laszczak return NULL; 5503d829045SPawel Laszczak 5513d829045SPawel Laszczak temp_trb = &cur_seg->trbs[TRBS_PER_SEGMENT - 1]; 5523d829045SPawel Laszczak /* We may get an event for a Link TRB in the middle of a TD */ 5533d829045SPawel Laszczak end_seg_dma = cdnsp_trb_virt_to_dma(cur_seg, temp_trb); 5543d829045SPawel Laszczak /* If the end TRB isn't in this segment, this is set to 0 */ 5553d829045SPawel Laszczak end_trb_dma = cdnsp_trb_virt_to_dma(cur_seg, end_trb); 5563d829045SPawel Laszczak 557*118b2a32SPawel Laszczak trace_cdnsp_looking_trb_in_td(suspect_dma, start_dma, 558*118b2a32SPawel Laszczak end_trb_dma, cur_seg->dma, 559*118b2a32SPawel Laszczak end_seg_dma); 560*118b2a32SPawel Laszczak 5613d829045SPawel Laszczak if (end_trb_dma > 0) { 5623d829045SPawel Laszczak /* 5633d829045SPawel Laszczak * The end TRB is in this segment, so suspect should 5643d829045SPawel Laszczak * be here 5653d829045SPawel Laszczak */ 5663d829045SPawel Laszczak if (start_dma <= end_trb_dma) { 5673d829045SPawel Laszczak if (suspect_dma >= start_dma && 5683d829045SPawel Laszczak suspect_dma <= end_trb_dma) { 5693d829045SPawel Laszczak return cur_seg; 5703d829045SPawel Laszczak } 5713d829045SPawel Laszczak } else { 5723d829045SPawel Laszczak /* 5733d829045SPawel Laszczak * Case for one segment with a 5743d829045SPawel Laszczak * TD wrapped around to the top 5753d829045SPawel Laszczak */ 5763d829045SPawel Laszczak if ((suspect_dma >= start_dma && 5773d829045SPawel Laszczak suspect_dma <= end_seg_dma) || 5783d829045SPawel Laszczak (suspect_dma >= cur_seg->dma && 5793d829045SPawel Laszczak suspect_dma <= end_trb_dma)) { 5803d829045SPawel Laszczak return cur_seg; 5813d829045SPawel Laszczak } 5823d829045SPawel Laszczak } 5833d829045SPawel Laszczak 5843d829045SPawel Laszczak return NULL; 5853d829045SPawel Laszczak } 5863d829045SPawel Laszczak 5873d829045SPawel Laszczak /* Might still be somewhere in this segment */ 5883d829045SPawel Laszczak if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma) 5893d829045SPawel Laszczak return cur_seg; 5903d829045SPawel Laszczak 5913d829045SPawel Laszczak cur_seg = cur_seg->next; 5923d829045SPawel Laszczak start_dma = cdnsp_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]); 5933d829045SPawel Laszczak } while (cur_seg != start_seg); 5943d829045SPawel Laszczak 5953d829045SPawel Laszczak return NULL; 5963d829045SPawel Laszczak } 5973d829045SPawel Laszczak 5983d829045SPawel Laszczak static void cdnsp_unmap_td_bounce_buffer(struct cdnsp_device *pdev, 5993d829045SPawel Laszczak struct cdnsp_ring *ring, 6003d829045SPawel Laszczak struct cdnsp_td *td) 6013d829045SPawel Laszczak { 6023d829045SPawel Laszczak struct cdnsp_segment *seg = td->bounce_seg; 6033d829045SPawel Laszczak struct cdnsp_request *preq; 6043d829045SPawel Laszczak size_t len; 6053d829045SPawel Laszczak 6063d829045SPawel Laszczak if (!seg) 6073d829045SPawel Laszczak return; 6083d829045SPawel Laszczak 6093d829045SPawel Laszczak preq = td->preq; 6103d829045SPawel Laszczak 611*118b2a32SPawel Laszczak trace_cdnsp_bounce_unmap(td->preq, seg->bounce_len, seg->bounce_offs, 612*118b2a32SPawel Laszczak seg->bounce_dma, 0); 613*118b2a32SPawel Laszczak 6143d829045SPawel Laszczak if (!preq->direction) { 6153d829045SPawel Laszczak dma_unmap_single(pdev->dev, seg->bounce_dma, 6163d829045SPawel Laszczak ring->bounce_buf_len, DMA_TO_DEVICE); 6173d829045SPawel Laszczak return; 6183d829045SPawel Laszczak } 6193d829045SPawel Laszczak 6203d829045SPawel Laszczak dma_unmap_single(pdev->dev, seg->bounce_dma, ring->bounce_buf_len, 6213d829045SPawel Laszczak DMA_FROM_DEVICE); 6223d829045SPawel Laszczak 6233d829045SPawel Laszczak /* For in transfers we need to copy the data from bounce to sg */ 6243d829045SPawel Laszczak len = sg_pcopy_from_buffer(preq->request.sg, preq->request.num_sgs, 6253d829045SPawel Laszczak seg->bounce_buf, seg->bounce_len, 6263d829045SPawel Laszczak seg->bounce_offs); 6273d829045SPawel Laszczak if (len != seg->bounce_len) 6283d829045SPawel Laszczak dev_warn(pdev->dev, "WARN Wrong bounce buffer read length: %zu != %d\n", 6293d829045SPawel Laszczak len, seg->bounce_len); 6303d829045SPawel Laszczak 6313d829045SPawel Laszczak seg->bounce_len = 0; 6323d829045SPawel Laszczak seg->bounce_offs = 0; 6333d829045SPawel Laszczak } 6343d829045SPawel Laszczak 6353d829045SPawel Laszczak static int cdnsp_cmd_set_deq(struct cdnsp_device *pdev, 6363d829045SPawel Laszczak struct cdnsp_ep *pep, 6373d829045SPawel Laszczak struct cdnsp_dequeue_state *deq_state) 6383d829045SPawel Laszczak { 6393d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 6403d829045SPawel Laszczak int ret; 6413d829045SPawel Laszczak 6423d829045SPawel Laszczak if (!deq_state->new_deq_ptr || !deq_state->new_deq_seg) { 6433d829045SPawel Laszczak cdnsp_ring_doorbell_for_active_rings(pdev, pep); 6443d829045SPawel Laszczak return 0; 6453d829045SPawel Laszczak } 6463d829045SPawel Laszczak 6473d829045SPawel Laszczak cdnsp_queue_new_dequeue_state(pdev, pep, deq_state); 6483d829045SPawel Laszczak cdnsp_ring_cmd_db(pdev); 6493d829045SPawel Laszczak ret = cdnsp_wait_for_cmd_compl(pdev); 6503d829045SPawel Laszczak 651*118b2a32SPawel Laszczak trace_cdnsp_handle_cmd_set_deq(cdnsp_get_slot_ctx(&pdev->out_ctx)); 652*118b2a32SPawel Laszczak trace_cdnsp_handle_cmd_set_deq_ep(pep->out_ctx); 653*118b2a32SPawel Laszczak 6543d829045SPawel Laszczak /* 6553d829045SPawel Laszczak * Update the ring's dequeue segment and dequeue pointer 6563d829045SPawel Laszczak * to reflect the new position. 6573d829045SPawel Laszczak */ 6583d829045SPawel Laszczak ep_ring = cdnsp_get_transfer_ring(pdev, pep, deq_state->stream_id); 6593d829045SPawel Laszczak 6603d829045SPawel Laszczak if (cdnsp_trb_is_link(ep_ring->dequeue)) { 6613d829045SPawel Laszczak ep_ring->deq_seg = ep_ring->deq_seg->next; 6623d829045SPawel Laszczak ep_ring->dequeue = ep_ring->deq_seg->trbs; 6633d829045SPawel Laszczak } 6643d829045SPawel Laszczak 6653d829045SPawel Laszczak while (ep_ring->dequeue != deq_state->new_deq_ptr) { 6663d829045SPawel Laszczak ep_ring->num_trbs_free++; 6673d829045SPawel Laszczak ep_ring->dequeue++; 6683d829045SPawel Laszczak 6693d829045SPawel Laszczak if (cdnsp_trb_is_link(ep_ring->dequeue)) { 6703d829045SPawel Laszczak if (ep_ring->dequeue == deq_state->new_deq_ptr) 6713d829045SPawel Laszczak break; 6723d829045SPawel Laszczak 6733d829045SPawel Laszczak ep_ring->deq_seg = ep_ring->deq_seg->next; 6743d829045SPawel Laszczak ep_ring->dequeue = ep_ring->deq_seg->trbs; 6753d829045SPawel Laszczak } 6763d829045SPawel Laszczak } 6773d829045SPawel Laszczak 6783d829045SPawel Laszczak /* 6793d829045SPawel Laszczak * Probably there was TIMEOUT during handling Set Dequeue Pointer 6803d829045SPawel Laszczak * command. It's critical error and controller will be stopped. 6813d829045SPawel Laszczak */ 6823d829045SPawel Laszczak if (ret) 6833d829045SPawel Laszczak return -ESHUTDOWN; 6843d829045SPawel Laszczak 6853d829045SPawel Laszczak /* Restart any rings with pending requests */ 6863d829045SPawel Laszczak cdnsp_ring_doorbell_for_active_rings(pdev, pep); 6873d829045SPawel Laszczak 6883d829045SPawel Laszczak return 0; 6893d829045SPawel Laszczak } 6903d829045SPawel Laszczak 6913d829045SPawel Laszczak int cdnsp_remove_request(struct cdnsp_device *pdev, 6923d829045SPawel Laszczak struct cdnsp_request *preq, 6933d829045SPawel Laszczak struct cdnsp_ep *pep) 6943d829045SPawel Laszczak { 6953d829045SPawel Laszczak struct cdnsp_dequeue_state deq_state; 6963d829045SPawel Laszczak struct cdnsp_td *cur_td = NULL; 6973d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 6983d829045SPawel Laszczak struct cdnsp_segment *seg; 6993d829045SPawel Laszczak int status = -ECONNRESET; 7003d829045SPawel Laszczak int ret = 0; 7013d829045SPawel Laszczak u64 hw_deq; 7023d829045SPawel Laszczak 7033d829045SPawel Laszczak memset(&deq_state, 0, sizeof(deq_state)); 7043d829045SPawel Laszczak 705*118b2a32SPawel Laszczak trace_cdnsp_remove_request(pep->out_ctx); 706*118b2a32SPawel Laszczak trace_cdnsp_remove_request_td(preq); 707*118b2a32SPawel Laszczak 7083d829045SPawel Laszczak cur_td = &preq->td; 7093d829045SPawel Laszczak ep_ring = cdnsp_request_to_transfer_ring(pdev, preq); 7103d829045SPawel Laszczak 7113d829045SPawel Laszczak /* 7123d829045SPawel Laszczak * If we stopped on the TD we need to cancel, then we have to 7133d829045SPawel Laszczak * move the controller endpoint ring dequeue pointer past 7143d829045SPawel Laszczak * this TD. 7153d829045SPawel Laszczak */ 7163d829045SPawel Laszczak hw_deq = cdnsp_get_hw_deq(pdev, pep->idx, preq->request.stream_id); 7173d829045SPawel Laszczak hw_deq &= ~0xf; 7183d829045SPawel Laszczak 7193d829045SPawel Laszczak seg = cdnsp_trb_in_td(pdev, cur_td->start_seg, cur_td->first_trb, 7203d829045SPawel Laszczak cur_td->last_trb, hw_deq); 7213d829045SPawel Laszczak 7223d829045SPawel Laszczak if (seg && (pep->ep_state & EP_ENABLED)) 7233d829045SPawel Laszczak cdnsp_find_new_dequeue_state(pdev, pep, preq->request.stream_id, 7243d829045SPawel Laszczak cur_td, &deq_state); 7253d829045SPawel Laszczak else 7263d829045SPawel Laszczak cdnsp_td_to_noop(pdev, ep_ring, cur_td, false); 7273d829045SPawel Laszczak 7283d829045SPawel Laszczak /* 7293d829045SPawel Laszczak * The event handler won't see a completion for this TD anymore, 7303d829045SPawel Laszczak * so remove it from the endpoint ring's TD list. 7313d829045SPawel Laszczak */ 7323d829045SPawel Laszczak list_del_init(&cur_td->td_list); 7333d829045SPawel Laszczak ep_ring->num_tds--; 7343d829045SPawel Laszczak pep->stream_info.td_count--; 7353d829045SPawel Laszczak 7363d829045SPawel Laszczak /* 7373d829045SPawel Laszczak * During disconnecting all endpoint will be disabled so we don't 7383d829045SPawel Laszczak * have to worry about updating dequeue pointer. 7393d829045SPawel Laszczak */ 7403d829045SPawel Laszczak if (pdev->cdnsp_state & CDNSP_STATE_DISCONNECT_PENDING) { 7413d829045SPawel Laszczak status = -ESHUTDOWN; 7423d829045SPawel Laszczak ret = cdnsp_cmd_set_deq(pdev, pep, &deq_state); 7433d829045SPawel Laszczak } 7443d829045SPawel Laszczak 7453d829045SPawel Laszczak cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, cur_td); 7463d829045SPawel Laszczak cdnsp_gadget_giveback(pep, cur_td->preq, status); 7473d829045SPawel Laszczak 7483d829045SPawel Laszczak return ret; 7493d829045SPawel Laszczak } 7503d829045SPawel Laszczak 7513d829045SPawel Laszczak static int cdnsp_update_port_id(struct cdnsp_device *pdev, u32 port_id) 7523d829045SPawel Laszczak { 7533d829045SPawel Laszczak struct cdnsp_port *port = pdev->active_port; 7543d829045SPawel Laszczak u8 old_port = 0; 7553d829045SPawel Laszczak 7563d829045SPawel Laszczak if (port && port->port_num == port_id) 7573d829045SPawel Laszczak return 0; 7583d829045SPawel Laszczak 7593d829045SPawel Laszczak if (port) 7603d829045SPawel Laszczak old_port = port->port_num; 7613d829045SPawel Laszczak 7623d829045SPawel Laszczak if (port_id == pdev->usb2_port.port_num) { 7633d829045SPawel Laszczak port = &pdev->usb2_port; 7643d829045SPawel Laszczak } else if (port_id == pdev->usb3_port.port_num) { 7653d829045SPawel Laszczak port = &pdev->usb3_port; 7663d829045SPawel Laszczak } else { 7673d829045SPawel Laszczak dev_err(pdev->dev, "Port event with invalid port ID %d\n", 7683d829045SPawel Laszczak port_id); 7693d829045SPawel Laszczak return -EINVAL; 7703d829045SPawel Laszczak } 7713d829045SPawel Laszczak 7723d829045SPawel Laszczak if (port_id != old_port) { 7733d829045SPawel Laszczak cdnsp_disable_slot(pdev); 7743d829045SPawel Laszczak pdev->active_port = port; 7753d829045SPawel Laszczak cdnsp_enable_slot(pdev); 7763d829045SPawel Laszczak } 7773d829045SPawel Laszczak 7783d829045SPawel Laszczak if (port_id == pdev->usb2_port.port_num) 7793d829045SPawel Laszczak cdnsp_set_usb2_hardware_lpm(pdev, NULL, 1); 7803d829045SPawel Laszczak else 7813d829045SPawel Laszczak writel(PORT_U1_TIMEOUT(1) | PORT_U2_TIMEOUT(1), 7823d829045SPawel Laszczak &pdev->usb3_port.regs->portpmsc); 7833d829045SPawel Laszczak 7843d829045SPawel Laszczak return 0; 7853d829045SPawel Laszczak } 7863d829045SPawel Laszczak 7873d829045SPawel Laszczak static void cdnsp_handle_port_status(struct cdnsp_device *pdev, 7883d829045SPawel Laszczak union cdnsp_trb *event) 7893d829045SPawel Laszczak { 7903d829045SPawel Laszczak struct cdnsp_port_regs __iomem *port_regs; 7913d829045SPawel Laszczak u32 portsc, cmd_regs; 7923d829045SPawel Laszczak bool port2 = false; 7933d829045SPawel Laszczak u32 link_state; 7943d829045SPawel Laszczak u32 port_id; 7953d829045SPawel Laszczak 7963d829045SPawel Laszczak /* Port status change events always have a successful completion code */ 7973d829045SPawel Laszczak if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS) 7983d829045SPawel Laszczak dev_err(pdev->dev, "ERR: incorrect PSC event\n"); 7993d829045SPawel Laszczak 8003d829045SPawel Laszczak port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0])); 8013d829045SPawel Laszczak 8023d829045SPawel Laszczak if (cdnsp_update_port_id(pdev, port_id)) 8033d829045SPawel Laszczak goto cleanup; 8043d829045SPawel Laszczak 8053d829045SPawel Laszczak port_regs = pdev->active_port->regs; 8063d829045SPawel Laszczak 8073d829045SPawel Laszczak if (port_id == pdev->usb2_port.port_num) 8083d829045SPawel Laszczak port2 = true; 8093d829045SPawel Laszczak 8103d829045SPawel Laszczak new_event: 8113d829045SPawel Laszczak portsc = readl(&port_regs->portsc); 8123d829045SPawel Laszczak writel(cdnsp_port_state_to_neutral(portsc) | 8133d829045SPawel Laszczak (portsc & PORT_CHANGE_BITS), &port_regs->portsc); 8143d829045SPawel Laszczak 815*118b2a32SPawel Laszczak trace_cdnsp_handle_port_status(pdev->active_port->port_num, portsc); 816*118b2a32SPawel Laszczak 8173d829045SPawel Laszczak pdev->gadget.speed = cdnsp_port_speed(portsc); 8183d829045SPawel Laszczak link_state = portsc & PORT_PLS_MASK; 8193d829045SPawel Laszczak 8203d829045SPawel Laszczak /* Port Link State change detected. */ 8213d829045SPawel Laszczak if ((portsc & PORT_PLC)) { 8223d829045SPawel Laszczak if (!(pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) && 8233d829045SPawel Laszczak link_state == XDEV_RESUME) { 8243d829045SPawel Laszczak cmd_regs = readl(&pdev->op_regs->command); 8253d829045SPawel Laszczak if (!(cmd_regs & CMD_R_S)) 8263d829045SPawel Laszczak goto cleanup; 8273d829045SPawel Laszczak 8283d829045SPawel Laszczak if (DEV_SUPERSPEED_ANY(portsc)) { 8293d829045SPawel Laszczak cdnsp_set_link_state(pdev, &port_regs->portsc, 8303d829045SPawel Laszczak XDEV_U0); 8313d829045SPawel Laszczak 8323d829045SPawel Laszczak cdnsp_resume_gadget(pdev); 8333d829045SPawel Laszczak } 8343d829045SPawel Laszczak } 8353d829045SPawel Laszczak 8363d829045SPawel Laszczak if ((pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) && 8373d829045SPawel Laszczak link_state == XDEV_U0) { 8383d829045SPawel Laszczak pdev->cdnsp_state &= ~CDNSP_WAKEUP_PENDING; 8393d829045SPawel Laszczak 8403d829045SPawel Laszczak cdnsp_force_header_wakeup(pdev, 1); 8413d829045SPawel Laszczak cdnsp_ring_cmd_db(pdev); 8423d829045SPawel Laszczak cdnsp_wait_for_cmd_compl(pdev); 8433d829045SPawel Laszczak } 8443d829045SPawel Laszczak 8453d829045SPawel Laszczak if (link_state == XDEV_U0 && pdev->link_state == XDEV_U3 && 8463d829045SPawel Laszczak !DEV_SUPERSPEED_ANY(portsc)) 8473d829045SPawel Laszczak cdnsp_resume_gadget(pdev); 8483d829045SPawel Laszczak 8493d829045SPawel Laszczak if (link_state == XDEV_U3 && pdev->link_state != XDEV_U3) 8503d829045SPawel Laszczak cdnsp_suspend_gadget(pdev); 8513d829045SPawel Laszczak 8523d829045SPawel Laszczak pdev->link_state = link_state; 8533d829045SPawel Laszczak } 8543d829045SPawel Laszczak 8553d829045SPawel Laszczak if (portsc & PORT_CSC) { 8563d829045SPawel Laszczak /* Detach device. */ 8573d829045SPawel Laszczak if (pdev->gadget.connected && !(portsc & PORT_CONNECT)) 8583d829045SPawel Laszczak cdnsp_disconnect_gadget(pdev); 8593d829045SPawel Laszczak 8603d829045SPawel Laszczak /* Attach device. */ 8613d829045SPawel Laszczak if (portsc & PORT_CONNECT) { 8623d829045SPawel Laszczak if (!port2) 8633d829045SPawel Laszczak cdnsp_irq_reset(pdev); 8643d829045SPawel Laszczak 8653d829045SPawel Laszczak usb_gadget_set_state(&pdev->gadget, USB_STATE_ATTACHED); 8663d829045SPawel Laszczak } 8673d829045SPawel Laszczak } 8683d829045SPawel Laszczak 8693d829045SPawel Laszczak /* Port reset. */ 8703d829045SPawel Laszczak if ((portsc & (PORT_RC | PORT_WRC)) && (portsc & PORT_CONNECT)) { 8713d829045SPawel Laszczak cdnsp_irq_reset(pdev); 8723d829045SPawel Laszczak pdev->u1_allowed = 0; 8733d829045SPawel Laszczak pdev->u2_allowed = 0; 8743d829045SPawel Laszczak pdev->may_wakeup = 0; 8753d829045SPawel Laszczak } 8763d829045SPawel Laszczak 8773d829045SPawel Laszczak if (portsc & PORT_CEC) 8783d829045SPawel Laszczak dev_err(pdev->dev, "Port Over Current detected\n"); 8793d829045SPawel Laszczak 8803d829045SPawel Laszczak if (portsc & PORT_CEC) 8813d829045SPawel Laszczak dev_err(pdev->dev, "Port Configure Error detected\n"); 8823d829045SPawel Laszczak 8833d829045SPawel Laszczak if (readl(&port_regs->portsc) & PORT_CHANGE_BITS) 8843d829045SPawel Laszczak goto new_event; 8853d829045SPawel Laszczak 8863d829045SPawel Laszczak cleanup: 8873d829045SPawel Laszczak cdnsp_inc_deq(pdev, pdev->event_ring); 8883d829045SPawel Laszczak } 8893d829045SPawel Laszczak 8903d829045SPawel Laszczak static void cdnsp_td_cleanup(struct cdnsp_device *pdev, 8913d829045SPawel Laszczak struct cdnsp_td *td, 8923d829045SPawel Laszczak struct cdnsp_ring *ep_ring, 8933d829045SPawel Laszczak int *status) 8943d829045SPawel Laszczak { 8953d829045SPawel Laszczak struct cdnsp_request *preq = td->preq; 8963d829045SPawel Laszczak 8973d829045SPawel Laszczak /* if a bounce buffer was used to align this td then unmap it */ 8983d829045SPawel Laszczak cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, td); 8993d829045SPawel Laszczak 9003d829045SPawel Laszczak /* 9013d829045SPawel Laszczak * If the controller said we transferred more data than the buffer 9023d829045SPawel Laszczak * length, Play it safe and say we didn't transfer anything. 9033d829045SPawel Laszczak */ 9043d829045SPawel Laszczak if (preq->request.actual > preq->request.length) { 9053d829045SPawel Laszczak preq->request.actual = 0; 9063d829045SPawel Laszczak *status = 0; 9073d829045SPawel Laszczak } 9083d829045SPawel Laszczak 9093d829045SPawel Laszczak list_del_init(&td->td_list); 9103d829045SPawel Laszczak ep_ring->num_tds--; 9113d829045SPawel Laszczak preq->pep->stream_info.td_count--; 9123d829045SPawel Laszczak 9133d829045SPawel Laszczak cdnsp_gadget_giveback(preq->pep, preq, *status); 9143d829045SPawel Laszczak } 9153d829045SPawel Laszczak 9163d829045SPawel Laszczak static void cdnsp_finish_td(struct cdnsp_device *pdev, 9173d829045SPawel Laszczak struct cdnsp_td *td, 9183d829045SPawel Laszczak struct cdnsp_transfer_event *event, 9193d829045SPawel Laszczak struct cdnsp_ep *ep, 9203d829045SPawel Laszczak int *status) 9213d829045SPawel Laszczak { 9223d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 9233d829045SPawel Laszczak u32 trb_comp_code; 9243d829045SPawel Laszczak 9253d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); 9263d829045SPawel Laszczak trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 9273d829045SPawel Laszczak 9283d829045SPawel Laszczak if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID || 9293d829045SPawel Laszczak trb_comp_code == COMP_STOPPED || 9303d829045SPawel Laszczak trb_comp_code == COMP_STOPPED_SHORT_PACKET) { 9313d829045SPawel Laszczak /* 9323d829045SPawel Laszczak * The Endpoint Stop Command completion will take care of any 9333d829045SPawel Laszczak * stopped TDs. A stopped TD may be restarted, so don't update 9343d829045SPawel Laszczak * the ring dequeue pointer or take this TD off any lists yet. 9353d829045SPawel Laszczak */ 9363d829045SPawel Laszczak return; 9373d829045SPawel Laszczak } 9383d829045SPawel Laszczak 9393d829045SPawel Laszczak /* Update ring dequeue pointer */ 9403d829045SPawel Laszczak while (ep_ring->dequeue != td->last_trb) 9413d829045SPawel Laszczak cdnsp_inc_deq(pdev, ep_ring); 9423d829045SPawel Laszczak 9433d829045SPawel Laszczak cdnsp_inc_deq(pdev, ep_ring); 9443d829045SPawel Laszczak 9453d829045SPawel Laszczak cdnsp_td_cleanup(pdev, td, ep_ring, status); 9463d829045SPawel Laszczak } 9473d829045SPawel Laszczak 9483d829045SPawel Laszczak /* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */ 9493d829045SPawel Laszczak static int cdnsp_sum_trb_lengths(struct cdnsp_device *pdev, 9503d829045SPawel Laszczak struct cdnsp_ring *ring, 9513d829045SPawel Laszczak union cdnsp_trb *stop_trb) 9523d829045SPawel Laszczak { 9533d829045SPawel Laszczak struct cdnsp_segment *seg = ring->deq_seg; 9543d829045SPawel Laszczak union cdnsp_trb *trb = ring->dequeue; 9553d829045SPawel Laszczak u32 sum; 9563d829045SPawel Laszczak 9573d829045SPawel Laszczak for (sum = 0; trb != stop_trb; cdnsp_next_trb(pdev, ring, &seg, &trb)) { 9583d829045SPawel Laszczak if (!cdnsp_trb_is_noop(trb) && !cdnsp_trb_is_link(trb)) 9593d829045SPawel Laszczak sum += TRB_LEN(le32_to_cpu(trb->generic.field[2])); 9603d829045SPawel Laszczak } 9613d829045SPawel Laszczak return sum; 9623d829045SPawel Laszczak } 9633d829045SPawel Laszczak 9643d829045SPawel Laszczak static int cdnsp_giveback_first_trb(struct cdnsp_device *pdev, 9653d829045SPawel Laszczak struct cdnsp_ep *pep, 9663d829045SPawel Laszczak unsigned int stream_id, 9673d829045SPawel Laszczak int start_cycle, 9683d829045SPawel Laszczak struct cdnsp_generic_trb *start_trb) 9693d829045SPawel Laszczak { 9703d829045SPawel Laszczak /* 9713d829045SPawel Laszczak * Pass all the TRBs to the hardware at once and make sure this write 9723d829045SPawel Laszczak * isn't reordered. 9733d829045SPawel Laszczak */ 9743d829045SPawel Laszczak wmb(); 9753d829045SPawel Laszczak 9763d829045SPawel Laszczak if (start_cycle) 9773d829045SPawel Laszczak start_trb->field[3] |= cpu_to_le32(start_cycle); 9783d829045SPawel Laszczak else 9793d829045SPawel Laszczak start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE); 9803d829045SPawel Laszczak 9813d829045SPawel Laszczak if ((pep->ep_state & EP_HAS_STREAMS) && 982*118b2a32SPawel Laszczak !pep->stream_info.first_prime_det) { 983*118b2a32SPawel Laszczak trace_cdnsp_wait_for_prime(pep, stream_id); 9843d829045SPawel Laszczak return 0; 985*118b2a32SPawel Laszczak } 9863d829045SPawel Laszczak 9873d829045SPawel Laszczak return cdnsp_ring_ep_doorbell(pdev, pep, stream_id); 9883d829045SPawel Laszczak } 9893d829045SPawel Laszczak 9903d829045SPawel Laszczak /* 9913d829045SPawel Laszczak * Process control tds, update USB request status and actual_length. 9923d829045SPawel Laszczak */ 9933d829045SPawel Laszczak static void cdnsp_process_ctrl_td(struct cdnsp_device *pdev, 9943d829045SPawel Laszczak struct cdnsp_td *td, 9953d829045SPawel Laszczak union cdnsp_trb *event_trb, 9963d829045SPawel Laszczak struct cdnsp_transfer_event *event, 9973d829045SPawel Laszczak struct cdnsp_ep *pep, 9983d829045SPawel Laszczak int *status) 9993d829045SPawel Laszczak { 10003d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 10013d829045SPawel Laszczak u32 remaining; 10023d829045SPawel Laszczak u32 trb_type; 10033d829045SPawel Laszczak 10043d829045SPawel Laszczak trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event_trb->generic.field[3])); 10053d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 10063d829045SPawel Laszczak remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 10073d829045SPawel Laszczak 10083d829045SPawel Laszczak /* 10093d829045SPawel Laszczak * if on data stage then update the actual_length of the USB 10103d829045SPawel Laszczak * request and flag it as set, so it won't be overwritten in the event 10113d829045SPawel Laszczak * for the last TRB. 10123d829045SPawel Laszczak */ 10133d829045SPawel Laszczak if (trb_type == TRB_DATA) { 10143d829045SPawel Laszczak td->request_length_set = true; 10153d829045SPawel Laszczak td->preq->request.actual = td->preq->request.length - remaining; 10163d829045SPawel Laszczak } 10173d829045SPawel Laszczak 10183d829045SPawel Laszczak /* at status stage */ 10193d829045SPawel Laszczak if (!td->request_length_set) 10203d829045SPawel Laszczak td->preq->request.actual = td->preq->request.length; 10213d829045SPawel Laszczak 10223d829045SPawel Laszczak if (pdev->ep0_stage == CDNSP_DATA_STAGE && pep->number == 0 && 10233d829045SPawel Laszczak pdev->three_stage_setup) { 10243d829045SPawel Laszczak td = list_entry(ep_ring->td_list.next, struct cdnsp_td, 10253d829045SPawel Laszczak td_list); 10263d829045SPawel Laszczak pdev->ep0_stage = CDNSP_STATUS_STAGE; 10273d829045SPawel Laszczak 10283d829045SPawel Laszczak cdnsp_giveback_first_trb(pdev, pep, 0, ep_ring->cycle_state, 10293d829045SPawel Laszczak &td->last_trb->generic); 10303d829045SPawel Laszczak return; 10313d829045SPawel Laszczak } 10323d829045SPawel Laszczak 10333d829045SPawel Laszczak cdnsp_finish_td(pdev, td, event, pep, status); 10343d829045SPawel Laszczak } 10353d829045SPawel Laszczak 10363d829045SPawel Laszczak /* 10373d829045SPawel Laszczak * Process isochronous tds, update usb request status and actual_length. 10383d829045SPawel Laszczak */ 10393d829045SPawel Laszczak static void cdnsp_process_isoc_td(struct cdnsp_device *pdev, 10403d829045SPawel Laszczak struct cdnsp_td *td, 10413d829045SPawel Laszczak union cdnsp_trb *ep_trb, 10423d829045SPawel Laszczak struct cdnsp_transfer_event *event, 10433d829045SPawel Laszczak struct cdnsp_ep *pep, 10443d829045SPawel Laszczak int status) 10453d829045SPawel Laszczak { 10463d829045SPawel Laszczak struct cdnsp_request *preq = td->preq; 10473d829045SPawel Laszczak u32 remaining, requested, ep_trb_len; 10483d829045SPawel Laszczak bool sum_trbs_for_length = false; 10493d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 10503d829045SPawel Laszczak u32 trb_comp_code; 10513d829045SPawel Laszczak u32 td_length; 10523d829045SPawel Laszczak 10533d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 10543d829045SPawel Laszczak trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 10553d829045SPawel Laszczak remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 10563d829045SPawel Laszczak ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); 10573d829045SPawel Laszczak 10583d829045SPawel Laszczak requested = preq->request.length; 10593d829045SPawel Laszczak 10603d829045SPawel Laszczak /* handle completion code */ 10613d829045SPawel Laszczak switch (trb_comp_code) { 10623d829045SPawel Laszczak case COMP_SUCCESS: 10633d829045SPawel Laszczak preq->request.status = 0; 10643d829045SPawel Laszczak break; 10653d829045SPawel Laszczak case COMP_SHORT_PACKET: 10663d829045SPawel Laszczak preq->request.status = 0; 10673d829045SPawel Laszczak sum_trbs_for_length = true; 10683d829045SPawel Laszczak break; 10693d829045SPawel Laszczak case COMP_ISOCH_BUFFER_OVERRUN: 10703d829045SPawel Laszczak case COMP_BABBLE_DETECTED_ERROR: 10713d829045SPawel Laszczak preq->request.status = -EOVERFLOW; 10723d829045SPawel Laszczak break; 10733d829045SPawel Laszczak case COMP_STOPPED: 10743d829045SPawel Laszczak sum_trbs_for_length = true; 10753d829045SPawel Laszczak break; 10763d829045SPawel Laszczak case COMP_STOPPED_SHORT_PACKET: 10773d829045SPawel Laszczak /* field normally containing residue now contains transferred */ 10783d829045SPawel Laszczak preq->request.status = 0; 10793d829045SPawel Laszczak requested = remaining; 10803d829045SPawel Laszczak break; 10813d829045SPawel Laszczak case COMP_STOPPED_LENGTH_INVALID: 10823d829045SPawel Laszczak requested = 0; 10833d829045SPawel Laszczak remaining = 0; 10843d829045SPawel Laszczak break; 10853d829045SPawel Laszczak default: 10863d829045SPawel Laszczak sum_trbs_for_length = true; 10873d829045SPawel Laszczak preq->request.status = -1; 10883d829045SPawel Laszczak break; 10893d829045SPawel Laszczak } 10903d829045SPawel Laszczak 10913d829045SPawel Laszczak if (sum_trbs_for_length) { 10923d829045SPawel Laszczak td_length = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb); 10933d829045SPawel Laszczak td_length += ep_trb_len - remaining; 10943d829045SPawel Laszczak } else { 10953d829045SPawel Laszczak td_length = requested; 10963d829045SPawel Laszczak } 10973d829045SPawel Laszczak 10983d829045SPawel Laszczak td->preq->request.actual += td_length; 10993d829045SPawel Laszczak 11003d829045SPawel Laszczak cdnsp_finish_td(pdev, td, event, pep, &status); 11013d829045SPawel Laszczak } 11023d829045SPawel Laszczak 11033d829045SPawel Laszczak static void cdnsp_skip_isoc_td(struct cdnsp_device *pdev, 11043d829045SPawel Laszczak struct cdnsp_td *td, 11053d829045SPawel Laszczak struct cdnsp_transfer_event *event, 11063d829045SPawel Laszczak struct cdnsp_ep *pep, 11073d829045SPawel Laszczak int status) 11083d829045SPawel Laszczak { 11093d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 11103d829045SPawel Laszczak 11113d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 11123d829045SPawel Laszczak td->preq->request.status = -EXDEV; 11133d829045SPawel Laszczak td->preq->request.actual = 0; 11143d829045SPawel Laszczak 11153d829045SPawel Laszczak /* Update ring dequeue pointer */ 11163d829045SPawel Laszczak while (ep_ring->dequeue != td->last_trb) 11173d829045SPawel Laszczak cdnsp_inc_deq(pdev, ep_ring); 11183d829045SPawel Laszczak 11193d829045SPawel Laszczak cdnsp_inc_deq(pdev, ep_ring); 11203d829045SPawel Laszczak 11213d829045SPawel Laszczak cdnsp_td_cleanup(pdev, td, ep_ring, &status); 11223d829045SPawel Laszczak } 11233d829045SPawel Laszczak 11243d829045SPawel Laszczak /* 11253d829045SPawel Laszczak * Process bulk and interrupt tds, update usb request status and actual_length. 11263d829045SPawel Laszczak */ 11273d829045SPawel Laszczak static void cdnsp_process_bulk_intr_td(struct cdnsp_device *pdev, 11283d829045SPawel Laszczak struct cdnsp_td *td, 11293d829045SPawel Laszczak union cdnsp_trb *ep_trb, 11303d829045SPawel Laszczak struct cdnsp_transfer_event *event, 11313d829045SPawel Laszczak struct cdnsp_ep *ep, 11323d829045SPawel Laszczak int *status) 11333d829045SPawel Laszczak { 11343d829045SPawel Laszczak u32 remaining, requested, ep_trb_len; 11353d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 11363d829045SPawel Laszczak u32 trb_comp_code; 11373d829045SPawel Laszczak 11383d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer)); 11393d829045SPawel Laszczak trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 11403d829045SPawel Laszczak remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)); 11413d829045SPawel Laszczak ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2])); 11423d829045SPawel Laszczak requested = td->preq->request.length; 11433d829045SPawel Laszczak 11443d829045SPawel Laszczak switch (trb_comp_code) { 11453d829045SPawel Laszczak case COMP_SUCCESS: 11463d829045SPawel Laszczak case COMP_SHORT_PACKET: 11473d829045SPawel Laszczak *status = 0; 11483d829045SPawel Laszczak break; 11493d829045SPawel Laszczak case COMP_STOPPED_SHORT_PACKET: 11503d829045SPawel Laszczak td->preq->request.actual = remaining; 11513d829045SPawel Laszczak goto finish_td; 11523d829045SPawel Laszczak case COMP_STOPPED_LENGTH_INVALID: 11533d829045SPawel Laszczak /* Stopped on ep trb with invalid length, exclude it. */ 11543d829045SPawel Laszczak ep_trb_len = 0; 11553d829045SPawel Laszczak remaining = 0; 11563d829045SPawel Laszczak break; 11573d829045SPawel Laszczak } 11583d829045SPawel Laszczak 11593d829045SPawel Laszczak if (ep_trb == td->last_trb) 11603d829045SPawel Laszczak ep_trb_len = requested - remaining; 11613d829045SPawel Laszczak else 11623d829045SPawel Laszczak ep_trb_len = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb) + 11633d829045SPawel Laszczak ep_trb_len - remaining; 11643d829045SPawel Laszczak td->preq->request.actual = ep_trb_len; 11653d829045SPawel Laszczak 11663d829045SPawel Laszczak finish_td: 11673d829045SPawel Laszczak ep->stream_info.drbls_count--; 11683d829045SPawel Laszczak 11693d829045SPawel Laszczak cdnsp_finish_td(pdev, td, event, ep, status); 11703d829045SPawel Laszczak } 11713d829045SPawel Laszczak 11723d829045SPawel Laszczak static void cdnsp_handle_tx_nrdy(struct cdnsp_device *pdev, 11733d829045SPawel Laszczak struct cdnsp_transfer_event *event) 11743d829045SPawel Laszczak { 11753d829045SPawel Laszczak struct cdnsp_generic_trb *generic; 11763d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 11773d829045SPawel Laszczak struct cdnsp_ep *pep; 11783d829045SPawel Laszczak int cur_stream; 11793d829045SPawel Laszczak int ep_index; 11803d829045SPawel Laszczak int host_sid; 11813d829045SPawel Laszczak int dev_sid; 11823d829045SPawel Laszczak 11833d829045SPawel Laszczak generic = (struct cdnsp_generic_trb *)event; 11843d829045SPawel Laszczak ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; 11853d829045SPawel Laszczak dev_sid = TRB_TO_DEV_STREAM(le32_to_cpu(generic->field[0])); 11863d829045SPawel Laszczak host_sid = TRB_TO_HOST_STREAM(le32_to_cpu(generic->field[2])); 11873d829045SPawel Laszczak 11883d829045SPawel Laszczak pep = &pdev->eps[ep_index]; 11893d829045SPawel Laszczak 11903d829045SPawel Laszczak if (!(pep->ep_state & EP_HAS_STREAMS)) 11913d829045SPawel Laszczak return; 11923d829045SPawel Laszczak 11933d829045SPawel Laszczak if (host_sid == STREAM_PRIME_ACK) { 11943d829045SPawel Laszczak pep->stream_info.first_prime_det = 1; 11953d829045SPawel Laszczak for (cur_stream = 1; cur_stream < pep->stream_info.num_streams; 11963d829045SPawel Laszczak cur_stream++) { 11973d829045SPawel Laszczak ep_ring = pep->stream_info.stream_rings[cur_stream]; 11983d829045SPawel Laszczak ep_ring->stream_active = 1; 11993d829045SPawel Laszczak ep_ring->stream_rejected = 0; 12003d829045SPawel Laszczak } 12013d829045SPawel Laszczak } 12023d829045SPawel Laszczak 12033d829045SPawel Laszczak if (host_sid == STREAM_REJECTED) { 12043d829045SPawel Laszczak struct cdnsp_td *td, *td_temp; 12053d829045SPawel Laszczak 12063d829045SPawel Laszczak pep->stream_info.drbls_count--; 12073d829045SPawel Laszczak ep_ring = pep->stream_info.stream_rings[dev_sid]; 12083d829045SPawel Laszczak ep_ring->stream_active = 0; 12093d829045SPawel Laszczak ep_ring->stream_rejected = 1; 12103d829045SPawel Laszczak 12113d829045SPawel Laszczak list_for_each_entry_safe(td, td_temp, &ep_ring->td_list, 12123d829045SPawel Laszczak td_list) { 12133d829045SPawel Laszczak td->drbl = 0; 12143d829045SPawel Laszczak } 12153d829045SPawel Laszczak } 12163d829045SPawel Laszczak 12173d829045SPawel Laszczak cdnsp_ring_doorbell_for_active_rings(pdev, pep); 12183d829045SPawel Laszczak } 12193d829045SPawel Laszczak 12203d829045SPawel Laszczak /* 12213d829045SPawel Laszczak * If this function returns an error condition, it means it got a Transfer 12223d829045SPawel Laszczak * event with a corrupted TRB DMA address or endpoint is disabled. 12233d829045SPawel Laszczak */ 12243d829045SPawel Laszczak static int cdnsp_handle_tx_event(struct cdnsp_device *pdev, 12253d829045SPawel Laszczak struct cdnsp_transfer_event *event) 12263d829045SPawel Laszczak { 12273d829045SPawel Laszczak const struct usb_endpoint_descriptor *desc; 12283d829045SPawel Laszczak bool handling_skipped_tds = false; 12293d829045SPawel Laszczak struct cdnsp_segment *ep_seg; 12303d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 12313d829045SPawel Laszczak int status = -EINPROGRESS; 12323d829045SPawel Laszczak union cdnsp_trb *ep_trb; 12333d829045SPawel Laszczak dma_addr_t ep_trb_dma; 12343d829045SPawel Laszczak struct cdnsp_ep *pep; 12353d829045SPawel Laszczak struct cdnsp_td *td; 12363d829045SPawel Laszczak u32 trb_comp_code; 12373d829045SPawel Laszczak int invalidate; 12383d829045SPawel Laszczak int ep_index; 12393d829045SPawel Laszczak 12403d829045SPawel Laszczak invalidate = le32_to_cpu(event->flags) & TRB_EVENT_INVALIDATE; 12413d829045SPawel Laszczak ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1; 12423d829045SPawel Laszczak trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len)); 12433d829045SPawel Laszczak ep_trb_dma = le64_to_cpu(event->buffer); 12443d829045SPawel Laszczak 12453d829045SPawel Laszczak pep = &pdev->eps[ep_index]; 12463d829045SPawel Laszczak ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer)); 12473d829045SPawel Laszczak 12483d829045SPawel Laszczak /* 12493d829045SPawel Laszczak * If device is disconnect then all requests will be dequeued 12503d829045SPawel Laszczak * by upper layers as part of disconnect sequence. 12513d829045SPawel Laszczak * We don't want handle such event to avoid racing. 12523d829045SPawel Laszczak */ 12533d829045SPawel Laszczak if (invalidate || !pdev->gadget.connected) 12543d829045SPawel Laszczak goto cleanup; 12553d829045SPawel Laszczak 1256*118b2a32SPawel Laszczak if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_DISABLED) { 1257*118b2a32SPawel Laszczak trace_cdnsp_ep_disabled(pep->out_ctx); 12583d829045SPawel Laszczak goto err_out; 1259*118b2a32SPawel Laszczak } 12603d829045SPawel Laszczak 12613d829045SPawel Laszczak /* Some transfer events don't always point to a trb*/ 12623d829045SPawel Laszczak if (!ep_ring) { 12633d829045SPawel Laszczak switch (trb_comp_code) { 12643d829045SPawel Laszczak case COMP_INVALID_STREAM_TYPE_ERROR: 12653d829045SPawel Laszczak case COMP_INVALID_STREAM_ID_ERROR: 12663d829045SPawel Laszczak case COMP_RING_UNDERRUN: 12673d829045SPawel Laszczak case COMP_RING_OVERRUN: 12683d829045SPawel Laszczak goto cleanup; 12693d829045SPawel Laszczak default: 12703d829045SPawel Laszczak dev_err(pdev->dev, "ERROR: %s event for unknown ring\n", 12713d829045SPawel Laszczak pep->name); 12723d829045SPawel Laszczak goto err_out; 12733d829045SPawel Laszczak } 12743d829045SPawel Laszczak } 12753d829045SPawel Laszczak 12763d829045SPawel Laszczak /* Look for some error cases that need special treatment. */ 12773d829045SPawel Laszczak switch (trb_comp_code) { 12783d829045SPawel Laszczak case COMP_BABBLE_DETECTED_ERROR: 12793d829045SPawel Laszczak status = -EOVERFLOW; 12803d829045SPawel Laszczak break; 12813d829045SPawel Laszczak case COMP_RING_UNDERRUN: 12823d829045SPawel Laszczak case COMP_RING_OVERRUN: 12833d829045SPawel Laszczak /* 12843d829045SPawel Laszczak * When the Isoch ring is empty, the controller will generate 12853d829045SPawel Laszczak * a Ring Overrun Event for IN Isoch endpoint or Ring 12863d829045SPawel Laszczak * Underrun Event for OUT Isoch endpoint. 12873d829045SPawel Laszczak */ 12883d829045SPawel Laszczak goto cleanup; 12893d829045SPawel Laszczak case COMP_MISSED_SERVICE_ERROR: 12903d829045SPawel Laszczak /* 12913d829045SPawel Laszczak * When encounter missed service error, one or more isoc tds 12923d829045SPawel Laszczak * may be missed by controller. 12933d829045SPawel Laszczak * Set skip flag of the ep_ring; Complete the missed tds as 12943d829045SPawel Laszczak * short transfer when process the ep_ring next time. 12953d829045SPawel Laszczak */ 12963d829045SPawel Laszczak pep->skip = true; 12973d829045SPawel Laszczak break; 12983d829045SPawel Laszczak } 12993d829045SPawel Laszczak 13003d829045SPawel Laszczak do { 13013d829045SPawel Laszczak /* 13023d829045SPawel Laszczak * This TRB should be in the TD at the head of this ring's TD 13033d829045SPawel Laszczak * list. 13043d829045SPawel Laszczak */ 13053d829045SPawel Laszczak if (list_empty(&ep_ring->td_list)) { 1306*118b2a32SPawel Laszczak /* 1307*118b2a32SPawel Laszczak * Don't print warnings if it's due to a stopped 1308*118b2a32SPawel Laszczak * endpoint generating an extra completion event, or 1309*118b2a32SPawel Laszczak * a event for the last TRB of a short TD we already 1310*118b2a32SPawel Laszczak * got a short event for. 1311*118b2a32SPawel Laszczak * The short TD is already removed from the TD list. 1312*118b2a32SPawel Laszczak */ 1313*118b2a32SPawel Laszczak if (!(trb_comp_code == COMP_STOPPED || 1314*118b2a32SPawel Laszczak trb_comp_code == COMP_STOPPED_LENGTH_INVALID || 1315*118b2a32SPawel Laszczak ep_ring->last_td_was_short)) 1316*118b2a32SPawel Laszczak trace_cdnsp_trb_without_td(ep_ring, 1317*118b2a32SPawel Laszczak (struct cdnsp_generic_trb *)event); 1318*118b2a32SPawel Laszczak 1319*118b2a32SPawel Laszczak if (pep->skip) { 13203d829045SPawel Laszczak pep->skip = false; 1321*118b2a32SPawel Laszczak trace_cdnsp_ep_list_empty_with_skip(pep, 0); 1322*118b2a32SPawel Laszczak } 13233d829045SPawel Laszczak 13243d829045SPawel Laszczak goto cleanup; 13253d829045SPawel Laszczak } 13263d829045SPawel Laszczak 13273d829045SPawel Laszczak td = list_entry(ep_ring->td_list.next, struct cdnsp_td, 13283d829045SPawel Laszczak td_list); 13293d829045SPawel Laszczak 13303d829045SPawel Laszczak /* Is this a TRB in the currently executing TD? */ 13313d829045SPawel Laszczak ep_seg = cdnsp_trb_in_td(pdev, ep_ring->deq_seg, 13323d829045SPawel Laszczak ep_ring->dequeue, td->last_trb, 13333d829045SPawel Laszczak ep_trb_dma); 13343d829045SPawel Laszczak 13353d829045SPawel Laszczak /* 13363d829045SPawel Laszczak * Skip the Force Stopped Event. The event_trb(ep_trb_dma) 13373d829045SPawel Laszczak * of FSE is not in the current TD pointed by ep_ring->dequeue 13383d829045SPawel Laszczak * because that the hardware dequeue pointer still at the 13393d829045SPawel Laszczak * previous TRB of the current TD. The previous TRB maybe a 13403d829045SPawel Laszczak * Link TD or the last TRB of the previous TD. The command 13413d829045SPawel Laszczak * completion handle will take care the rest. 13423d829045SPawel Laszczak */ 13433d829045SPawel Laszczak if (!ep_seg && (trb_comp_code == COMP_STOPPED || 13443d829045SPawel Laszczak trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) { 13453d829045SPawel Laszczak pep->skip = false; 13463d829045SPawel Laszczak goto cleanup; 13473d829045SPawel Laszczak } 13483d829045SPawel Laszczak 13493d829045SPawel Laszczak desc = td->preq->pep->endpoint.desc; 13503d829045SPawel Laszczak if (!ep_seg) { 13513d829045SPawel Laszczak if (!pep->skip || !usb_endpoint_xfer_isoc(desc)) { 13523d829045SPawel Laszczak /* Something is busted, give up! */ 13533d829045SPawel Laszczak dev_err(pdev->dev, 13543d829045SPawel Laszczak "ERROR Transfer event TRB DMA ptr not " 13553d829045SPawel Laszczak "part of current TD ep_index %d " 13563d829045SPawel Laszczak "comp_code %u\n", ep_index, 13573d829045SPawel Laszczak trb_comp_code); 13583d829045SPawel Laszczak return -EINVAL; 13593d829045SPawel Laszczak } 13603d829045SPawel Laszczak 13613d829045SPawel Laszczak cdnsp_skip_isoc_td(pdev, td, event, pep, status); 13623d829045SPawel Laszczak goto cleanup; 13633d829045SPawel Laszczak } 13643d829045SPawel Laszczak 13653d829045SPawel Laszczak if (trb_comp_code == COMP_SHORT_PACKET) 13663d829045SPawel Laszczak ep_ring->last_td_was_short = true; 13673d829045SPawel Laszczak else 13683d829045SPawel Laszczak ep_ring->last_td_was_short = false; 13693d829045SPawel Laszczak 13703d829045SPawel Laszczak if (pep->skip) { 13713d829045SPawel Laszczak pep->skip = false; 13723d829045SPawel Laszczak cdnsp_skip_isoc_td(pdev, td, event, pep, status); 13733d829045SPawel Laszczak goto cleanup; 13743d829045SPawel Laszczak } 13753d829045SPawel Laszczak 13763d829045SPawel Laszczak ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) 13773d829045SPawel Laszczak / sizeof(*ep_trb)]; 13783d829045SPawel Laszczak 1379*118b2a32SPawel Laszczak trace_cdnsp_handle_transfer(ep_ring, 1380*118b2a32SPawel Laszczak (struct cdnsp_generic_trb *)ep_trb); 1381*118b2a32SPawel Laszczak 13823d829045SPawel Laszczak if (cdnsp_trb_is_noop(ep_trb)) 13833d829045SPawel Laszczak goto cleanup; 13843d829045SPawel Laszczak 13853d829045SPawel Laszczak if (usb_endpoint_xfer_control(desc)) 13863d829045SPawel Laszczak cdnsp_process_ctrl_td(pdev, td, ep_trb, event, pep, 13873d829045SPawel Laszczak &status); 13883d829045SPawel Laszczak else if (usb_endpoint_xfer_isoc(desc)) 13893d829045SPawel Laszczak cdnsp_process_isoc_td(pdev, td, ep_trb, event, pep, 13903d829045SPawel Laszczak status); 13913d829045SPawel Laszczak else 13923d829045SPawel Laszczak cdnsp_process_bulk_intr_td(pdev, td, ep_trb, event, pep, 13933d829045SPawel Laszczak &status); 13943d829045SPawel Laszczak cleanup: 13953d829045SPawel Laszczak handling_skipped_tds = pep->skip; 13963d829045SPawel Laszczak 13973d829045SPawel Laszczak /* 13983d829045SPawel Laszczak * Do not update event ring dequeue pointer if we're in a loop 13993d829045SPawel Laszczak * processing missed tds. 14003d829045SPawel Laszczak */ 14013d829045SPawel Laszczak if (!handling_skipped_tds) 14023d829045SPawel Laszczak cdnsp_inc_deq(pdev, pdev->event_ring); 14033d829045SPawel Laszczak 14043d829045SPawel Laszczak /* 14053d829045SPawel Laszczak * If ep->skip is set, it means there are missed tds on the 14063d829045SPawel Laszczak * endpoint ring need to take care of. 14073d829045SPawel Laszczak * Process them as short transfer until reach the td pointed by 14083d829045SPawel Laszczak * the event. 14093d829045SPawel Laszczak */ 14103d829045SPawel Laszczak } while (handling_skipped_tds); 14113d829045SPawel Laszczak return 0; 14123d829045SPawel Laszczak 14133d829045SPawel Laszczak err_out: 14143d829045SPawel Laszczak dev_err(pdev->dev, "@%016llx %08x %08x %08x %08x\n", 14153d829045SPawel Laszczak (unsigned long long) 14163d829045SPawel Laszczak cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg, 14173d829045SPawel Laszczak pdev->event_ring->dequeue), 14183d829045SPawel Laszczak lower_32_bits(le64_to_cpu(event->buffer)), 14193d829045SPawel Laszczak upper_32_bits(le64_to_cpu(event->buffer)), 14203d829045SPawel Laszczak le32_to_cpu(event->transfer_len), 14213d829045SPawel Laszczak le32_to_cpu(event->flags)); 14223d829045SPawel Laszczak return -EINVAL; 14233d829045SPawel Laszczak } 14243d829045SPawel Laszczak 14253d829045SPawel Laszczak /* 14263d829045SPawel Laszczak * This function handles all events on the event ring. 14273d829045SPawel Laszczak * Returns true for "possibly more events to process" (caller should call 14283d829045SPawel Laszczak * again), otherwise false if done. 14293d829045SPawel Laszczak */ 14303d829045SPawel Laszczak static bool cdnsp_handle_event(struct cdnsp_device *pdev) 14313d829045SPawel Laszczak { 14323d829045SPawel Laszczak unsigned int comp_code; 14333d829045SPawel Laszczak union cdnsp_trb *event; 14343d829045SPawel Laszczak bool update_ptrs = true; 14353d829045SPawel Laszczak __le32 cycle_bit; 14363d829045SPawel Laszczak int ret = 0; 14373d829045SPawel Laszczak u32 flags; 14383d829045SPawel Laszczak 14393d829045SPawel Laszczak event = pdev->event_ring->dequeue; 14403d829045SPawel Laszczak flags = le32_to_cpu(event->event_cmd.flags); 14413d829045SPawel Laszczak cycle_bit = (flags & TRB_CYCLE); 14423d829045SPawel Laszczak 14433d829045SPawel Laszczak /* Does the controller or driver own the TRB? */ 14443d829045SPawel Laszczak if (cycle_bit != pdev->event_ring->cycle_state) 14453d829045SPawel Laszczak return false; 14463d829045SPawel Laszczak 1447*118b2a32SPawel Laszczak trace_cdnsp_handle_event(pdev->event_ring, &event->generic); 1448*118b2a32SPawel Laszczak 14493d829045SPawel Laszczak /* 14503d829045SPawel Laszczak * Barrier between reading the TRB_CYCLE (valid) flag above and any 14513d829045SPawel Laszczak * reads of the event's flags/data below. 14523d829045SPawel Laszczak */ 14533d829045SPawel Laszczak rmb(); 14543d829045SPawel Laszczak 14553d829045SPawel Laszczak switch (flags & TRB_TYPE_BITMASK) { 14563d829045SPawel Laszczak case TRB_TYPE(TRB_COMPLETION): 14573d829045SPawel Laszczak /* 14583d829045SPawel Laszczak * Command can't be handled in interrupt context so just 14593d829045SPawel Laszczak * increment command ring dequeue pointer. 14603d829045SPawel Laszczak */ 14613d829045SPawel Laszczak cdnsp_inc_deq(pdev, pdev->cmd_ring); 14623d829045SPawel Laszczak break; 14633d829045SPawel Laszczak case TRB_TYPE(TRB_PORT_STATUS): 14643d829045SPawel Laszczak cdnsp_handle_port_status(pdev, event); 14653d829045SPawel Laszczak update_ptrs = false; 14663d829045SPawel Laszczak break; 14673d829045SPawel Laszczak case TRB_TYPE(TRB_TRANSFER): 14683d829045SPawel Laszczak ret = cdnsp_handle_tx_event(pdev, &event->trans_event); 14693d829045SPawel Laszczak if (ret >= 0) 14703d829045SPawel Laszczak update_ptrs = false; 14713d829045SPawel Laszczak break; 14723d829045SPawel Laszczak case TRB_TYPE(TRB_SETUP): 14733d829045SPawel Laszczak pdev->ep0_stage = CDNSP_SETUP_STAGE; 14743d829045SPawel Laszczak pdev->setup_id = TRB_SETUPID_TO_TYPE(flags); 14753d829045SPawel Laszczak pdev->setup_speed = TRB_SETUP_SPEEDID(flags); 14763d829045SPawel Laszczak pdev->setup = *((struct usb_ctrlrequest *) 14773d829045SPawel Laszczak &event->trans_event.buffer); 14783d829045SPawel Laszczak 14793d829045SPawel Laszczak cdnsp_setup_analyze(pdev); 14803d829045SPawel Laszczak break; 14813d829045SPawel Laszczak case TRB_TYPE(TRB_ENDPOINT_NRDY): 14823d829045SPawel Laszczak cdnsp_handle_tx_nrdy(pdev, &event->trans_event); 14833d829045SPawel Laszczak break; 14843d829045SPawel Laszczak case TRB_TYPE(TRB_HC_EVENT): { 14853d829045SPawel Laszczak comp_code = GET_COMP_CODE(le32_to_cpu(event->generic.field[2])); 14863d829045SPawel Laszczak 14873d829045SPawel Laszczak switch (comp_code) { 14883d829045SPawel Laszczak case COMP_EVENT_RING_FULL_ERROR: 14893d829045SPawel Laszczak dev_err(pdev->dev, "Event Ring Full\n"); 14903d829045SPawel Laszczak break; 14913d829045SPawel Laszczak default: 14923d829045SPawel Laszczak dev_err(pdev->dev, "Controller error code 0x%02x\n", 14933d829045SPawel Laszczak comp_code); 14943d829045SPawel Laszczak } 14953d829045SPawel Laszczak 14963d829045SPawel Laszczak break; 14973d829045SPawel Laszczak } 14983d829045SPawel Laszczak case TRB_TYPE(TRB_MFINDEX_WRAP): 14993d829045SPawel Laszczak case TRB_TYPE(TRB_DRB_OVERFLOW): 15003d829045SPawel Laszczak break; 15013d829045SPawel Laszczak default: 15023d829045SPawel Laszczak dev_warn(pdev->dev, "ERROR unknown event type %ld\n", 15033d829045SPawel Laszczak TRB_FIELD_TO_TYPE(flags)); 15043d829045SPawel Laszczak } 15053d829045SPawel Laszczak 15063d829045SPawel Laszczak if (update_ptrs) 15073d829045SPawel Laszczak /* Update SW event ring dequeue pointer. */ 15083d829045SPawel Laszczak cdnsp_inc_deq(pdev, pdev->event_ring); 15093d829045SPawel Laszczak 15103d829045SPawel Laszczak /* 15113d829045SPawel Laszczak * Caller will call us again to check if there are more items 15123d829045SPawel Laszczak * on the event ring. 15133d829045SPawel Laszczak */ 15143d829045SPawel Laszczak return true; 15153d829045SPawel Laszczak } 15163d829045SPawel Laszczak 15173d829045SPawel Laszczak irqreturn_t cdnsp_thread_irq_handler(int irq, void *data) 15183d829045SPawel Laszczak { 15193d829045SPawel Laszczak struct cdnsp_device *pdev = (struct cdnsp_device *)data; 15203d829045SPawel Laszczak union cdnsp_trb *event_ring_deq; 15213d829045SPawel Laszczak int counter = 0; 15223d829045SPawel Laszczak 15233d829045SPawel Laszczak spin_lock(&pdev->lock); 15243d829045SPawel Laszczak 15253d829045SPawel Laszczak if (pdev->cdnsp_state & (CDNSP_STATE_HALTED | CDNSP_STATE_DYING)) { 15263d829045SPawel Laszczak cdnsp_died(pdev); 15273d829045SPawel Laszczak spin_unlock(&pdev->lock); 15283d829045SPawel Laszczak return IRQ_HANDLED; 15293d829045SPawel Laszczak } 15303d829045SPawel Laszczak 15313d829045SPawel Laszczak event_ring_deq = pdev->event_ring->dequeue; 15323d829045SPawel Laszczak 15333d829045SPawel Laszczak while (cdnsp_handle_event(pdev)) { 15343d829045SPawel Laszczak if (++counter >= TRBS_PER_EV_DEQ_UPDATE) { 15353d829045SPawel Laszczak cdnsp_update_erst_dequeue(pdev, event_ring_deq, 0); 15363d829045SPawel Laszczak event_ring_deq = pdev->event_ring->dequeue; 15373d829045SPawel Laszczak counter = 0; 15383d829045SPawel Laszczak } 15393d829045SPawel Laszczak } 15403d829045SPawel Laszczak 15413d829045SPawel Laszczak cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1); 15423d829045SPawel Laszczak 15433d829045SPawel Laszczak spin_unlock(&pdev->lock); 15443d829045SPawel Laszczak 15453d829045SPawel Laszczak return IRQ_HANDLED; 15463d829045SPawel Laszczak } 15473d829045SPawel Laszczak 15483d829045SPawel Laszczak irqreturn_t cdnsp_irq_handler(int irq, void *priv) 15493d829045SPawel Laszczak { 15503d829045SPawel Laszczak struct cdnsp_device *pdev = (struct cdnsp_device *)priv; 15513d829045SPawel Laszczak u32 irq_pending; 15523d829045SPawel Laszczak u32 status; 15533d829045SPawel Laszczak 15543d829045SPawel Laszczak status = readl(&pdev->op_regs->status); 15553d829045SPawel Laszczak 15563d829045SPawel Laszczak if (status == ~(u32)0) { 15573d829045SPawel Laszczak cdnsp_died(pdev); 15583d829045SPawel Laszczak return IRQ_HANDLED; 15593d829045SPawel Laszczak } 15603d829045SPawel Laszczak 15613d829045SPawel Laszczak if (!(status & STS_EINT)) 15623d829045SPawel Laszczak return IRQ_NONE; 15633d829045SPawel Laszczak 15643d829045SPawel Laszczak writel(status | STS_EINT, &pdev->op_regs->status); 15653d829045SPawel Laszczak irq_pending = readl(&pdev->ir_set->irq_pending); 15663d829045SPawel Laszczak irq_pending |= IMAN_IP; 15673d829045SPawel Laszczak writel(irq_pending, &pdev->ir_set->irq_pending); 15683d829045SPawel Laszczak 15693d829045SPawel Laszczak if (status & STS_FATAL) { 15703d829045SPawel Laszczak cdnsp_died(pdev); 15713d829045SPawel Laszczak return IRQ_HANDLED; 15723d829045SPawel Laszczak } 15733d829045SPawel Laszczak 15743d829045SPawel Laszczak return IRQ_WAKE_THREAD; 15753d829045SPawel Laszczak } 15763d829045SPawel Laszczak 15773d829045SPawel Laszczak /* 15783d829045SPawel Laszczak * Generic function for queuing a TRB on a ring. 15793d829045SPawel Laszczak * The caller must have checked to make sure there's room on the ring. 15803d829045SPawel Laszczak * 15813d829045SPawel Laszczak * @more_trbs_coming: Will you enqueue more TRBs before setting doorbell? 15823d829045SPawel Laszczak */ 15833d829045SPawel Laszczak static void cdnsp_queue_trb(struct cdnsp_device *pdev, struct cdnsp_ring *ring, 15843d829045SPawel Laszczak bool more_trbs_coming, u32 field1, u32 field2, 15853d829045SPawel Laszczak u32 field3, u32 field4) 15863d829045SPawel Laszczak { 15873d829045SPawel Laszczak struct cdnsp_generic_trb *trb; 15883d829045SPawel Laszczak 15893d829045SPawel Laszczak trb = &ring->enqueue->generic; 15903d829045SPawel Laszczak 15913d829045SPawel Laszczak trb->field[0] = cpu_to_le32(field1); 15923d829045SPawel Laszczak trb->field[1] = cpu_to_le32(field2); 15933d829045SPawel Laszczak trb->field[2] = cpu_to_le32(field3); 15943d829045SPawel Laszczak trb->field[3] = cpu_to_le32(field4); 15953d829045SPawel Laszczak 1596*118b2a32SPawel Laszczak trace_cdnsp_queue_trb(ring, trb); 15973d829045SPawel Laszczak cdnsp_inc_enq(pdev, ring, more_trbs_coming); 15983d829045SPawel Laszczak } 15993d829045SPawel Laszczak 16003d829045SPawel Laszczak /* 16013d829045SPawel Laszczak * Does various checks on the endpoint ring, and makes it ready to 16023d829045SPawel Laszczak * queue num_trbs. 16033d829045SPawel Laszczak */ 16043d829045SPawel Laszczak static int cdnsp_prepare_ring(struct cdnsp_device *pdev, 16053d829045SPawel Laszczak struct cdnsp_ring *ep_ring, 16063d829045SPawel Laszczak u32 ep_state, unsigned 16073d829045SPawel Laszczak int num_trbs, 16083d829045SPawel Laszczak gfp_t mem_flags) 16093d829045SPawel Laszczak { 16103d829045SPawel Laszczak unsigned int num_trbs_needed; 16113d829045SPawel Laszczak 16123d829045SPawel Laszczak /* Make sure the endpoint has been added to controller schedule. */ 16133d829045SPawel Laszczak switch (ep_state) { 16143d829045SPawel Laszczak case EP_STATE_STOPPED: 16153d829045SPawel Laszczak case EP_STATE_RUNNING: 16163d829045SPawel Laszczak case EP_STATE_HALTED: 16173d829045SPawel Laszczak break; 16183d829045SPawel Laszczak default: 16193d829045SPawel Laszczak dev_err(pdev->dev, "ERROR: incorrect endpoint state\n"); 16203d829045SPawel Laszczak return -EINVAL; 16213d829045SPawel Laszczak } 16223d829045SPawel Laszczak 16233d829045SPawel Laszczak while (1) { 16243d829045SPawel Laszczak if (cdnsp_room_on_ring(pdev, ep_ring, num_trbs)) 16253d829045SPawel Laszczak break; 16263d829045SPawel Laszczak 1627*118b2a32SPawel Laszczak trace_cdnsp_no_room_on_ring("try ring expansion"); 1628*118b2a32SPawel Laszczak 16293d829045SPawel Laszczak num_trbs_needed = num_trbs - ep_ring->num_trbs_free; 16303d829045SPawel Laszczak if (cdnsp_ring_expansion(pdev, ep_ring, num_trbs_needed, 16313d829045SPawel Laszczak mem_flags)) { 16323d829045SPawel Laszczak dev_err(pdev->dev, "Ring expansion failed\n"); 16333d829045SPawel Laszczak return -ENOMEM; 16343d829045SPawel Laszczak } 16353d829045SPawel Laszczak } 16363d829045SPawel Laszczak 16373d829045SPawel Laszczak while (cdnsp_trb_is_link(ep_ring->enqueue)) { 16383d829045SPawel Laszczak ep_ring->enqueue->link.control |= cpu_to_le32(TRB_CHAIN); 16393d829045SPawel Laszczak /* The cycle bit must be set as the last operation. */ 16403d829045SPawel Laszczak wmb(); 16413d829045SPawel Laszczak ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE); 16423d829045SPawel Laszczak 16433d829045SPawel Laszczak /* Toggle the cycle bit after the last ring segment. */ 16443d829045SPawel Laszczak if (cdnsp_link_trb_toggles_cycle(ep_ring->enqueue)) 16453d829045SPawel Laszczak ep_ring->cycle_state ^= 1; 16463d829045SPawel Laszczak ep_ring->enq_seg = ep_ring->enq_seg->next; 16473d829045SPawel Laszczak ep_ring->enqueue = ep_ring->enq_seg->trbs; 16483d829045SPawel Laszczak } 16493d829045SPawel Laszczak return 0; 16503d829045SPawel Laszczak } 16513d829045SPawel Laszczak 16523d829045SPawel Laszczak static int cdnsp_prepare_transfer(struct cdnsp_device *pdev, 16533d829045SPawel Laszczak struct cdnsp_request *preq, 16543d829045SPawel Laszczak unsigned int num_trbs) 16553d829045SPawel Laszczak { 16563d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 16573d829045SPawel Laszczak int ret; 16583d829045SPawel Laszczak 16593d829045SPawel Laszczak ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep, 16603d829045SPawel Laszczak preq->request.stream_id); 16613d829045SPawel Laszczak if (!ep_ring) 16623d829045SPawel Laszczak return -EINVAL; 16633d829045SPawel Laszczak 16643d829045SPawel Laszczak ret = cdnsp_prepare_ring(pdev, ep_ring, 16653d829045SPawel Laszczak GET_EP_CTX_STATE(preq->pep->out_ctx), 16663d829045SPawel Laszczak num_trbs, GFP_ATOMIC); 16673d829045SPawel Laszczak if (ret) 16683d829045SPawel Laszczak return ret; 16693d829045SPawel Laszczak 16703d829045SPawel Laszczak INIT_LIST_HEAD(&preq->td.td_list); 16713d829045SPawel Laszczak preq->td.preq = preq; 16723d829045SPawel Laszczak 16733d829045SPawel Laszczak /* Add this TD to the tail of the endpoint ring's TD list. */ 16743d829045SPawel Laszczak list_add_tail(&preq->td.td_list, &ep_ring->td_list); 16753d829045SPawel Laszczak ep_ring->num_tds++; 16763d829045SPawel Laszczak preq->pep->stream_info.td_count++; 16773d829045SPawel Laszczak 16783d829045SPawel Laszczak preq->td.start_seg = ep_ring->enq_seg; 16793d829045SPawel Laszczak preq->td.first_trb = ep_ring->enqueue; 16803d829045SPawel Laszczak 16813d829045SPawel Laszczak return 0; 16823d829045SPawel Laszczak } 16833d829045SPawel Laszczak 16843d829045SPawel Laszczak static unsigned int cdnsp_count_trbs(u64 addr, u64 len) 16853d829045SPawel Laszczak { 16863d829045SPawel Laszczak unsigned int num_trbs; 16873d829045SPawel Laszczak 16883d829045SPawel Laszczak num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)), 16893d829045SPawel Laszczak TRB_MAX_BUFF_SIZE); 16903d829045SPawel Laszczak if (num_trbs == 0) 16913d829045SPawel Laszczak num_trbs++; 16923d829045SPawel Laszczak 16933d829045SPawel Laszczak return num_trbs; 16943d829045SPawel Laszczak } 16953d829045SPawel Laszczak 16963d829045SPawel Laszczak static unsigned int count_trbs_needed(struct cdnsp_request *preq) 16973d829045SPawel Laszczak { 16983d829045SPawel Laszczak return cdnsp_count_trbs(preq->request.dma, preq->request.length); 16993d829045SPawel Laszczak } 17003d829045SPawel Laszczak 17013d829045SPawel Laszczak static unsigned int count_sg_trbs_needed(struct cdnsp_request *preq) 17023d829045SPawel Laszczak { 17033d829045SPawel Laszczak unsigned int i, len, full_len, num_trbs = 0; 17043d829045SPawel Laszczak struct scatterlist *sg; 17053d829045SPawel Laszczak 17063d829045SPawel Laszczak full_len = preq->request.length; 17073d829045SPawel Laszczak 17083d829045SPawel Laszczak for_each_sg(preq->request.sg, sg, preq->request.num_sgs, i) { 17093d829045SPawel Laszczak len = sg_dma_len(sg); 17103d829045SPawel Laszczak num_trbs += cdnsp_count_trbs(sg_dma_address(sg), len); 17113d829045SPawel Laszczak len = min(len, full_len); 17123d829045SPawel Laszczak full_len -= len; 17133d829045SPawel Laszczak if (full_len == 0) 17143d829045SPawel Laszczak break; 17153d829045SPawel Laszczak } 17163d829045SPawel Laszczak 17173d829045SPawel Laszczak return num_trbs; 17183d829045SPawel Laszczak } 17193d829045SPawel Laszczak 17203d829045SPawel Laszczak static unsigned int count_isoc_trbs_needed(struct cdnsp_request *preq) 17213d829045SPawel Laszczak { 17223d829045SPawel Laszczak return cdnsp_count_trbs(preq->request.dma, preq->request.length); 17233d829045SPawel Laszczak } 17243d829045SPawel Laszczak 17253d829045SPawel Laszczak static void cdnsp_check_trb_math(struct cdnsp_request *preq, int running_total) 17263d829045SPawel Laszczak { 17273d829045SPawel Laszczak if (running_total != preq->request.length) 17283d829045SPawel Laszczak dev_err(preq->pep->pdev->dev, 17293d829045SPawel Laszczak "%s - Miscalculated tx length, " 17303d829045SPawel Laszczak "queued %#x, asked for %#x (%d)\n", 17313d829045SPawel Laszczak preq->pep->name, running_total, 17323d829045SPawel Laszczak preq->request.length, preq->request.actual); 17333d829045SPawel Laszczak } 17343d829045SPawel Laszczak 17353d829045SPawel Laszczak /* 17363d829045SPawel Laszczak * TD size is the number of max packet sized packets remaining in the TD 17373d829045SPawel Laszczak * (*not* including this TRB). 17383d829045SPawel Laszczak * 17393d829045SPawel Laszczak * Total TD packet count = total_packet_count = 17403d829045SPawel Laszczak * DIV_ROUND_UP(TD size in bytes / wMaxPacketSize) 17413d829045SPawel Laszczak * 17423d829045SPawel Laszczak * Packets transferred up to and including this TRB = packets_transferred = 17433d829045SPawel Laszczak * rounddown(total bytes transferred including this TRB / wMaxPacketSize) 17443d829045SPawel Laszczak * 17453d829045SPawel Laszczak * TD size = total_packet_count - packets_transferred 17463d829045SPawel Laszczak * 17473d829045SPawel Laszczak * It must fit in bits 21:17, so it can't be bigger than 31. 17483d829045SPawel Laszczak * This is taken care of in the TRB_TD_SIZE() macro 17493d829045SPawel Laszczak * 17503d829045SPawel Laszczak * The last TRB in a TD must have the TD size set to zero. 17513d829045SPawel Laszczak */ 17523d829045SPawel Laszczak static u32 cdnsp_td_remainder(struct cdnsp_device *pdev, 17533d829045SPawel Laszczak int transferred, 17543d829045SPawel Laszczak int trb_buff_len, 17553d829045SPawel Laszczak unsigned int td_total_len, 17563d829045SPawel Laszczak struct cdnsp_request *preq, 17573d829045SPawel Laszczak bool more_trbs_coming) 17583d829045SPawel Laszczak { 17593d829045SPawel Laszczak u32 maxp, total_packet_count; 17603d829045SPawel Laszczak 17613d829045SPawel Laszczak /* One TRB with a zero-length data packet. */ 17623d829045SPawel Laszczak if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) || 17633d829045SPawel Laszczak trb_buff_len == td_total_len) 17643d829045SPawel Laszczak return 0; 17653d829045SPawel Laszczak 17663d829045SPawel Laszczak maxp = usb_endpoint_maxp(preq->pep->endpoint.desc); 17673d829045SPawel Laszczak total_packet_count = DIV_ROUND_UP(td_total_len, maxp); 17683d829045SPawel Laszczak 17693d829045SPawel Laszczak /* Queuing functions don't count the current TRB into transferred. */ 17703d829045SPawel Laszczak return (total_packet_count - ((transferred + trb_buff_len) / maxp)); 17713d829045SPawel Laszczak } 17723d829045SPawel Laszczak 17733d829045SPawel Laszczak static int cdnsp_align_td(struct cdnsp_device *pdev, 17743d829045SPawel Laszczak struct cdnsp_request *preq, u32 enqd_len, 17753d829045SPawel Laszczak u32 *trb_buff_len, struct cdnsp_segment *seg) 17763d829045SPawel Laszczak { 17773d829045SPawel Laszczak struct device *dev = pdev->dev; 17783d829045SPawel Laszczak unsigned int unalign; 17793d829045SPawel Laszczak unsigned int max_pkt; 17803d829045SPawel Laszczak u32 new_buff_len; 17813d829045SPawel Laszczak 17823d829045SPawel Laszczak max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc); 17833d829045SPawel Laszczak unalign = (enqd_len + *trb_buff_len) % max_pkt; 17843d829045SPawel Laszczak 17853d829045SPawel Laszczak /* We got lucky, last normal TRB data on segment is packet aligned. */ 17863d829045SPawel Laszczak if (unalign == 0) 17873d829045SPawel Laszczak return 0; 17883d829045SPawel Laszczak 17893d829045SPawel Laszczak /* Is the last nornal TRB alignable by splitting it. */ 17903d829045SPawel Laszczak if (*trb_buff_len > unalign) { 17913d829045SPawel Laszczak *trb_buff_len -= unalign; 1792*118b2a32SPawel Laszczak trace_cdnsp_bounce_align_td_split(preq, *trb_buff_len, 1793*118b2a32SPawel Laszczak enqd_len, 0, unalign); 17943d829045SPawel Laszczak return 0; 17953d829045SPawel Laszczak } 17963d829045SPawel Laszczak 17973d829045SPawel Laszczak /* 17983d829045SPawel Laszczak * We want enqd_len + trb_buff_len to sum up to a number aligned to 17993d829045SPawel Laszczak * number which is divisible by the endpoint's wMaxPacketSize. IOW: 18003d829045SPawel Laszczak * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0. 18013d829045SPawel Laszczak */ 18023d829045SPawel Laszczak new_buff_len = max_pkt - (enqd_len % max_pkt); 18033d829045SPawel Laszczak 18043d829045SPawel Laszczak if (new_buff_len > (preq->request.length - enqd_len)) 18053d829045SPawel Laszczak new_buff_len = (preq->request.length - enqd_len); 18063d829045SPawel Laszczak 18073d829045SPawel Laszczak /* Create a max max_pkt sized bounce buffer pointed to by last trb. */ 18083d829045SPawel Laszczak if (preq->direction) { 18093d829045SPawel Laszczak sg_pcopy_to_buffer(preq->request.sg, 18103d829045SPawel Laszczak preq->request.num_mapped_sgs, 18113d829045SPawel Laszczak seg->bounce_buf, new_buff_len, enqd_len); 18123d829045SPawel Laszczak seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, 18133d829045SPawel Laszczak max_pkt, DMA_TO_DEVICE); 18143d829045SPawel Laszczak } else { 18153d829045SPawel Laszczak seg->bounce_dma = dma_map_single(dev, seg->bounce_buf, 18163d829045SPawel Laszczak max_pkt, DMA_FROM_DEVICE); 18173d829045SPawel Laszczak } 18183d829045SPawel Laszczak 18193d829045SPawel Laszczak if (dma_mapping_error(dev, seg->bounce_dma)) { 18203d829045SPawel Laszczak /* Try without aligning.*/ 18213d829045SPawel Laszczak dev_warn(pdev->dev, 18223d829045SPawel Laszczak "Failed mapping bounce buffer, not aligning\n"); 18233d829045SPawel Laszczak return 0; 18243d829045SPawel Laszczak } 18253d829045SPawel Laszczak 18263d829045SPawel Laszczak *trb_buff_len = new_buff_len; 18273d829045SPawel Laszczak seg->bounce_len = new_buff_len; 18283d829045SPawel Laszczak seg->bounce_offs = enqd_len; 18293d829045SPawel Laszczak 1830*118b2a32SPawel Laszczak trace_cdnsp_bounce_map(preq, new_buff_len, enqd_len, seg->bounce_dma, 1831*118b2a32SPawel Laszczak unalign); 1832*118b2a32SPawel Laszczak 18333d829045SPawel Laszczak /* 18343d829045SPawel Laszczak * Bounce buffer successful aligned and seg->bounce_dma will be used 18353d829045SPawel Laszczak * in transfer TRB as new transfer buffer address. 18363d829045SPawel Laszczak */ 18373d829045SPawel Laszczak return 1; 18383d829045SPawel Laszczak } 18393d829045SPawel Laszczak 18403d829045SPawel Laszczak int cdnsp_queue_bulk_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq) 18413d829045SPawel Laszczak { 18423d829045SPawel Laszczak unsigned int enqd_len, block_len, trb_buff_len, full_len; 18433d829045SPawel Laszczak unsigned int start_cycle, num_sgs = 0; 18443d829045SPawel Laszczak struct cdnsp_generic_trb *start_trb; 18453d829045SPawel Laszczak u32 field, length_field, remainder; 18463d829045SPawel Laszczak struct scatterlist *sg = NULL; 18473d829045SPawel Laszczak bool more_trbs_coming = true; 18483d829045SPawel Laszczak bool need_zero_pkt = false; 18493d829045SPawel Laszczak bool zero_len_trb = false; 18503d829045SPawel Laszczak struct cdnsp_ring *ring; 18513d829045SPawel Laszczak bool first_trb = true; 18523d829045SPawel Laszczak unsigned int num_trbs; 18533d829045SPawel Laszczak struct cdnsp_ep *pep; 18543d829045SPawel Laszczak u64 addr, send_addr; 18553d829045SPawel Laszczak int sent_len, ret; 18563d829045SPawel Laszczak 18573d829045SPawel Laszczak ring = cdnsp_request_to_transfer_ring(pdev, preq); 18583d829045SPawel Laszczak if (!ring) 18593d829045SPawel Laszczak return -EINVAL; 18603d829045SPawel Laszczak 18613d829045SPawel Laszczak full_len = preq->request.length; 18623d829045SPawel Laszczak 18633d829045SPawel Laszczak if (preq->request.num_sgs) { 18643d829045SPawel Laszczak num_sgs = preq->request.num_sgs; 18653d829045SPawel Laszczak sg = preq->request.sg; 18663d829045SPawel Laszczak addr = (u64)sg_dma_address(sg); 18673d829045SPawel Laszczak block_len = sg_dma_len(sg); 18683d829045SPawel Laszczak num_trbs = count_sg_trbs_needed(preq); 18693d829045SPawel Laszczak } else { 18703d829045SPawel Laszczak num_trbs = count_trbs_needed(preq); 18713d829045SPawel Laszczak addr = (u64)preq->request.dma; 18723d829045SPawel Laszczak block_len = full_len; 18733d829045SPawel Laszczak } 18743d829045SPawel Laszczak 18753d829045SPawel Laszczak pep = preq->pep; 18763d829045SPawel Laszczak 18773d829045SPawel Laszczak /* Deal with request.zero - need one more td/trb. */ 18783d829045SPawel Laszczak if (preq->request.zero && preq->request.length && 18793d829045SPawel Laszczak IS_ALIGNED(full_len, usb_endpoint_maxp(pep->endpoint.desc))) { 18803d829045SPawel Laszczak need_zero_pkt = true; 18813d829045SPawel Laszczak num_trbs++; 18823d829045SPawel Laszczak } 18833d829045SPawel Laszczak 18843d829045SPawel Laszczak ret = cdnsp_prepare_transfer(pdev, preq, num_trbs); 18853d829045SPawel Laszczak if (ret) 18863d829045SPawel Laszczak return ret; 18873d829045SPawel Laszczak 18883d829045SPawel Laszczak /* 18893d829045SPawel Laszczak * Don't give the first TRB to the hardware (by toggling the cycle bit) 18903d829045SPawel Laszczak * until we've finished creating all the other TRBs. The ring's cycle 18913d829045SPawel Laszczak * state may change as we enqueue the other TRBs, so save it too. 18923d829045SPawel Laszczak */ 18933d829045SPawel Laszczak start_trb = &ring->enqueue->generic; 18943d829045SPawel Laszczak start_cycle = ring->cycle_state; 18953d829045SPawel Laszczak send_addr = addr; 18963d829045SPawel Laszczak 18973d829045SPawel Laszczak /* Queue the TRBs, even if they are zero-length */ 18983d829045SPawel Laszczak for (enqd_len = 0; zero_len_trb || first_trb || enqd_len < full_len; 18993d829045SPawel Laszczak enqd_len += trb_buff_len) { 19003d829045SPawel Laszczak field = TRB_TYPE(TRB_NORMAL); 19013d829045SPawel Laszczak 19023d829045SPawel Laszczak /* TRB buffer should not cross 64KB boundaries */ 19033d829045SPawel Laszczak trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr); 19043d829045SPawel Laszczak trb_buff_len = min(trb_buff_len, block_len); 19053d829045SPawel Laszczak if (enqd_len + trb_buff_len > full_len) 19063d829045SPawel Laszczak trb_buff_len = full_len - enqd_len; 19073d829045SPawel Laszczak 19083d829045SPawel Laszczak /* Don't change the cycle bit of the first TRB until later */ 19093d829045SPawel Laszczak if (first_trb) { 19103d829045SPawel Laszczak first_trb = false; 19113d829045SPawel Laszczak if (start_cycle == 0) 19123d829045SPawel Laszczak field |= TRB_CYCLE; 19133d829045SPawel Laszczak } else { 19143d829045SPawel Laszczak field |= ring->cycle_state; 19153d829045SPawel Laszczak } 19163d829045SPawel Laszczak 19173d829045SPawel Laszczak /* 19183d829045SPawel Laszczak * Chain all the TRBs together; clear the chain bit in the last 19193d829045SPawel Laszczak * TRB to indicate it's the last TRB in the chain. 19203d829045SPawel Laszczak */ 19213d829045SPawel Laszczak if (enqd_len + trb_buff_len < full_len || need_zero_pkt) { 19223d829045SPawel Laszczak field |= TRB_CHAIN; 19233d829045SPawel Laszczak if (cdnsp_trb_is_link(ring->enqueue + 1)) { 19243d829045SPawel Laszczak if (cdnsp_align_td(pdev, preq, enqd_len, 19253d829045SPawel Laszczak &trb_buff_len, 19263d829045SPawel Laszczak ring->enq_seg)) { 19273d829045SPawel Laszczak send_addr = ring->enq_seg->bounce_dma; 19283d829045SPawel Laszczak /* Assuming TD won't span 2 segs */ 19293d829045SPawel Laszczak preq->td.bounce_seg = ring->enq_seg; 19303d829045SPawel Laszczak } 19313d829045SPawel Laszczak } 19323d829045SPawel Laszczak } 19333d829045SPawel Laszczak 19343d829045SPawel Laszczak if (enqd_len + trb_buff_len >= full_len) { 19353d829045SPawel Laszczak if (need_zero_pkt && zero_len_trb) { 19363d829045SPawel Laszczak zero_len_trb = true; 19373d829045SPawel Laszczak } else { 19383d829045SPawel Laszczak field &= ~TRB_CHAIN; 19393d829045SPawel Laszczak field |= TRB_IOC; 19403d829045SPawel Laszczak more_trbs_coming = false; 19413d829045SPawel Laszczak need_zero_pkt = false; 19423d829045SPawel Laszczak preq->td.last_trb = ring->enqueue; 19433d829045SPawel Laszczak } 19443d829045SPawel Laszczak } 19453d829045SPawel Laszczak 19463d829045SPawel Laszczak /* Only set interrupt on short packet for OUT endpoints. */ 19473d829045SPawel Laszczak if (!preq->direction) 19483d829045SPawel Laszczak field |= TRB_ISP; 19493d829045SPawel Laszczak 19503d829045SPawel Laszczak /* Set the TRB length, TD size, and interrupter fields. */ 19513d829045SPawel Laszczak remainder = cdnsp_td_remainder(pdev, enqd_len, trb_buff_len, 19523d829045SPawel Laszczak full_len, preq, 19533d829045SPawel Laszczak more_trbs_coming); 19543d829045SPawel Laszczak 19553d829045SPawel Laszczak length_field = TRB_LEN(trb_buff_len) | TRB_TD_SIZE(remainder) | 19563d829045SPawel Laszczak TRB_INTR_TARGET(0); 19573d829045SPawel Laszczak 19583d829045SPawel Laszczak cdnsp_queue_trb(pdev, ring, more_trbs_coming | need_zero_pkt, 19593d829045SPawel Laszczak lower_32_bits(send_addr), 19603d829045SPawel Laszczak upper_32_bits(send_addr), 19613d829045SPawel Laszczak length_field, 19623d829045SPawel Laszczak field); 19633d829045SPawel Laszczak 19643d829045SPawel Laszczak addr += trb_buff_len; 19653d829045SPawel Laszczak sent_len = trb_buff_len; 19663d829045SPawel Laszczak while (sg && sent_len >= block_len) { 19673d829045SPawel Laszczak /* New sg entry */ 19683d829045SPawel Laszczak --num_sgs; 19693d829045SPawel Laszczak sent_len -= block_len; 19703d829045SPawel Laszczak if (num_sgs != 0) { 19713d829045SPawel Laszczak sg = sg_next(sg); 19723d829045SPawel Laszczak block_len = sg_dma_len(sg); 19733d829045SPawel Laszczak addr = (u64)sg_dma_address(sg); 19743d829045SPawel Laszczak addr += sent_len; 19753d829045SPawel Laszczak } 19763d829045SPawel Laszczak } 19773d829045SPawel Laszczak block_len -= sent_len; 19783d829045SPawel Laszczak send_addr = addr; 19793d829045SPawel Laszczak } 19803d829045SPawel Laszczak 19813d829045SPawel Laszczak cdnsp_check_trb_math(preq, enqd_len); 19823d829045SPawel Laszczak ret = cdnsp_giveback_first_trb(pdev, pep, preq->request.stream_id, 19833d829045SPawel Laszczak start_cycle, start_trb); 19843d829045SPawel Laszczak 19853d829045SPawel Laszczak if (ret) 19863d829045SPawel Laszczak preq->td.drbl = 1; 19873d829045SPawel Laszczak 19883d829045SPawel Laszczak return 0; 19893d829045SPawel Laszczak } 19903d829045SPawel Laszczak 19913d829045SPawel Laszczak int cdnsp_queue_ctrl_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq) 19923d829045SPawel Laszczak { 19933d829045SPawel Laszczak u32 field, length_field, remainder; 19943d829045SPawel Laszczak struct cdnsp_ep *pep = preq->pep; 19953d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 19963d829045SPawel Laszczak int num_trbs; 19973d829045SPawel Laszczak int ret; 19983d829045SPawel Laszczak 19993d829045SPawel Laszczak ep_ring = cdnsp_request_to_transfer_ring(pdev, preq); 20003d829045SPawel Laszczak if (!ep_ring) 20013d829045SPawel Laszczak return -EINVAL; 20023d829045SPawel Laszczak 20033d829045SPawel Laszczak /* 1 TRB for data, 1 for status */ 20043d829045SPawel Laszczak num_trbs = (pdev->three_stage_setup) ? 2 : 1; 20053d829045SPawel Laszczak 20063d829045SPawel Laszczak ret = cdnsp_prepare_transfer(pdev, preq, num_trbs); 20073d829045SPawel Laszczak if (ret) 20083d829045SPawel Laszczak return ret; 20093d829045SPawel Laszczak 20103d829045SPawel Laszczak /* If there's data, queue data TRBs */ 20113d829045SPawel Laszczak if (pdev->ep0_expect_in) 20123d829045SPawel Laszczak field = TRB_TYPE(TRB_DATA) | TRB_IOC; 20133d829045SPawel Laszczak else 20143d829045SPawel Laszczak field = TRB_ISP | TRB_TYPE(TRB_DATA) | TRB_IOC; 20153d829045SPawel Laszczak 20163d829045SPawel Laszczak if (preq->request.length > 0) { 20173d829045SPawel Laszczak remainder = cdnsp_td_remainder(pdev, 0, preq->request.length, 20183d829045SPawel Laszczak preq->request.length, preq, 1); 20193d829045SPawel Laszczak 20203d829045SPawel Laszczak length_field = TRB_LEN(preq->request.length) | 20213d829045SPawel Laszczak TRB_TD_SIZE(remainder) | TRB_INTR_TARGET(0); 20223d829045SPawel Laszczak 20233d829045SPawel Laszczak if (pdev->ep0_expect_in) 20243d829045SPawel Laszczak field |= TRB_DIR_IN; 20253d829045SPawel Laszczak 20263d829045SPawel Laszczak cdnsp_queue_trb(pdev, ep_ring, true, 20273d829045SPawel Laszczak lower_32_bits(preq->request.dma), 20283d829045SPawel Laszczak upper_32_bits(preq->request.dma), length_field, 20293d829045SPawel Laszczak field | ep_ring->cycle_state | 20303d829045SPawel Laszczak TRB_SETUPID(pdev->setup_id) | 20313d829045SPawel Laszczak pdev->setup_speed); 20323d829045SPawel Laszczak 20333d829045SPawel Laszczak pdev->ep0_stage = CDNSP_DATA_STAGE; 20343d829045SPawel Laszczak } 20353d829045SPawel Laszczak 20363d829045SPawel Laszczak /* Save the DMA address of the last TRB in the TD. */ 20373d829045SPawel Laszczak preq->td.last_trb = ep_ring->enqueue; 20383d829045SPawel Laszczak 20393d829045SPawel Laszczak /* Queue status TRB. */ 20403d829045SPawel Laszczak if (preq->request.length == 0) 20413d829045SPawel Laszczak field = ep_ring->cycle_state; 20423d829045SPawel Laszczak else 20433d829045SPawel Laszczak field = (ep_ring->cycle_state ^ 1); 20443d829045SPawel Laszczak 20453d829045SPawel Laszczak if (preq->request.length > 0 && pdev->ep0_expect_in) 20463d829045SPawel Laszczak field |= TRB_DIR_IN; 20473d829045SPawel Laszczak 20483d829045SPawel Laszczak if (pep->ep_state & EP0_HALTED_STATUS) { 20493d829045SPawel Laszczak pep->ep_state &= ~EP0_HALTED_STATUS; 20503d829045SPawel Laszczak field |= TRB_SETUPSTAT(TRB_SETUPSTAT_STALL); 20513d829045SPawel Laszczak } else { 20523d829045SPawel Laszczak field |= TRB_SETUPSTAT(TRB_SETUPSTAT_ACK); 20533d829045SPawel Laszczak } 20543d829045SPawel Laszczak 20553d829045SPawel Laszczak cdnsp_queue_trb(pdev, ep_ring, false, 0, 0, TRB_INTR_TARGET(0), 20563d829045SPawel Laszczak field | TRB_IOC | TRB_SETUPID(pdev->setup_id) | 20573d829045SPawel Laszczak TRB_TYPE(TRB_STATUS) | pdev->setup_speed); 20583d829045SPawel Laszczak 20593d829045SPawel Laszczak cdnsp_ring_ep_doorbell(pdev, pep, preq->request.stream_id); 20603d829045SPawel Laszczak 20613d829045SPawel Laszczak return 0; 20623d829045SPawel Laszczak } 20633d829045SPawel Laszczak 20643d829045SPawel Laszczak int cdnsp_cmd_stop_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep) 20653d829045SPawel Laszczak { 20663d829045SPawel Laszczak u32 ep_state = GET_EP_CTX_STATE(pep->out_ctx); 20673d829045SPawel Laszczak int ret = 0; 20683d829045SPawel Laszczak 2069*118b2a32SPawel Laszczak if (ep_state == EP_STATE_STOPPED || ep_state == EP_STATE_DISABLED) { 2070*118b2a32SPawel Laszczak trace_cdnsp_ep_stopped_or_disabled(pep->out_ctx); 20713d829045SPawel Laszczak goto ep_stopped; 2072*118b2a32SPawel Laszczak } 20733d829045SPawel Laszczak 20743d829045SPawel Laszczak cdnsp_queue_stop_endpoint(pdev, pep->idx); 20753d829045SPawel Laszczak cdnsp_ring_cmd_db(pdev); 20763d829045SPawel Laszczak ret = cdnsp_wait_for_cmd_compl(pdev); 20773d829045SPawel Laszczak 2078*118b2a32SPawel Laszczak trace_cdnsp_handle_cmd_stop_ep(pep->out_ctx); 2079*118b2a32SPawel Laszczak 20803d829045SPawel Laszczak ep_stopped: 20813d829045SPawel Laszczak pep->ep_state |= EP_STOPPED; 20823d829045SPawel Laszczak return ret; 20833d829045SPawel Laszczak } 20843d829045SPawel Laszczak 20853d829045SPawel Laszczak int cdnsp_cmd_flush_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep) 20863d829045SPawel Laszczak { 20873d829045SPawel Laszczak int ret; 20883d829045SPawel Laszczak 20893d829045SPawel Laszczak cdnsp_queue_flush_endpoint(pdev, pep->idx); 20903d829045SPawel Laszczak cdnsp_ring_cmd_db(pdev); 20913d829045SPawel Laszczak ret = cdnsp_wait_for_cmd_compl(pdev); 20923d829045SPawel Laszczak 2093*118b2a32SPawel Laszczak trace_cdnsp_handle_cmd_flush_ep(pep->out_ctx); 2094*118b2a32SPawel Laszczak 20953d829045SPawel Laszczak return ret; 20963d829045SPawel Laszczak } 20973d829045SPawel Laszczak 20983d829045SPawel Laszczak /* 20993d829045SPawel Laszczak * The transfer burst count field of the isochronous TRB defines the number of 21003d829045SPawel Laszczak * bursts that are required to move all packets in this TD. Only SuperSpeed 21013d829045SPawel Laszczak * devices can burst up to bMaxBurst number of packets per service interval. 21023d829045SPawel Laszczak * This field is zero based, meaning a value of zero in the field means one 21033d829045SPawel Laszczak * burst. Basically, for everything but SuperSpeed devices, this field will be 21043d829045SPawel Laszczak * zero. 21053d829045SPawel Laszczak */ 21063d829045SPawel Laszczak static unsigned int cdnsp_get_burst_count(struct cdnsp_device *pdev, 21073d829045SPawel Laszczak struct cdnsp_request *preq, 21083d829045SPawel Laszczak unsigned int total_packet_count) 21093d829045SPawel Laszczak { 21103d829045SPawel Laszczak unsigned int max_burst; 21113d829045SPawel Laszczak 21123d829045SPawel Laszczak if (pdev->gadget.speed < USB_SPEED_SUPER) 21133d829045SPawel Laszczak return 0; 21143d829045SPawel Laszczak 21153d829045SPawel Laszczak max_burst = preq->pep->endpoint.comp_desc->bMaxBurst; 21163d829045SPawel Laszczak return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1; 21173d829045SPawel Laszczak } 21183d829045SPawel Laszczak 21193d829045SPawel Laszczak /* 21203d829045SPawel Laszczak * Returns the number of packets in the last "burst" of packets. This field is 21213d829045SPawel Laszczak * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so 21223d829045SPawel Laszczak * the last burst packet count is equal to the total number of packets in the 21233d829045SPawel Laszczak * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst 21243d829045SPawel Laszczak * must contain (bMaxBurst + 1) number of packets, but the last burst can 21253d829045SPawel Laszczak * contain 1 to (bMaxBurst + 1) packets. 21263d829045SPawel Laszczak */ 21273d829045SPawel Laszczak static unsigned int 21283d829045SPawel Laszczak cdnsp_get_last_burst_packet_count(struct cdnsp_device *pdev, 21293d829045SPawel Laszczak struct cdnsp_request *preq, 21303d829045SPawel Laszczak unsigned int total_packet_count) 21313d829045SPawel Laszczak { 21323d829045SPawel Laszczak unsigned int max_burst; 21333d829045SPawel Laszczak unsigned int residue; 21343d829045SPawel Laszczak 21353d829045SPawel Laszczak if (pdev->gadget.speed >= USB_SPEED_SUPER) { 21363d829045SPawel Laszczak /* bMaxBurst is zero based: 0 means 1 packet per burst. */ 21373d829045SPawel Laszczak max_burst = preq->pep->endpoint.comp_desc->bMaxBurst; 21383d829045SPawel Laszczak residue = total_packet_count % (max_burst + 1); 21393d829045SPawel Laszczak 21403d829045SPawel Laszczak /* 21413d829045SPawel Laszczak * If residue is zero, the last burst contains (max_burst + 1) 21423d829045SPawel Laszczak * number of packets, but the TLBPC field is zero-based. 21433d829045SPawel Laszczak */ 21443d829045SPawel Laszczak if (residue == 0) 21453d829045SPawel Laszczak return max_burst; 21463d829045SPawel Laszczak 21473d829045SPawel Laszczak return residue - 1; 21483d829045SPawel Laszczak } 21493d829045SPawel Laszczak if (total_packet_count == 0) 21503d829045SPawel Laszczak return 0; 21513d829045SPawel Laszczak 21523d829045SPawel Laszczak return total_packet_count - 1; 21533d829045SPawel Laszczak } 21543d829045SPawel Laszczak 21553d829045SPawel Laszczak /* Queue function isoc transfer */ 21563d829045SPawel Laszczak static int cdnsp_queue_isoc_tx(struct cdnsp_device *pdev, 21573d829045SPawel Laszczak struct cdnsp_request *preq) 21583d829045SPawel Laszczak { 21593d829045SPawel Laszczak int trb_buff_len, td_len, td_remain_len, ret; 21603d829045SPawel Laszczak unsigned int burst_count, last_burst_pkt; 21613d829045SPawel Laszczak unsigned int total_pkt_count, max_pkt; 21623d829045SPawel Laszczak struct cdnsp_generic_trb *start_trb; 21633d829045SPawel Laszczak bool more_trbs_coming = true; 21643d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 21653d829045SPawel Laszczak int running_total = 0; 21663d829045SPawel Laszczak u32 field, length_field; 21673d829045SPawel Laszczak int start_cycle; 21683d829045SPawel Laszczak int trbs_per_td; 21693d829045SPawel Laszczak u64 addr; 21703d829045SPawel Laszczak int i; 21713d829045SPawel Laszczak 21723d829045SPawel Laszczak ep_ring = preq->pep->ring; 21733d829045SPawel Laszczak start_trb = &ep_ring->enqueue->generic; 21743d829045SPawel Laszczak start_cycle = ep_ring->cycle_state; 21753d829045SPawel Laszczak td_len = preq->request.length; 21763d829045SPawel Laszczak addr = (u64)preq->request.dma; 21773d829045SPawel Laszczak td_remain_len = td_len; 21783d829045SPawel Laszczak 21793d829045SPawel Laszczak max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc); 21803d829045SPawel Laszczak total_pkt_count = DIV_ROUND_UP(td_len, max_pkt); 21813d829045SPawel Laszczak 21823d829045SPawel Laszczak /* A zero-length transfer still involves at least one packet. */ 21833d829045SPawel Laszczak if (total_pkt_count == 0) 21843d829045SPawel Laszczak total_pkt_count++; 21853d829045SPawel Laszczak 21863d829045SPawel Laszczak burst_count = cdnsp_get_burst_count(pdev, preq, total_pkt_count); 21873d829045SPawel Laszczak last_burst_pkt = cdnsp_get_last_burst_packet_count(pdev, preq, 21883d829045SPawel Laszczak total_pkt_count); 21893d829045SPawel Laszczak trbs_per_td = count_isoc_trbs_needed(preq); 21903d829045SPawel Laszczak 21913d829045SPawel Laszczak ret = cdnsp_prepare_transfer(pdev, preq, trbs_per_td); 21923d829045SPawel Laszczak if (ret) 21933d829045SPawel Laszczak goto cleanup; 21943d829045SPawel Laszczak 21953d829045SPawel Laszczak /* 21963d829045SPawel Laszczak * Set isoc specific data for the first TRB in a TD. 21973d829045SPawel Laszczak * Prevent HW from getting the TRBs by keeping the cycle state 21983d829045SPawel Laszczak * inverted in the first TDs isoc TRB. 21993d829045SPawel Laszczak */ 22003d829045SPawel Laszczak field = TRB_TYPE(TRB_ISOC) | TRB_TLBPC(last_burst_pkt) | 22013d829045SPawel Laszczak !start_cycle | TRB_SIA | TRB_TBC(burst_count); 22023d829045SPawel Laszczak 22033d829045SPawel Laszczak /* Fill the rest of the TRB fields, and remaining normal TRBs. */ 22043d829045SPawel Laszczak for (i = 0; i < trbs_per_td; i++) { 22053d829045SPawel Laszczak u32 remainder; 22063d829045SPawel Laszczak 22073d829045SPawel Laszczak /* Calculate TRB length. */ 22083d829045SPawel Laszczak trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr); 22093d829045SPawel Laszczak if (trb_buff_len > td_remain_len) 22103d829045SPawel Laszczak trb_buff_len = td_remain_len; 22113d829045SPawel Laszczak 22123d829045SPawel Laszczak /* Set the TRB length, TD size, & interrupter fields. */ 22133d829045SPawel Laszczak remainder = cdnsp_td_remainder(pdev, running_total, 22143d829045SPawel Laszczak trb_buff_len, td_len, preq, 22153d829045SPawel Laszczak more_trbs_coming); 22163d829045SPawel Laszczak 22173d829045SPawel Laszczak length_field = TRB_LEN(trb_buff_len) | TRB_INTR_TARGET(0); 22183d829045SPawel Laszczak 22193d829045SPawel Laszczak /* Only first TRB is isoc, overwrite otherwise. */ 22203d829045SPawel Laszczak if (i) { 22213d829045SPawel Laszczak field = TRB_TYPE(TRB_NORMAL) | ep_ring->cycle_state; 22223d829045SPawel Laszczak length_field |= TRB_TD_SIZE(remainder); 22233d829045SPawel Laszczak } else { 22243d829045SPawel Laszczak length_field |= TRB_TD_SIZE_TBC(burst_count); 22253d829045SPawel Laszczak } 22263d829045SPawel Laszczak 22273d829045SPawel Laszczak /* Only set interrupt on short packet for OUT EPs. */ 22283d829045SPawel Laszczak if (usb_endpoint_dir_out(preq->pep->endpoint.desc)) 22293d829045SPawel Laszczak field |= TRB_ISP; 22303d829045SPawel Laszczak 22313d829045SPawel Laszczak /* Set the chain bit for all except the last TRB. */ 22323d829045SPawel Laszczak if (i < trbs_per_td - 1) { 22333d829045SPawel Laszczak more_trbs_coming = true; 22343d829045SPawel Laszczak field |= TRB_CHAIN; 22353d829045SPawel Laszczak } else { 22363d829045SPawel Laszczak more_trbs_coming = false; 22373d829045SPawel Laszczak preq->td.last_trb = ep_ring->enqueue; 22383d829045SPawel Laszczak field |= TRB_IOC; 22393d829045SPawel Laszczak } 22403d829045SPawel Laszczak 22413d829045SPawel Laszczak cdnsp_queue_trb(pdev, ep_ring, more_trbs_coming, 22423d829045SPawel Laszczak lower_32_bits(addr), upper_32_bits(addr), 22433d829045SPawel Laszczak length_field, field); 22443d829045SPawel Laszczak 22453d829045SPawel Laszczak running_total += trb_buff_len; 22463d829045SPawel Laszczak addr += trb_buff_len; 22473d829045SPawel Laszczak td_remain_len -= trb_buff_len; 22483d829045SPawel Laszczak } 22493d829045SPawel Laszczak 22503d829045SPawel Laszczak /* Check TD length */ 22513d829045SPawel Laszczak if (running_total != td_len) { 22523d829045SPawel Laszczak dev_err(pdev->dev, "ISOC TD length unmatch\n"); 22533d829045SPawel Laszczak ret = -EINVAL; 22543d829045SPawel Laszczak goto cleanup; 22553d829045SPawel Laszczak } 22563d829045SPawel Laszczak 22573d829045SPawel Laszczak cdnsp_giveback_first_trb(pdev, preq->pep, preq->request.stream_id, 22583d829045SPawel Laszczak start_cycle, start_trb); 22593d829045SPawel Laszczak 22603d829045SPawel Laszczak return 0; 22613d829045SPawel Laszczak 22623d829045SPawel Laszczak cleanup: 22633d829045SPawel Laszczak /* Clean up a partially enqueued isoc transfer. */ 22643d829045SPawel Laszczak list_del_init(&preq->td.td_list); 22653d829045SPawel Laszczak ep_ring->num_tds--; 22663d829045SPawel Laszczak 22673d829045SPawel Laszczak /* 22683d829045SPawel Laszczak * Use the first TD as a temporary variable to turn the TDs we've 22693d829045SPawel Laszczak * queued into No-ops with a software-owned cycle bit. 22703d829045SPawel Laszczak * That way the hardware won't accidentally start executing bogus TDs 22713d829045SPawel Laszczak * when we partially overwrite them. 22723d829045SPawel Laszczak * td->first_trb and td->start_seg are already set. 22733d829045SPawel Laszczak */ 22743d829045SPawel Laszczak preq->td.last_trb = ep_ring->enqueue; 22753d829045SPawel Laszczak /* Every TRB except the first & last will have its cycle bit flipped. */ 22763d829045SPawel Laszczak cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true); 22773d829045SPawel Laszczak 22783d829045SPawel Laszczak /* Reset the ring enqueue back to the first TRB and its cycle bit. */ 22793d829045SPawel Laszczak ep_ring->enqueue = preq->td.first_trb; 22803d829045SPawel Laszczak ep_ring->enq_seg = preq->td.start_seg; 22813d829045SPawel Laszczak ep_ring->cycle_state = start_cycle; 22823d829045SPawel Laszczak return ret; 22833d829045SPawel Laszczak } 22843d829045SPawel Laszczak 22853d829045SPawel Laszczak int cdnsp_queue_isoc_tx_prepare(struct cdnsp_device *pdev, 22863d829045SPawel Laszczak struct cdnsp_request *preq) 22873d829045SPawel Laszczak { 22883d829045SPawel Laszczak struct cdnsp_ring *ep_ring; 22893d829045SPawel Laszczak u32 ep_state; 22903d829045SPawel Laszczak int num_trbs; 22913d829045SPawel Laszczak int ret; 22923d829045SPawel Laszczak 22933d829045SPawel Laszczak ep_ring = preq->pep->ring; 22943d829045SPawel Laszczak ep_state = GET_EP_CTX_STATE(preq->pep->out_ctx); 22953d829045SPawel Laszczak num_trbs = count_isoc_trbs_needed(preq); 22963d829045SPawel Laszczak 22973d829045SPawel Laszczak /* 22983d829045SPawel Laszczak * Check the ring to guarantee there is enough room for the whole 22993d829045SPawel Laszczak * request. Do not insert any td of the USB Request to the ring if the 23003d829045SPawel Laszczak * check failed. 23013d829045SPawel Laszczak */ 23023d829045SPawel Laszczak ret = cdnsp_prepare_ring(pdev, ep_ring, ep_state, num_trbs, GFP_ATOMIC); 23033d829045SPawel Laszczak if (ret) 23043d829045SPawel Laszczak return ret; 23053d829045SPawel Laszczak 23063d829045SPawel Laszczak return cdnsp_queue_isoc_tx(pdev, preq); 23073d829045SPawel Laszczak } 23083d829045SPawel Laszczak 23093d829045SPawel Laszczak /**** Command Ring Operations ****/ 23103d829045SPawel Laszczak /* 23113d829045SPawel Laszczak * Generic function for queuing a command TRB on the command ring. 23123d829045SPawel Laszczak * Driver queue only one command to ring in the moment. 23133d829045SPawel Laszczak */ 23143d829045SPawel Laszczak static void cdnsp_queue_command(struct cdnsp_device *pdev, 23153d829045SPawel Laszczak u32 field1, 23163d829045SPawel Laszczak u32 field2, 23173d829045SPawel Laszczak u32 field3, 23183d829045SPawel Laszczak u32 field4) 23193d829045SPawel Laszczak { 23203d829045SPawel Laszczak cdnsp_prepare_ring(pdev, pdev->cmd_ring, EP_STATE_RUNNING, 1, 23213d829045SPawel Laszczak GFP_ATOMIC); 23223d829045SPawel Laszczak 23233d829045SPawel Laszczak pdev->cmd.command_trb = pdev->cmd_ring->enqueue; 23243d829045SPawel Laszczak 23253d829045SPawel Laszczak cdnsp_queue_trb(pdev, pdev->cmd_ring, false, field1, field2, 23263d829045SPawel Laszczak field3, field4 | pdev->cmd_ring->cycle_state); 23273d829045SPawel Laszczak } 23283d829045SPawel Laszczak 23293d829045SPawel Laszczak /* Queue a slot enable or disable request on the command ring */ 23303d829045SPawel Laszczak void cdnsp_queue_slot_control(struct cdnsp_device *pdev, u32 trb_type) 23313d829045SPawel Laszczak { 23323d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(trb_type) | 23333d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id)); 23343d829045SPawel Laszczak } 23353d829045SPawel Laszczak 23363d829045SPawel Laszczak /* Queue an address device command TRB */ 23373d829045SPawel Laszczak void cdnsp_queue_address_device(struct cdnsp_device *pdev, 23383d829045SPawel Laszczak dma_addr_t in_ctx_ptr, 23393d829045SPawel Laszczak enum cdnsp_setup_dev setup) 23403d829045SPawel Laszczak { 23413d829045SPawel Laszczak cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr), 23423d829045SPawel Laszczak upper_32_bits(in_ctx_ptr), 0, 23433d829045SPawel Laszczak TRB_TYPE(TRB_ADDR_DEV) | 23443d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id) | 23453d829045SPawel Laszczak (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0)); 23463d829045SPawel Laszczak } 23473d829045SPawel Laszczak 23483d829045SPawel Laszczak /* Queue a reset device command TRB */ 23493d829045SPawel Laszczak void cdnsp_queue_reset_device(struct cdnsp_device *pdev) 23503d829045SPawel Laszczak { 23513d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_RESET_DEV) | 23523d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id)); 23533d829045SPawel Laszczak } 23543d829045SPawel Laszczak 23553d829045SPawel Laszczak /* Queue a configure endpoint command TRB */ 23563d829045SPawel Laszczak void cdnsp_queue_configure_endpoint(struct cdnsp_device *pdev, 23573d829045SPawel Laszczak dma_addr_t in_ctx_ptr) 23583d829045SPawel Laszczak { 23593d829045SPawel Laszczak cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr), 23603d829045SPawel Laszczak upper_32_bits(in_ctx_ptr), 0, 23613d829045SPawel Laszczak TRB_TYPE(TRB_CONFIG_EP) | 23623d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id)); 23633d829045SPawel Laszczak } 23643d829045SPawel Laszczak 23653d829045SPawel Laszczak /* 23663d829045SPawel Laszczak * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop 23673d829045SPawel Laszczak * activity on an endpoint that is about to be suspended. 23683d829045SPawel Laszczak */ 23693d829045SPawel Laszczak void cdnsp_queue_stop_endpoint(struct cdnsp_device *pdev, unsigned int ep_index) 23703d829045SPawel Laszczak { 23713d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, SLOT_ID_FOR_TRB(pdev->slot_id) | 23723d829045SPawel Laszczak EP_ID_FOR_TRB(ep_index) | TRB_TYPE(TRB_STOP_RING)); 23733d829045SPawel Laszczak } 23743d829045SPawel Laszczak 23753d829045SPawel Laszczak /* Set Transfer Ring Dequeue Pointer command. */ 23763d829045SPawel Laszczak void cdnsp_queue_new_dequeue_state(struct cdnsp_device *pdev, 23773d829045SPawel Laszczak struct cdnsp_ep *pep, 23783d829045SPawel Laszczak struct cdnsp_dequeue_state *deq_state) 23793d829045SPawel Laszczak { 23803d829045SPawel Laszczak u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id); 23813d829045SPawel Laszczak u32 trb_slot_id = SLOT_ID_FOR_TRB(pdev->slot_id); 23823d829045SPawel Laszczak u32 type = TRB_TYPE(TRB_SET_DEQ); 23833d829045SPawel Laszczak u32 trb_sct = 0; 23843d829045SPawel Laszczak dma_addr_t addr; 23853d829045SPawel Laszczak 23863d829045SPawel Laszczak addr = cdnsp_trb_virt_to_dma(deq_state->new_deq_seg, 23873d829045SPawel Laszczak deq_state->new_deq_ptr); 23883d829045SPawel Laszczak 23893d829045SPawel Laszczak if (deq_state->stream_id) 23903d829045SPawel Laszczak trb_sct = SCT_FOR_TRB(SCT_PRI_TR); 23913d829045SPawel Laszczak 23923d829045SPawel Laszczak cdnsp_queue_command(pdev, lower_32_bits(addr) | trb_sct | 23933d829045SPawel Laszczak deq_state->new_cycle_state, upper_32_bits(addr), 23943d829045SPawel Laszczak trb_stream_id, trb_slot_id | 23953d829045SPawel Laszczak EP_ID_FOR_TRB(pep->idx) | type); 23963d829045SPawel Laszczak } 23973d829045SPawel Laszczak 23983d829045SPawel Laszczak void cdnsp_queue_reset_ep(struct cdnsp_device *pdev, unsigned int ep_index) 23993d829045SPawel Laszczak { 24003d829045SPawel Laszczak return cdnsp_queue_command(pdev, 0, 0, 0, 24013d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id) | 24023d829045SPawel Laszczak EP_ID_FOR_TRB(ep_index) | 24033d829045SPawel Laszczak TRB_TYPE(TRB_RESET_EP)); 24043d829045SPawel Laszczak } 24053d829045SPawel Laszczak 24063d829045SPawel Laszczak /* 24073d829045SPawel Laszczak * Queue a halt endpoint request on the command ring. 24083d829045SPawel Laszczak */ 24093d829045SPawel Laszczak void cdnsp_queue_halt_endpoint(struct cdnsp_device *pdev, unsigned int ep_index) 24103d829045SPawel Laszczak { 24113d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_HALT_ENDPOINT) | 24123d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id) | 24133d829045SPawel Laszczak EP_ID_FOR_TRB(ep_index)); 24143d829045SPawel Laszczak } 24153d829045SPawel Laszczak 24163d829045SPawel Laszczak /* 24173d829045SPawel Laszczak * Queue a flush endpoint request on the command ring. 24183d829045SPawel Laszczak */ 24193d829045SPawel Laszczak void cdnsp_queue_flush_endpoint(struct cdnsp_device *pdev, 24203d829045SPawel Laszczak unsigned int ep_index) 24213d829045SPawel Laszczak { 24223d829045SPawel Laszczak cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_FLUSH_ENDPOINT) | 24233d829045SPawel Laszczak SLOT_ID_FOR_TRB(pdev->slot_id) | 24243d829045SPawel Laszczak EP_ID_FOR_TRB(ep_index)); 24253d829045SPawel Laszczak } 24263d829045SPawel Laszczak 24273d829045SPawel Laszczak void cdnsp_force_header_wakeup(struct cdnsp_device *pdev, int intf_num) 24283d829045SPawel Laszczak { 24293d829045SPawel Laszczak u32 lo, mid; 24303d829045SPawel Laszczak 24313d829045SPawel Laszczak lo = TRB_FH_TO_PACKET_TYPE(TRB_FH_TR_PACKET) | 24323d829045SPawel Laszczak TRB_FH_TO_DEVICE_ADDRESS(pdev->device_address); 24333d829045SPawel Laszczak mid = TRB_FH_TR_PACKET_DEV_NOT | 24343d829045SPawel Laszczak TRB_FH_TO_NOT_TYPE(TRB_FH_TR_PACKET_FUNCTION_WAKE) | 24353d829045SPawel Laszczak TRB_FH_TO_INTERFACE(intf_num); 24363d829045SPawel Laszczak 24373d829045SPawel Laszczak cdnsp_queue_command(pdev, lo, mid, 0, 24383d829045SPawel Laszczak TRB_TYPE(TRB_FORCE_HEADER) | SET_PORT_ID(2)); 24393d829045SPawel Laszczak } 2440