Lines Matching +full:ring +full:- +full:disable +full:- +full:pullup
1 // SPDX-License-Identifier: GPL-2.0
3 * Cadence USBHS-DEV Driver - gadget side.
28 #include <linux/dma-mapping.h>
35 #include "cdns2-gadget.h"
36 #include "cdns2-trace.h"
39 * set_reg_bit_32 - set bit in given 32 bits register.
50 * clear_reg_bit_32 - clear bit in given 32 bits register.
79 dma_index = readl(&pdev->adma_regs->ep_traddr) - pep->ring.dma; in cdns2_get_dma_pos()
92 if (pdev->selected_ep == ep) in cdns2_select_ep()
95 pdev->selected_ep = ep; in cdns2_select_ep()
96 writel(ep, &pdev->adma_regs->ep_sel); in cdns2_select_ep()
102 u32 offset = (char *)trb - (char *)pep->ring.trbs; in cdns2_trb_virt_to_dma()
104 return pep->ring.dma + offset; in cdns2_trb_virt_to_dma()
109 struct cdns2_device *pdev = pep->pdev; in cdns2_free_tr_segment()
110 struct cdns2_ring *ring = &pep->ring; in cdns2_free_tr_segment() local
112 if (pep->ring.trbs) { in cdns2_free_tr_segment()
113 dma_pool_free(pdev->eps_dma_pool, ring->trbs, ring->dma); in cdns2_free_tr_segment()
114 memset(ring, 0, sizeof(*ring)); in cdns2_free_tr_segment()
118 /* Allocates Transfer Ring segment. */
121 struct cdns2_device *pdev = pep->pdev; in cdns2_alloc_tr_segment()
123 struct cdns2_ring *ring; in cdns2_alloc_tr_segment() local
125 ring = &pep->ring; in cdns2_alloc_tr_segment()
127 if (!ring->trbs) { in cdns2_alloc_tr_segment()
128 ring->trbs = dma_pool_alloc(pdev->eps_dma_pool, in cdns2_alloc_tr_segment()
130 &ring->dma); in cdns2_alloc_tr_segment()
131 if (!ring->trbs) in cdns2_alloc_tr_segment()
132 return -ENOMEM; in cdns2_alloc_tr_segment()
135 memset(ring->trbs, 0, TR_SEG_SIZE); in cdns2_alloc_tr_segment()
137 if (!pep->num) in cdns2_alloc_tr_segment()
141 link_trb = (ring->trbs + (TRBS_PER_SEGMENT - 1)); in cdns2_alloc_tr_segment()
142 link_trb->buffer = cpu_to_le32(TRB_BUFFER(ring->dma)); in cdns2_alloc_tr_segment()
143 link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | in cdns2_alloc_tr_segment()
155 struct cdns2_device *pdev = pep->pdev; in cdns2_ep_stall_flush()
160 writel(DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd); in cdns2_ep_stall_flush()
163 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val, in cdns2_ep_stall_flush()
165 pep->ep_state |= EP_STALLED; in cdns2_ep_stall_flush()
166 pep->ep_state &= ~EP_STALL_PENDING; in cdns2_ep_stall_flush()
179 if (*index == (trb_in_seg - 1)) { in cdns2_ep_inc_trb()
185 static void cdns2_ep_inc_enq(struct cdns2_ring *ring) in cdns2_ep_inc_enq() argument
187 ring->free_trbs--; in cdns2_ep_inc_enq()
188 cdns2_ep_inc_trb(&ring->enqueue, &ring->pcs, TRBS_PER_SEGMENT); in cdns2_ep_inc_enq()
191 static void cdns2_ep_inc_deq(struct cdns2_ring *ring) in cdns2_ep_inc_deq() argument
193 ring->free_trbs++; in cdns2_ep_inc_deq()
194 cdns2_ep_inc_trb(&ring->dequeue, &ring->ccs, TRBS_PER_SEGMENT); in cdns2_ep_inc_deq()
198 * Enable/disable LPM.
208 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET); in cdns2_enable_l1()
209 writeb(LPMCLOCK_SLEEP_ENTRY, &pdev->usb_regs->lpmclock); in cdns2_enable_l1()
211 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET); in cdns2_enable_l1()
217 u8 speed = readb(&pdev->usb_regs->speedctrl); in cdns2_get_speed()
230 if (trb == (pep->ring.trbs + (TRBS_PER_SEGMENT - 1))) in cdns2_next_trb()
231 return pep->ring.trbs; in cdns2_next_trb()
240 struct usb_request *request = &preq->request; in cdns2_gadget_giveback()
241 struct cdns2_device *pdev = pep->pdev; in cdns2_gadget_giveback()
243 list_del_init(&preq->list); in cdns2_gadget_giveback()
245 if (request->status == -EINPROGRESS) in cdns2_gadget_giveback()
246 request->status = status; in cdns2_gadget_giveback()
248 usb_gadget_unmap_request_by_dev(pdev->dev, request, pep->dir); in cdns2_gadget_giveback()
251 preq->finished_trb = 0; in cdns2_gadget_giveback()
255 if (request->complete) { in cdns2_gadget_giveback()
256 spin_unlock(&pdev->lock); in cdns2_gadget_giveback()
257 usb_gadget_giveback_request(&pep->endpoint, request); in cdns2_gadget_giveback()
258 spin_lock(&pdev->lock); in cdns2_gadget_giveback()
261 if (request->buf == pdev->zlp_buf) in cdns2_gadget_giveback()
262 cdns2_gadget_ep_free_request(&pep->endpoint, request); in cdns2_gadget_giveback()
268 if (pep->wa1_set) { in cdns2_wa1_restore_cycle_bit()
271 pep->wa1_set = 0; in cdns2_wa1_restore_cycle_bit()
272 pep->wa1_trb_index = 0xFFFF; in cdns2_wa1_restore_cycle_bit()
273 if (pep->wa1_cycle_bit) in cdns2_wa1_restore_cycle_bit()
274 pep->wa1_trb->control |= cpu_to_le32(0x1); in cdns2_wa1_restore_cycle_bit()
276 pep->wa1_trb->control &= cpu_to_le32(~0x1); in cdns2_wa1_restore_cycle_bit()
283 struct cdns2_device *pdev = pep->pdev; in cdns2_wa1_update_guard()
285 if (!pep->wa1_set) { in cdns2_wa1_update_guard()
288 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY); in cdns2_wa1_update_guard()
291 pep->wa1_cycle_bit = pep->ring.pcs ? TRB_CYCLE : 0; in cdns2_wa1_update_guard()
292 pep->wa1_set = 1; in cdns2_wa1_update_guard()
293 pep->wa1_trb = trb; in cdns2_wa1_update_guard()
294 pep->wa1_trb_index = pep->ring.enqueue; in cdns2_wa1_update_guard()
308 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY); in cdns2_wa1_tray_restore_cycle_bit()
311 if (!doorbell || dma_index != pep->wa1_trb_index) in cdns2_wa1_tray_restore_cycle_bit()
321 struct cdns2_ring *ring; in cdns2_prepare_ring() local
324 ring = &pep->ring; in cdns2_prepare_ring()
326 if (num_trbs > ring->free_trbs) { in cdns2_prepare_ring()
327 pep->ep_state |= EP_RING_FULL; in cdns2_prepare_ring()
328 trace_cdns2_no_room_on_ring("Ring full\n"); in cdns2_prepare_ring()
329 return -ENOBUFS; in cdns2_prepare_ring()
332 if ((ring->enqueue + num_trbs) >= (TRBS_PER_SEGMENT - 1)) { in cdns2_prepare_ring()
333 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY); in cdns2_prepare_ring()
337 if (doorbell && dma_index == TRBS_PER_SEGMENT - 1) { in cdns2_prepare_ring()
338 pep->ep_state |= EP_DEFERRED_DRDY; in cdns2_prepare_ring()
339 return -ENOBUFS; in cdns2_prepare_ring()
343 link_trb = ring->trbs + (TRBS_PER_SEGMENT - 1); in cdns2_prepare_ring()
353 if (pep->type == USB_ENDPOINT_XFER_ISOC || TRBS_PER_SEGMENT > 2) in cdns2_prepare_ring()
356 link_trb->control = cpu_to_le32(((ring->pcs) ? TRB_CYCLE : 0) | in cdns2_prepare_ring()
366 struct cdns2_trb *link_trb = pep->ring.trbs + (TRBS_PER_SEGMENT - 1); in cdns2_dbg_request_trbs()
367 struct cdns2_trb *trb = preq->trb; in cdns2_dbg_request_trbs()
368 int num_trbs = preq->num_of_trb; in cdns2_dbg_request_trbs()
374 trb = pep->ring.trbs; in cdns2_dbg_request_trbs()
375 num_trbs = num_trbs - i; in cdns2_dbg_request_trbs()
388 if (pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_count_trbs()
395 (addr & (TRB_MAX_ISO_BUFF_SIZE - 1)), in cdns2_count_trbs()
398 if (pep->interval > 1) in cdns2_count_trbs()
399 num_trbs = pep->dir ? num_trbs * pep->interval : 1; in cdns2_count_trbs()
400 } else if (pep->dir) { in cdns2_count_trbs()
419 full_len = req->length; in cdns2_count_sg_trbs()
421 for_each_sg(req->sg, sg, req->num_sgs, i) { in cdns2_count_sg_trbs()
435 if (pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_count_sg_trbs()
445 num_trbs = num_trbs + temp - 1; in cdns2_count_sg_trbs()
447 trb_len = trb_len - (temp << 10); in cdns2_count_sg_trbs()
451 full_len -= len; in cdns2_count_sg_trbs()
464 * Let's assume that driver prepares trb with trb->length 700 and burst size
492 pdev->burst_opt[i] = axi_burst_option[j]; in cdsn2_isoc_burst_opt()
523 num_tds = pep->dir ? pep->interval : 1; in cdns2_ep_tx_isoc()
524 split_size = preq->request.num_sgs ? 1024 : 3072; in cdns2_ep_tx_isoc()
527 if (preq->request.num_sgs) { in cdns2_ep_tx_isoc()
528 sg = preq->request.sg; in cdns2_ep_tx_isoc()
532 trb_dma = preq->request.dma; in cdns2_ep_tx_isoc()
533 block_length = preq->request.length; in cdns2_ep_tx_isoc()
536 full_len = preq->request.length; in cdns2_ep_tx_isoc()
537 sg_iter = preq->request.num_sgs ? preq->request.num_sgs : 1; in cdns2_ep_tx_isoc()
546 * Calculate TRB length.- buffer can't across 4KB in cdns2_ep_tx_isoc()
553 if (trb_buff_len > full_len - enqd_len) in cdns2_ep_tx_isoc()
554 trb_buff_len = full_len - enqd_len; in cdns2_ep_tx_isoc()
564 if (enqd_len + trb_buff_len >= full_len || !pep->dir) in cdns2_ep_tx_isoc()
574 if (pep->ring.pcs == 0) in cdns2_ep_tx_isoc()
577 control |= pep->ring.pcs; in cdns2_ep_tx_isoc()
584 TRB_BURST(pep->pdev->burst_opt[trb_buff_len]); in cdns2_ep_tx_isoc()
586 trb = pep->ring.trbs + pep->ring.enqueue; in cdns2_ep_tx_isoc()
587 trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma)); in cdns2_ep_tx_isoc()
588 trb->length = cpu_to_le32(length); in cdns2_ep_tx_isoc()
589 trb->control = cpu_to_le32(control); in cdns2_ep_tx_isoc()
596 --sg_iter; in cdns2_ep_tx_isoc()
597 sent_len -= block_length; in cdns2_ep_tx_isoc()
605 remaining_packet_size -= trb_buff_len; in cdns2_ep_tx_isoc()
606 block_length -= sent_len; in cdns2_ep_tx_isoc()
607 preq->end_trb = pep->ring.enqueue; in cdns2_ep_tx_isoc()
609 cdns2_ep_inc_enq(&pep->ring); in cdns2_ep_tx_isoc()
619 struct cdns2_ring *ring; in cdns2_ep_tx_bulk() local
626 if (preq->request.num_sgs) { in cdns2_ep_tx_bulk()
627 sg = preq->request.sg; in cdns2_ep_tx_bulk()
631 trb_dma = preq->request.dma; in cdns2_ep_tx_bulk()
632 length = preq->request.length; in cdns2_ep_tx_bulk()
635 ring = &pep->ring; in cdns2_ep_tx_bulk()
638 control = TRB_TYPE(TRB_NORMAL) | ring->pcs | TRB_ISP; in cdns2_ep_tx_bulk()
639 trb = pep->ring.trbs + ring->enqueue; in cdns2_ep_tx_bulk()
641 if (pep->dir && sg_iter == trbs_per_td - 1) { in cdns2_ep_tx_bulk()
642 preq->end_trb = ring->enqueue; in cdns2_ep_tx_bulk()
643 control = ring->pcs | TRB_TYPE(TRB_LINK) | TRB_CHAIN in cdns2_ep_tx_bulk()
645 cdns2_ep_inc_enq(&pep->ring); in cdns2_ep_tx_bulk()
647 if (ring->enqueue == 0) in cdns2_ep_tx_bulk()
651 trb->buffer = cpu_to_le32(pep->ring.dma + in cdns2_ep_tx_bulk()
652 (ring->enqueue * TRB_SIZE)); in cdns2_ep_tx_bulk()
653 trb->length = 0; in cdns2_ep_tx_bulk()
654 trb->control = cpu_to_le32(control); in cdns2_ep_tx_bulk()
667 if (sg_iter == (trbs_per_td - (pep->dir ? 2 : 1))) in cdns2_ep_tx_bulk()
672 trb->buffer = cpu_to_le32(trb_dma); in cdns2_ep_tx_bulk()
673 trb->length = cpu_to_le32(TRB_BURST(pep->trb_burst_size) | in cdns2_ep_tx_bulk()
675 trb->control = cpu_to_le32(control); in cdns2_ep_tx_bulk()
677 if (sg && sg_iter < (trbs_per_td - 1)) { in cdns2_ep_tx_bulk()
683 preq->end_trb = ring->enqueue; in cdns2_ep_tx_bulk()
684 cdns2_ep_inc_enq(&pep->ring); in cdns2_ep_tx_bulk()
694 * Memory barrier - Cycle Bit must be set before doorbell. in cdns2_set_drdy()
700 &pdev->adma_regs->ep_sts); in cdns2_set_drdy()
701 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd); in cdns2_set_drdy()
703 if (readl(&pdev->adma_regs->ep_sts) & DMA_EP_STS_TRBERR) { in cdns2_set_drdy()
704 writel(DMA_EP_STS_TRBERR, &pdev->adma_regs->ep_sts); in cdns2_set_drdy()
705 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd); in cdns2_set_drdy()
708 trace_cdns2_doorbell_epx(pep, readl(&pdev->adma_regs->ep_traddr)); in cdns2_set_drdy()
718 if ((readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY)) in cdns2_prepare_first_isoc_transfer()
719 return -EBUSY; in cdns2_prepare_first_isoc_transfer()
721 if (!pep->dir) { in cdns2_prepare_first_isoc_transfer()
722 set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE); in cdns2_prepare_first_isoc_transfer()
723 writel(pep->ring.dma + pep->ring.dequeue, in cdns2_prepare_first_isoc_transfer()
724 &pdev->adma_regs->ep_traddr); in cdns2_prepare_first_isoc_transfer()
732 buffer = pep->ring.dma + pep->ring.dequeue * TRB_SIZE; in cdns2_prepare_first_isoc_transfer()
733 hw_ccs = !!DMA_EP_STS_CCS(readl(&pdev->adma_regs->ep_sts)); in cdns2_prepare_first_isoc_transfer()
735 trb = &pep->ring.trbs[TRBS_PER_SEGMENT]; in cdns2_prepare_first_isoc_transfer()
736 trb->length = 0; in cdns2_prepare_first_isoc_transfer()
737 trb->buffer = cpu_to_le32(TRB_BUFFER(buffer)); in cdns2_prepare_first_isoc_transfer()
738 trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) | TRB_TYPE(TRB_NORMAL)); in cdns2_prepare_first_isoc_transfer()
742 * move to correct place in transfer ring. in cdns2_prepare_first_isoc_transfer()
745 trb->length = 0; in cdns2_prepare_first_isoc_transfer()
746 trb->buffer = cpu_to_le32(TRB_BUFFER(buffer)); in cdns2_prepare_first_isoc_transfer()
747 trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) | in cdns2_prepare_first_isoc_transfer()
750 if (hw_ccs != pep->ring.ccs) in cdns2_prepare_first_isoc_transfer()
751 trb->control |= cpu_to_le32(TRB_TOGGLE); in cdns2_prepare_first_isoc_transfer()
753 set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE); in cdns2_prepare_first_isoc_transfer()
754 writel(pep->ring.dma + (TRBS_PER_SEGMENT * TRB_SIZE), in cdns2_prepare_first_isoc_transfer()
755 &pdev->adma_regs->ep_traddr); in cdns2_prepare_first_isoc_transfer()
760 /* Prepare and start transfer on no-default endpoint. */
764 struct cdns2_device *pdev = pep->pdev; in cdns2_ep_run_transfer()
765 struct cdns2_ring *ring; in cdns2_ep_run_transfer() local
770 cdns2_select_ep(pdev, pep->endpoint.address); in cdns2_ep_run_transfer()
772 if (preq->request.sg) in cdns2_ep_run_transfer()
773 num_trbs = cdns2_count_sg_trbs(pep, &preq->request); in cdns2_ep_run_transfer()
775 num_trbs = cdns2_count_trbs(pep, preq->request.dma, in cdns2_ep_run_transfer()
776 preq->request.length); in cdns2_ep_run_transfer()
782 ring = &pep->ring; in cdns2_ep_run_transfer()
783 preq->start_trb = ring->enqueue; in cdns2_ep_run_transfer()
784 preq->trb = ring->trbs + ring->enqueue; in cdns2_ep_run_transfer()
786 if (usb_endpoint_xfer_isoc(pep->endpoint.desc)) { in cdns2_ep_run_transfer()
789 togle_pcs = cdns2_wa1_update_guard(pep, ring->trbs + ring->enqueue); in cdns2_ep_run_transfer()
793 preq->num_of_trb = num_trbs; in cdns2_ep_run_transfer()
796 * Memory barrier - cycle bit must be set as the last operation. in cdns2_ep_run_transfer()
802 preq->trb->control = preq->trb->control ^ cpu_to_le32(1); in cdns2_ep_run_transfer()
807 if (!pep->wa1_set && !(pep->ep_state & EP_STALLED) && !pep->skip) { in cdns2_ep_run_transfer()
808 if (pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_ep_run_transfer()
827 while (!list_empty(&pep->deferred_list)) { in cdns2_start_all_request()
828 preq = cdns2_next_preq(&pep->deferred_list); in cdns2_start_all_request()
834 list_move_tail(&preq->list, &pep->pending_list); in cdns2_start_all_request()
837 pep->ep_state &= ~EP_RING_FULL; in cdns2_start_all_request()
849 * SR - start ring
850 * ER - end ring
851 * DQ = ring->dequeue - dequeue position
852 * EQ = ring->enqueue - enqueue position
853 * ST = preq->start_trb - index of first TRB in transfer ring
854 * ET = preq->end_trb - index of last TRB in transfer ring
855 * CI = current_index - index of processed TRB by DMA.
858 * Then, we check if cycle bit for index pep->dequeue
862 * 1. ring->dequeue never equals to current_index.
863 * 2 ring->enqueue never exceed ring->dequeue
864 * 3. exception: ring->enqueue == ring->dequeue
865 * and ring->free_trbs is zero.
869 * Case 1 - ring->dequeue < current_index
873 * Case 2 - ring->dequeue > current_index
875 * transfer ring.
881 struct cdns2_device *pdev = pep->pdev; in cdns2_trb_handled()
882 struct cdns2_ring *ring; in cdns2_trb_handled() local
888 ring = &pep->ring; in cdns2_trb_handled()
890 doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY); in cdns2_trb_handled()
894 * Transfer Ring. First of them is used as zero length packet and the in cdns2_trb_handled()
901 if (preq->start_trb < preq->end_trb) { in cdns2_trb_handled()
902 if (ring->dequeue > preq->end_trb) in cdns2_trb_handled()
905 if (ring->dequeue < preq->start_trb) in cdns2_trb_handled()
909 if (preq->start_trb > preq->end_trb && ring->dequeue > preq->end_trb && in cdns2_trb_handled()
910 ring->dequeue < preq->start_trb) in cdns2_trb_handled()
913 if (preq->start_trb == preq->end_trb && ring->dequeue != preq->end_trb) in cdns2_trb_handled()
916 trb = &ring->trbs[ring->dequeue]; in cdns2_trb_handled()
918 if ((le32_to_cpu(trb->control) & TRB_CYCLE) != ring->ccs) in cdns2_trb_handled()
921 if (doorbell == 1 && current_index == ring->dequeue) in cdns2_trb_handled()
925 if (TRBS_PER_SEGMENT == 2 && pep->type != USB_ENDPOINT_XFER_ISOC) { in cdns2_trb_handled()
930 if (ring->enqueue == ring->dequeue && in cdns2_trb_handled()
931 ring->free_trbs == 0) { in cdns2_trb_handled()
933 } else if (ring->dequeue < current_index) { in cdns2_trb_handled()
934 if ((current_index == (TRBS_PER_SEGMENT - 1)) && in cdns2_trb_handled()
935 !ring->dequeue) in cdns2_trb_handled()
939 } else if (ring->dequeue > current_index) { in cdns2_trb_handled()
956 trb = pep->ring.trbs + pep->ring.dequeue; in cdns2_skip_isoc_td()
958 for (i = preq->finished_trb ; i < preq->num_of_trb; i++) { in cdns2_skip_isoc_td()
959 preq->finished_trb++; in cdns2_skip_isoc_td()
961 cdns2_ep_inc_deq(&pep->ring); in cdns2_skip_isoc_td()
967 pep->skip = false; in cdns2_skip_isoc_td()
978 while (!list_empty(&pep->pending_list)) { in cdns2_transfer_completed()
979 preq = cdns2_next_preq(&pep->pending_list); in cdns2_transfer_completed()
980 trb = pep->ring.trbs + pep->ring.dequeue; in cdns2_transfer_completed()
986 while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK && in cdns2_transfer_completed()
987 le32_to_cpu(trb->length)) { in cdns2_transfer_completed()
989 cdns2_ep_inc_deq(&pep->ring); in cdns2_transfer_completed()
990 trb = pep->ring.trbs + pep->ring.dequeue; in cdns2_transfer_completed()
994 * Re-select endpoint. It could be changed by other CPU in cdns2_transfer_completed()
997 cdns2_select_ep(pdev, pep->endpoint.address); in cdns2_transfer_completed()
1000 preq->finished_trb++; in cdns2_transfer_completed()
1002 if (preq->finished_trb >= preq->num_of_trb) in cdns2_transfer_completed()
1005 trb = pep->ring.trbs + pep->ring.dequeue; in cdns2_transfer_completed()
1008 if (pep->dir && pep->type == USB_ENDPOINT_XFER_ISOC) in cdns2_transfer_completed()
1011 * trb->length. in cdns2_transfer_completed()
1013 preq->request.actual = preq->request.length; in cdns2_transfer_completed()
1015 preq->request.actual += in cdns2_transfer_completed()
1016 TRB_LEN(le32_to_cpu(trb->length)); in cdns2_transfer_completed()
1018 cdns2_ep_inc_deq(&pep->ring); in cdns2_transfer_completed()
1028 if (pep->type != USB_ENDPOINT_XFER_ISOC && in cdns2_transfer_completed()
1034 if (pep->skip && preq) in cdns2_transfer_completed()
1037 if (!(pep->ep_state & EP_STALLED) && in cdns2_transfer_completed()
1038 !(pep->ep_state & EP_STALL_PENDING)) in cdns2_transfer_completed()
1044 if (!pdev->may_wakeup) in cdns2_wakeup()
1048 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_SIGRSUME); in cdns2_wakeup()
1053 struct cdns2_device *pdev = pep->pdev; in cdns2_rearm_transfer()
1063 writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd); in cdns2_rearm_transfer()
1068 readl(&pdev->adma_regs->ep_traddr)); in cdns2_rearm_transfer()
1074 struct cdns2_device *pdev = pep->pdev; in cdns2_handle_epx_interrupt()
1079 cdns2_select_ep(pdev, pep->endpoint.address); in cdns2_handle_epx_interrupt()
1083 ep_sts_reg = readl(&pdev->adma_regs->ep_sts); in cdns2_handle_epx_interrupt()
1084 writel(ep_sts_reg, &pdev->adma_regs->ep_sts); in cdns2_handle_epx_interrupt()
1086 if (pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_handle_epx_interrupt()
1090 mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize); in cdns2_handle_epx_interrupt()
1091 cs = pep->dir ? readb(&pdev->epx_regs->ep[pep->num - 1].txcs) : in cdns2_handle_epx_interrupt()
1092 readb(&pdev->epx_regs->ep[pep->num - 1].rxcs); in cdns2_handle_epx_interrupt()
1103 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE); in cdns2_handle_epx_interrupt()
1106 readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val, in cdns2_handle_epx_interrupt()
1109 writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd); in cdns2_handle_epx_interrupt()
1112 readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val, in cdns2_handle_epx_interrupt()
1115 pep->skip = true; in cdns2_handle_epx_interrupt()
1118 if (ep_sts_reg & DMA_EP_STS_TRBERR || pep->skip) { in cdns2_handle_epx_interrupt()
1119 if (pep->ep_state & EP_STALL_PENDING && in cdns2_handle_epx_interrupt()
1126 * OUT data packet. If host disable stream or lost some packet in cdns2_handle_epx_interrupt()
1130 if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->wa1_set) { in cdns2_handle_epx_interrupt()
1131 if (!pep->dir) in cdns2_handle_epx_interrupt()
1132 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, in cdns2_handle_epx_interrupt()
1136 if (pep->ep_state & EP_DEFERRED_DRDY) { in cdns2_handle_epx_interrupt()
1137 pep->ep_state &= ~EP_DEFERRED_DRDY; in cdns2_handle_epx_interrupt()
1146 if (!(pep->ep_state & EP_STALLED) && in cdns2_handle_epx_interrupt()
1147 !(pep->ep_state & EP_STALL_PENDING)) { in cdns2_handle_epx_interrupt()
1148 if (pep->ep_state & EP_DEFERRED_DRDY) { in cdns2_handle_epx_interrupt()
1149 pep->ep_state &= ~EP_DEFERRED_DRDY; in cdns2_handle_epx_interrupt()
1152 cdns2_rearm_transfer(pep, pep->wa1_set); in cdns2_handle_epx_interrupt()
1165 if (pdev->gadget_driver && pdev->gadget_driver->disconnect) in cdns2_disconnect_gadget()
1166 pdev->gadget_driver->disconnect(&pdev->gadget); in cdns2_disconnect_gadget()
1178 if (pdev->in_lpm) in cdns2_usb_irq_handler()
1181 reg_usb_irq_m = readb(&pdev->interrupt_regs->usbien); in cdns2_usb_irq_handler()
1182 reg_ext_irq_m = readb(&pdev->interrupt_regs->extien); in cdns2_usb_irq_handler()
1185 writeb(0, &pdev->interrupt_regs->usbien); in cdns2_usb_irq_handler()
1186 writeb(0, &pdev->interrupt_regs->extien); in cdns2_usb_irq_handler()
1187 writel(0, &pdev->adma_regs->ep_ien); in cdns2_usb_irq_handler()
1190 writel(0, &pdev->adma_regs->ep_sts); in cdns2_usb_irq_handler()
1191 writeb(0, &pdev->interrupt_regs->usbirq); in cdns2_usb_irq_handler()
1192 writeb(0, &pdev->interrupt_regs->extirq); in cdns2_usb_irq_handler()
1194 reg_ep_ists = readl(&pdev->adma_regs->ep_ists); in cdns2_usb_irq_handler()
1195 reg_usb_irq = readb(&pdev->interrupt_regs->usbirq); in cdns2_usb_irq_handler()
1196 reg_ext_irq = readb(&pdev->interrupt_regs->extirq); in cdns2_usb_irq_handler()
1202 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien); in cdns2_usb_irq_handler()
1203 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien); in cdns2_usb_irq_handler()
1204 writel(~0, &pdev->adma_regs->ep_ien); in cdns2_usb_irq_handler()
1215 ext_irq = readb(&pdev->interrupt_regs->extirq) & EXTIRQ_WAKEUP; in cdns2_thread_usb_irq_handler()
1216 writeb(ext_irq, &pdev->interrupt_regs->extirq); in cdns2_thread_usb_irq_handler()
1218 usb_irq = readb(&pdev->interrupt_regs->usbirq) & USB_IEN_INIT; in cdns2_thread_usb_irq_handler()
1219 writeb(usb_irq, &pdev->interrupt_regs->usbirq); in cdns2_thread_usb_irq_handler()
1227 if (pdev->gadget_driver && pdev->gadget_driver->resume) { in cdns2_thread_usb_irq_handler()
1228 spin_unlock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1229 pdev->gadget_driver->resume(&pdev->gadget); in cdns2_thread_usb_irq_handler()
1230 spin_lock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1235 u8 reg = readb(&pdev->usb_regs->lpmctrl); in cdns2_thread_usb_irq_handler()
1239 writeb(0, &pdev->usb_regs->sleep_clkgate); in cdns2_thread_usb_irq_handler()
1243 if (pdev->gadget_driver && pdev->gadget_driver->suspend) { in cdns2_thread_usb_irq_handler()
1244 spin_unlock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1245 pdev->gadget_driver->suspend(&pdev->gadget); in cdns2_thread_usb_irq_handler()
1246 spin_lock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1251 if (pdev->gadget_driver) { in cdns2_thread_usb_irq_handler()
1252 pdev->dev_address = 0; in cdns2_thread_usb_irq_handler()
1254 spin_unlock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1255 usb_gadget_udc_reset(&pdev->gadget, in cdns2_thread_usb_irq_handler()
1256 pdev->gadget_driver); in cdns2_thread_usb_irq_handler()
1257 spin_lock(&pdev->lock); in cdns2_thread_usb_irq_handler()
1262 * process. For high-speed reset procedure is completed in cdns2_thread_usb_irq_handler()
1272 pdev->gadget.speed = speed; in cdns2_thread_usb_irq_handler()
1275 pdev->may_wakeup = 0; in cdns2_thread_usb_irq_handler()
1280 pdev->ep0_stage = CDNS2_SETUP_STAGE; in cdns2_thread_usb_irq_handler()
1296 spin_lock_irqsave(&pdev->lock, flags); in cdns2_thread_irq_handler()
1300 dma_ep_ists = readl(&pdev->adma_regs->ep_ists); in cdns2_thread_irq_handler()
1317 u8 ep_idx = bit > 16 ? (bit - 16) * 2 : (bit * 2) - 1; in cdns2_thread_irq_handler()
1320 * Endpoints in pdev->eps[] are held in order: in cdns2_thread_irq_handler()
1325 cdns2_handle_epx_interrupt(&pdev->eps[ep_idx]); in cdns2_thread_irq_handler()
1329 writel(~0, &pdev->adma_regs->ep_ien); in cdns2_thread_irq_handler()
1330 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien); in cdns2_thread_irq_handler()
1331 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien); in cdns2_thread_irq_handler()
1333 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_thread_irq_handler()
1351 pep = &pdev->eps[i]; in cdns2_eps_onchip_buffer_init()
1353 if (!(pep->ep_state & EP_CLAIMED)) in cdns2_eps_onchip_buffer_init()
1356 if (pep->dir) in cdns2_eps_onchip_buffer_init()
1357 min_buf_tx += pep->buffering; in cdns2_eps_onchip_buffer_init()
1359 min_buf_rx += pep->buffering; in cdns2_eps_onchip_buffer_init()
1363 pep = &pdev->eps[i]; in cdns2_eps_onchip_buffer_init()
1365 if (!(pep->ep_state & EP_CLAIMED)) in cdns2_eps_onchip_buffer_init()
1368 if (pep->dir) { in cdns2_eps_onchip_buffer_init()
1369 free = pdev->onchip_tx_buf - min_buf_tx; in cdns2_eps_onchip_buffer_init()
1371 if (free + pep->buffering >= 4) in cdns2_eps_onchip_buffer_init()
1374 free = free + pep->buffering; in cdns2_eps_onchip_buffer_init()
1376 min_buf_tx = min_buf_tx - pep->buffering + free; in cdns2_eps_onchip_buffer_init()
1378 pep->buffering = free; in cdns2_eps_onchip_buffer_init()
1381 &pdev->epx_regs->txstaddr[pep->num - 1]); in cdns2_eps_onchip_buffer_init()
1382 pdev->epx_regs->txstaddr[pep->num - 1] = tx_offset; in cdns2_eps_onchip_buffer_init()
1384 dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n", in cdns2_eps_onchip_buffer_init()
1385 pep->name, tx_offset, pep->buffering); in cdns2_eps_onchip_buffer_init()
1387 tx_offset += pep->buffering * 1024; in cdns2_eps_onchip_buffer_init()
1389 free = pdev->onchip_rx_buf - min_buf_rx; in cdns2_eps_onchip_buffer_init()
1391 if (free + pep->buffering >= 4) in cdns2_eps_onchip_buffer_init()
1394 free = free + pep->buffering; in cdns2_eps_onchip_buffer_init()
1396 min_buf_rx = min_buf_rx - pep->buffering + free; in cdns2_eps_onchip_buffer_init()
1398 pep->buffering = free; in cdns2_eps_onchip_buffer_init()
1400 &pdev->epx_regs->rxstaddr[pep->num - 1]); in cdns2_eps_onchip_buffer_init()
1402 dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n", in cdns2_eps_onchip_buffer_init()
1403 pep->name, rx_offset, pep->buffering); in cdns2_eps_onchip_buffer_init()
1405 rx_offset += pep->buffering * 1024; in cdns2_eps_onchip_buffer_init()
1413 bool is_iso_ep = (pep->type == USB_ENDPOINT_XFER_ISOC); in cdns2_ep_config()
1414 struct cdns2_device *pdev = pep->pdev; in cdns2_ep_config()
1422 switch (pep->type) { in cdns2_ep_config()
1430 mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize); in cdns2_ep_config()
1434 if (pep->dir) { in cdns2_ep_config()
1435 set_reg_bit_8(&pdev->epx_regs->isoautoarm, BIT(pep->num)); in cdns2_ep_config()
1436 set_reg_bit_8(&pdev->epx_regs->isoautodump, BIT(pep->num)); in cdns2_ep_config()
1437 set_reg_bit_8(&pdev->epx_regs->isodctrl, BIT(pep->num)); in cdns2_ep_config()
1441 switch (pdev->gadget.speed) { in cdns2_ep_config()
1450 return -EINVAL; in cdns2_ep_config()
1453 ep_cfg |= (EPX_CON_VAL | (pep->buffering - 1)); in cdns2_ep_config()
1455 if (pep->dir) { in cdns2_ep_config()
1457 writew(max_packet_size, &pdev->epx_regs->txmaxpack[pep->num - 1]); in cdns2_ep_config()
1458 writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].txcon); in cdns2_ep_config()
1460 writew(max_packet_size, &pdev->epx_regs->rxmaxpack[pep->num - 1]); in cdns2_ep_config()
1461 writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].rxcon); in cdns2_ep_config()
1464 writeb(pep->num | dir | FIFOCTRL_FIFOAUTO, in cdns2_ep_config()
1465 &pdev->usb_regs->fifoctrl); in cdns2_ep_config()
1466 writeb(pep->num | dir, &pdev->epx_regs->endprst); in cdns2_ep_config()
1467 writeb(pep->num | ENDPRST_FIFORST | ENDPRST_TOGRST | dir, in cdns2_ep_config()
1468 &pdev->epx_regs->endprst); in cdns2_ep_config()
1471 pep->trb_burst_size = 128; in cdns2_ep_config()
1473 pep->trb_burst_size = 64; in cdns2_ep_config()
1475 pep->trb_burst_size = 16; in cdns2_ep_config()
1477 cdns2_select_ep(pdev, pep->num | pep->dir); in cdns2_ep_config()
1478 writel(DMA_EP_CMD_EPRST | DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd); in cdns2_ep_config()
1480 ret = readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val, in cdns2_ep_config()
1488 writel(DMA_EP_STS_TRBERR | DMA_EP_STS_ISOERR, &pdev->adma_regs->ep_sts_en); in cdns2_ep_config()
1491 writel(DMA_EP_CFG_ENABLE, &pdev->adma_regs->ep_cfg); in cdns2_ep_config()
1495 dev_dbg(pdev->dev, "Configure %s: with MPS: %08x, ep con: %02x\n", in cdns2_ep_config()
1496 pep->name, max_packet_size, ep_cfg); in cdns2_ep_config()
1511 preq->pep = pep; in cdns2_gadget_ep_alloc_request()
1515 return &preq->request; in cdns2_gadget_ep_alloc_request()
1537 if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT || in cdns2_gadget_ep_enable()
1538 !desc->wMaxPacketSize) { in cdns2_gadget_ep_enable()
1539 return -EINVAL; in cdns2_gadget_ep_enable()
1543 pdev = pep->pdev; in cdns2_gadget_ep_enable()
1545 if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED, in cdns2_gadget_ep_enable()
1546 "%s is already enabled\n", pep->name)) in cdns2_gadget_ep_enable()
1549 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_ep_enable()
1551 pep->type = usb_endpoint_type(desc); in cdns2_gadget_ep_enable()
1552 pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0; in cdns2_gadget_ep_enable()
1554 if (pdev->gadget.speed == USB_SPEED_FULL) in cdns2_gadget_ep_enable()
1555 if (pep->type == USB_ENDPOINT_XFER_INT) in cdns2_gadget_ep_enable()
1556 pep->interval = desc->bInterval; in cdns2_gadget_ep_enable()
1558 if (pep->interval > ISO_MAX_INTERVAL && in cdns2_gadget_ep_enable()
1559 pep->type == USB_ENDPOINT_XFER_ISOC) { in cdns2_gadget_ep_enable()
1560 dev_err(pdev->dev, "ISO period is limited to %d (current: %d)\n", in cdns2_gadget_ep_enable()
1561 ISO_MAX_INTERVAL, pep->interval); in cdns2_gadget_ep_enable()
1563 ret = -EINVAL; in cdns2_gadget_ep_enable()
1568 * During ISO OUT traffic DMA reads Transfer Ring for the EP which has in cdns2_gadget_ep_enable()
1575 if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->dir) in cdns2_gadget_ep_enable()
1585 ret = -EINVAL; in cdns2_gadget_ep_enable()
1591 pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING); in cdns2_gadget_ep_enable()
1592 pep->ep_state |= EP_ENABLED; in cdns2_gadget_ep_enable()
1593 pep->wa1_set = 0; in cdns2_gadget_ep_enable()
1594 pep->ring.enqueue = 0; in cdns2_gadget_ep_enable()
1595 pep->ring.dequeue = 0; in cdns2_gadget_ep_enable()
1596 reg = readl(&pdev->adma_regs->ep_sts); in cdns2_gadget_ep_enable()
1597 pep->ring.pcs = !!DMA_EP_STS_CCS(reg); in cdns2_gadget_ep_enable()
1598 pep->ring.ccs = !!DMA_EP_STS_CCS(reg); in cdns2_gadget_ep_enable()
1600 writel(pep->ring.dma, &pdev->adma_regs->ep_traddr); in cdns2_gadget_ep_enable()
1603 pep->ring.free_trbs = TRBS_PER_SEGMENT - 1; in cdns2_gadget_ep_enable()
1606 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_ep_enable()
1620 return -EINVAL; in cdns2_gadget_ep_disable()
1623 pdev = pep->pdev; in cdns2_gadget_ep_disable()
1625 if (dev_WARN_ONCE(pdev->dev, !(pep->ep_state & EP_ENABLED), in cdns2_gadget_ep_disable()
1626 "%s is already disabled\n", pep->name)) in cdns2_gadget_ep_disable()
1629 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_ep_disable()
1633 cdns2_select_ep(pdev, ep->desc->bEndpointAddress); in cdns2_gadget_ep_disable()
1635 clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE); in cdns2_gadget_ep_disable()
1642 readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val, in cdns2_gadget_ep_disable()
1644 writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd); in cdns2_gadget_ep_disable()
1646 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val, in cdns2_gadget_ep_disable()
1650 while (!list_empty(&pep->pending_list)) { in cdns2_gadget_ep_disable()
1651 preq = cdns2_next_preq(&pep->pending_list); in cdns2_gadget_ep_disable()
1652 cdns2_gadget_giveback(pep, preq, -ESHUTDOWN); in cdns2_gadget_ep_disable()
1655 while (!list_empty(&pep->deferred_list)) { in cdns2_gadget_ep_disable()
1656 preq = cdns2_next_preq(&pep->deferred_list); in cdns2_gadget_ep_disable()
1657 cdns2_gadget_giveback(pep, preq, -ESHUTDOWN); in cdns2_gadget_ep_disable()
1660 ep->desc = NULL; in cdns2_gadget_ep_disable()
1661 pep->ep_state &= ~EP_ENABLED; in cdns2_gadget_ep_disable()
1663 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_ep_disable()
1672 struct cdns2_device *pdev = pep->pdev; in cdns2_ep_enqueue()
1676 request = &preq->request; in cdns2_ep_enqueue()
1677 request->actual = 0; in cdns2_ep_enqueue()
1678 request->status = -EINPROGRESS; in cdns2_ep_enqueue()
1680 ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->dir); in cdns2_ep_enqueue()
1686 list_add_tail(&preq->list, &pep->deferred_list); in cdns2_ep_enqueue()
1689 if (!(pep->ep_state & EP_STALLED) && !(pep->ep_state & EP_STALL_PENDING)) in cdns2_ep_enqueue()
1706 return -EINVAL; in cdns2_gadget_ep_queue()
1709 pdev = pep->pdev; in cdns2_gadget_ep_queue()
1711 if (!(pep->ep_state & EP_ENABLED)) { in cdns2_gadget_ep_queue()
1712 dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n", in cdns2_gadget_ep_queue()
1713 pep->name); in cdns2_gadget_ep_queue()
1714 return -EINVAL; in cdns2_gadget_ep_queue()
1717 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_ep_queue()
1722 if (ret == 0 && request->zero && request->length && in cdns2_gadget_ep_queue()
1723 (request->length % ep->maxpacket == 0)) { in cdns2_gadget_ep_queue()
1727 zlp_request->buf = pdev->zlp_buf; in cdns2_gadget_ep_queue()
1728 zlp_request->length = 0; in cdns2_gadget_ep_queue()
1734 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_ep_queue()
1749 if (!ep || !request || !ep->desc) in cdns2_gadget_ep_dequeue()
1750 return -EINVAL; in cdns2_gadget_ep_dequeue()
1753 if (!pep->endpoint.desc) { in cdns2_gadget_ep_dequeue()
1754 dev_err(pep->pdev->dev, "%s: can't dequeue to disabled endpoint\n", in cdns2_gadget_ep_dequeue()
1755 pep->name); in cdns2_gadget_ep_dequeue()
1756 return -ESHUTDOWN; in cdns2_gadget_ep_dequeue()
1760 if (!(pep->ep_state & EP_ENABLED)) in cdns2_gadget_ep_dequeue()
1763 spin_lock_irqsave(&pep->pdev->lock, flags); in cdns2_gadget_ep_dequeue()
1768 list_for_each_entry_safe(preq, preq_temp, &pep->pending_list, list) { in cdns2_gadget_ep_dequeue()
1775 list_for_each_entry_safe(preq, preq_temp, &pep->deferred_list, list) { in cdns2_gadget_ep_dequeue()
1783 link_trb = preq->trb; in cdns2_gadget_ep_dequeue()
1785 /* Update ring only if removed request is on pending_req_list list. */ in cdns2_gadget_ep_dequeue()
1788 writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd); in cdns2_gadget_ep_dequeue()
1791 readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val, in cdns2_gadget_ep_dequeue()
1794 buffer = cpu_to_le32(TRB_BUFFER(pep->ring.dma + in cdns2_gadget_ep_dequeue()
1795 ((preq->end_trb + 1) * TRB_SIZE))); in cdns2_gadget_ep_dequeue()
1797 for (i = 0; i < preq->num_of_trb; i++) { in cdns2_gadget_ep_dequeue()
1798 link_trb->buffer = buffer; in cdns2_gadget_ep_dequeue()
1799 link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) in cdns2_gadget_ep_dequeue()
1807 if (pep->wa1_trb == preq->trb) in cdns2_gadget_ep_dequeue()
1811 cdns2_gadget_giveback(pep, cur_preq, -ECONNRESET); in cdns2_gadget_ep_dequeue()
1813 preq = cdns2_next_preq(&pep->pending_list); in cdns2_gadget_ep_dequeue()
1818 spin_unlock_irqrestore(&pep->pdev->lock, flags); in cdns2_gadget_ep_dequeue()
1829 if (!(pep->ep_state & EP_ENABLED)) in cdns2_halt_endpoint()
1830 return -EPERM; in cdns2_halt_endpoint()
1832 if (pep->dir) { in cdns2_halt_endpoint()
1834 conf = &pdev->epx_regs->ep[pep->num - 1].txcon; in cdns2_halt_endpoint()
1836 conf = &pdev->epx_regs->ep[pep->num - 1].rxcon; in cdns2_halt_endpoint()
1844 preq = cdns2_next_preq(&pep->pending_list); in cdns2_halt_endpoint()
1846 trb = preq->trb; in cdns2_halt_endpoint()
1849 trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE); in cdns2_halt_endpoint()
1856 writeb(dir | pep->num, &pdev->epx_regs->endprst); in cdns2_halt_endpoint()
1857 writeb(dir | ENDPRST_TOGRST | pep->num, in cdns2_halt_endpoint()
1858 &pdev->epx_regs->endprst); in cdns2_halt_endpoint()
1862 pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING); in cdns2_halt_endpoint()
1875 writeb(dir | pep->num, &pdev->epx_regs->endprst); in cdns2_halt_endpoint()
1876 writeb(dir | ENDPRST_FIFORST | pep->num, in cdns2_halt_endpoint()
1877 &pdev->epx_regs->endprst); in cdns2_halt_endpoint()
1878 pep->ep_state |= EP_STALLED; in cdns2_halt_endpoint()
1888 struct cdns2_device *pdev = pep->pdev; in cdns2_gadget_ep_set_halt()
1893 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_ep_set_halt()
1895 preq = cdns2_next_preq(&pep->pending_list); in cdns2_gadget_ep_set_halt()
1898 ret = -EAGAIN; in cdns2_gadget_ep_set_halt()
1903 pep->ep_state &= ~EP_WEDGE; in cdns2_gadget_ep_set_halt()
1908 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_ep_set_halt()
1917 pep->ep_state |= EP_WEDGE; in cdns2_gadget_ep_set_wedge()
1930 list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) { in cdns2_find_available_ep()
1934 char c[2] = {ep->name[2], '\0'}; in cdns2_find_available_ep()
1941 if (pep->num != num) in cdns2_find_available_ep()
1944 ep_correct = (pep->endpoint.caps.dir_in && in cdns2_find_available_ep()
1946 (pep->endpoint.caps.dir_out && in cdns2_find_available_ep()
1949 if (ep_correct && !(pep->ep_state & EP_CLAIMED)) in cdns2_find_available_ep()
1953 return ERR_PTR(-ENOENT); in cdns2_find_available_ep()
1958 * on-chip memory usage.
1971 dev_err(pdev->dev, "no available ep\n"); in cdns2_gadget_match_ep()
1975 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_match_ep()
1978 pep->buffering = 4; in cdns2_gadget_match_ep()
1980 pep->buffering = 1; in cdns2_gadget_match_ep()
1982 pep->ep_state |= EP_CLAIMED; in cdns2_gadget_match_ep()
1983 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_match_ep()
1985 return &pep->endpoint; in cdns2_gadget_match_ep()
1990 .disable = cdns2_gadget_ep_disable,
2003 return readw(&pdev->usb_regs->frmnr); in cdns2_gadget_get_frame()
2011 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_wakeup()
2013 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_wakeup()
2024 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_set_selfpowered()
2025 pdev->is_selfpowered = !!is_selfpowered; in cdns2_gadget_set_selfpowered()
2026 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_set_selfpowered()
2030 /* Disable interrupts and begin the controller halting process. */
2033 set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON); in cdns2_quiesce()
2035 /* Disable interrupt. */ in cdns2_quiesce()
2036 writeb(0, &pdev->interrupt_regs->extien), in cdns2_quiesce()
2037 writeb(0, &pdev->interrupt_regs->usbien), in cdns2_quiesce()
2038 writew(0, &pdev->adma_regs->ep_ien); in cdns2_quiesce()
2041 writeb(0x0, &pdev->interrupt_regs->usbirq); in cdns2_quiesce()
2049 writel(~0x0, &pdev->adma_regs->ep_ien); in cdns2_gadget_config()
2051 writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien); in cdns2_gadget_config()
2052 writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien); in cdns2_gadget_config()
2053 writel(DMA_CONF_DMULT, &pdev->adma_regs->conf); in cdns2_gadget_config()
2064 * Disable events handling while controller is being in cdns2_gadget_pullup()
2067 disable_irq(pdev->irq); in cdns2_gadget_pullup()
2068 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_pullup()
2072 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON); in cdns2_gadget_pullup()
2077 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_pullup()
2078 enable_irq(pdev->irq); in cdns2_gadget_pullup()
2087 enum usb_device_speed max_speed = driver->max_speed; in cdns2_gadget_udc_start()
2090 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_udc_start()
2091 pdev->gadget_driver = driver; in cdns2_gadget_udc_start()
2094 max_speed = min(driver->max_speed, gadget->max_speed); in cdns2_gadget_udc_start()
2098 writeb(SPEEDCTRL_HSDISABLE, &pdev->usb_regs->speedctrl); in cdns2_gadget_udc_start()
2101 writeb(0, &pdev->usb_regs->speedctrl); in cdns2_gadget_udc_start()
2104 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n", in cdns2_gadget_udc_start()
2114 writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst); in cdns2_gadget_udc_start()
2116 &pdev->usb_regs->endprst); in cdns2_gadget_udc_start()
2117 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->usb_regs->endprst); in cdns2_gadget_udc_start()
2122 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_udc_start()
2135 pdev->gadget_driver = NULL; in cdns2_gadget_udc_stop()
2136 pdev->gadget.speed = USB_SPEED_UNKNOWN; in cdns2_gadget_udc_stop()
2138 list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) { in cdns2_gadget_udc_stop()
2140 bEndpointAddress = pep->num | pep->dir; in cdns2_gadget_udc_stop()
2142 writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd); in cdns2_gadget_udc_stop()
2143 readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val, in cdns2_gadget_udc_stop()
2149 writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst); in cdns2_gadget_udc_stop()
2151 &pdev->epx_regs->endprst); in cdns2_gadget_udc_stop()
2152 writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->epx_regs->endprst); in cdns2_gadget_udc_stop()
2161 .pullup = cdns2_gadget_pullup,
2172 cdns2_free_tr_segment(&pdev->eps[i]); in cdns2_free_all_eps()
2186 * Endpoints are being held in pdev->eps[] in form: in cdns2_init_eps()
2192 pep = &pdev->eps[i]; in cdns2_init_eps()
2193 pep->pdev = pdev; in cdns2_init_eps()
2194 pep->num = epnum; in cdns2_init_eps()
2196 pep->dir = direction ? USB_DIR_IN : USB_DIR_OUT; in cdns2_init_eps()
2197 pep->idx = i; in cdns2_init_eps()
2199 /* Ep0in and ep0out are represented by pdev->eps[0]. */ in cdns2_init_eps()
2203 snprintf(pep->name, sizeof(pep->name), "ep%d%s", in cdns2_init_eps()
2210 dev_err(pdev->dev, "Failed to init ep0\n"); in cdns2_init_eps()
2214 snprintf(pep->name, sizeof(pep->name), "ep%d%s", in cdns2_init_eps()
2216 pep->endpoint.name = pep->name; in cdns2_init_eps()
2218 usb_ep_set_maxpacket_limit(&pep->endpoint, 1024); in cdns2_init_eps()
2219 pep->endpoint.ops = &cdns2_gadget_ep_ops; in cdns2_init_eps()
2220 list_add_tail(&pep->endpoint.ep_list, &pdev->gadget.ep_list); in cdns2_init_eps()
2222 pep->endpoint.caps.dir_in = direction; in cdns2_init_eps()
2223 pep->endpoint.caps.dir_out = !direction; in cdns2_init_eps()
2225 pep->endpoint.caps.type_iso = 1; in cdns2_init_eps()
2226 pep->endpoint.caps.type_bulk = 1; in cdns2_init_eps()
2227 pep->endpoint.caps.type_int = 1; in cdns2_init_eps()
2230 pep->endpoint.name = pep->name; in cdns2_init_eps()
2231 pep->ep_state = 0; in cdns2_init_eps()
2233 dev_dbg(pdev->dev, "Init %s, SupType: CTRL: %s, INT: %s, " in cdns2_init_eps()
2235 pep->name, in cdns2_init_eps()
2236 (pep->endpoint.caps.type_control) ? "yes" : "no", in cdns2_init_eps()
2237 (pep->endpoint.caps.type_int) ? "yes" : "no", in cdns2_init_eps()
2238 (pep->endpoint.caps.type_bulk) ? "yes" : "no", in cdns2_init_eps()
2239 (pep->endpoint.caps.type_iso) ? "yes" : "no", in cdns2_init_eps()
2240 (pep->endpoint.caps.dir_in) ? "yes" : "no", in cdns2_init_eps()
2241 (pep->endpoint.caps.dir_out) ? "yes" : "no"); in cdns2_init_eps()
2243 INIT_LIST_HEAD(&pep->pending_list); in cdns2_init_eps()
2244 INIT_LIST_HEAD(&pep->deferred_list); in cdns2_init_eps()
2256 pdev->usb_regs = pdev->regs; in cdns2_gadget_start()
2257 pdev->ep0_regs = pdev->regs; in cdns2_gadget_start()
2258 pdev->epx_regs = pdev->regs; in cdns2_gadget_start()
2259 pdev->interrupt_regs = pdev->regs; in cdns2_gadget_start()
2260 pdev->adma_regs = pdev->regs + CDNS2_ADMA_REGS_OFFSET; in cdns2_gadget_start()
2264 &pdev->usb_regs->cpuctrl); in cdns2_gadget_start()
2267 usb_initialize_gadget(pdev->dev, &pdev->gadget, NULL); in cdns2_gadget_start()
2269 device_property_read_u16(pdev->dev, "cdns,on-chip-tx-buff-size", in cdns2_gadget_start()
2270 &pdev->onchip_tx_buf); in cdns2_gadget_start()
2271 device_property_read_u16(pdev->dev, "cdns,on-chip-rx-buff-size", in cdns2_gadget_start()
2272 &pdev->onchip_rx_buf); in cdns2_gadget_start()
2273 device_property_read_u32(pdev->dev, "cdns,avail-endpoints", in cdns2_gadget_start()
2274 &pdev->eps_supported); in cdns2_gadget_start()
2280 if (!pdev->onchip_tx_buf && !pdev->onchip_rx_buf) { in cdns2_gadget_start()
2281 ret = -EINVAL; in cdns2_gadget_start()
2282 dev_err(pdev->dev, "Invalid on-chip memory configuration\n"); in cdns2_gadget_start()
2286 if (!(pdev->eps_supported & ~0x00010001)) { in cdns2_gadget_start()
2287 ret = -EINVAL; in cdns2_gadget_start()
2288 dev_err(pdev->dev, "No hardware endpoints available\n"); in cdns2_gadget_start()
2292 max_speed = usb_get_maximum_speed(pdev->dev); in cdns2_gadget_start()
2299 dev_err(pdev->dev, "invalid maximum_speed parameter %d\n", in cdns2_gadget_start()
2307 pdev->gadget.max_speed = max_speed; in cdns2_gadget_start()
2308 pdev->gadget.speed = USB_SPEED_UNKNOWN; in cdns2_gadget_start()
2309 pdev->gadget.ops = &cdns2_gadget_ops; in cdns2_gadget_start()
2310 pdev->gadget.name = "usbhs-gadget"; in cdns2_gadget_start()
2311 pdev->gadget.quirk_avoids_skb_reserve = 1; in cdns2_gadget_start()
2312 pdev->gadget.irq = pdev->irq; in cdns2_gadget_start()
2314 spin_lock_init(&pdev->lock); in cdns2_gadget_start()
2315 INIT_WORK(&pdev->pending_status_wq, cdns2_pending_setup_status_handler); in cdns2_gadget_start()
2318 INIT_LIST_HEAD(&pdev->gadget.ep_list); in cdns2_gadget_start()
2319 pdev->eps_dma_pool = dma_pool_create("cdns2_eps_dma_pool", pdev->dev, in cdns2_gadget_start()
2321 if (!pdev->eps_dma_pool) { in cdns2_gadget_start()
2322 dev_err(pdev->dev, "Failed to create TRB dma pool\n"); in cdns2_gadget_start()
2323 ret = -ENOMEM; in cdns2_gadget_start()
2329 dev_err(pdev->dev, "Failed to create endpoints\n"); in cdns2_gadget_start()
2333 pdev->gadget.sg_supported = 1; in cdns2_gadget_start()
2335 pdev->zlp_buf = kzalloc(CDNS2_EP_ZLP_BUF_SIZE, GFP_KERNEL); in cdns2_gadget_start()
2336 if (!pdev->zlp_buf) { in cdns2_gadget_start()
2337 ret = -ENOMEM; in cdns2_gadget_start()
2342 buf = dma_alloc_coherent(pdev->dev, 8, &pdev->ep0_preq.request.dma, in cdns2_gadget_start()
2344 pdev->ep0_preq.request.buf = buf; in cdns2_gadget_start()
2346 if (!pdev->ep0_preq.request.buf) { in cdns2_gadget_start()
2347 ret = -ENOMEM; in cdns2_gadget_start()
2352 ret = usb_add_gadget(&pdev->gadget); in cdns2_gadget_start()
2354 dev_err(pdev->dev, "Failed to add gadget\n"); in cdns2_gadget_start()
2361 dma_free_coherent(pdev->dev, 8, pdev->ep0_preq.request.buf, in cdns2_gadget_start()
2362 pdev->ep0_preq.request.dma); in cdns2_gadget_start()
2364 kfree(pdev->zlp_buf); in cdns2_gadget_start()
2366 dma_pool_destroy(pdev->eps_dma_pool); in cdns2_gadget_start()
2368 usb_put_gadget(&pdev->gadget); in cdns2_gadget_start()
2379 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_suspend()
2380 pdev->gadget.speed = USB_SPEED_UNKNOWN; in cdns2_gadget_suspend()
2383 usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED); in cdns2_gadget_suspend()
2386 /* Disable interrupt for device. */ in cdns2_gadget_suspend()
2387 writeb(0, &pdev->interrupt_regs->usbien); in cdns2_gadget_suspend()
2388 writel(0, &pdev->adma_regs->ep_ien); in cdns2_gadget_suspend()
2389 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_suspend()
2398 spin_lock_irqsave(&pdev->lock, flags); in cdns2_gadget_resume()
2400 if (!pdev->gadget_driver) { in cdns2_gadget_resume()
2401 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_resume()
2408 clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON); in cdns2_gadget_resume()
2410 spin_unlock_irqrestore(&pdev->lock, flags); in cdns2_gadget_resume()
2417 pm_runtime_mark_last_busy(pdev->dev); in cdns2_gadget_remove()
2418 pm_runtime_put_autosuspend(pdev->dev); in cdns2_gadget_remove()
2420 usb_del_gadget(&pdev->gadget); in cdns2_gadget_remove()
2423 dma_pool_destroy(pdev->eps_dma_pool); in cdns2_gadget_remove()
2424 kfree(pdev->zlp_buf); in cdns2_gadget_remove()
2425 usb_put_gadget(&pdev->gadget); in cdns2_gadget_remove()
2432 /* Ensure 32-bit DMA Mask. */ in cdns2_gadget_init()
2433 ret = dma_set_mask_and_coherent(pdev->dev, DMA_BIT_MASK(32)); in cdns2_gadget_init()
2435 dev_err(pdev->dev, "Failed to set dma mask: %d\n", ret); in cdns2_gadget_init()
2439 pm_runtime_get_sync(pdev->dev); in cdns2_gadget_init()
2445 pm_runtime_put_sync(pdev->dev); in cdns2_gadget_init()
2453 ret = devm_request_threaded_irq(pdev->dev, pdev->irq, in cdns2_gadget_init()
2457 dev_name(pdev->dev), in cdns2_gadget_init()