Lines Matching refs:hs_ep
115 static inline void dwc2_gadget_incr_frame_num(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_incr_frame_num() argument
117 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_incr_frame_num()
123 hs_ep->target_frame += hs_ep->interval; in dwc2_gadget_incr_frame_num()
124 if (hs_ep->target_frame > limit) { in dwc2_gadget_incr_frame_num()
125 hs_ep->frame_overrun = true; in dwc2_gadget_incr_frame_num()
126 hs_ep->target_frame &= limit; in dwc2_gadget_incr_frame_num()
128 hs_ep->frame_overrun = false; in dwc2_gadget_incr_frame_num()
142 static inline void dwc2_gadget_dec_frame_num_by_one(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_dec_frame_num_by_one() argument
144 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_dec_frame_num_by_one()
150 if (hs_ep->target_frame) in dwc2_gadget_dec_frame_num_by_one()
151 hs_ep->target_frame -= 1; in dwc2_gadget_dec_frame_num_by_one()
153 hs_ep->target_frame = limit; in dwc2_gadget_dec_frame_num_by_one()
416 static inline int is_ep_periodic(struct dwc2_hsotg_ep *hs_ep) in is_ep_periodic() argument
418 return hs_ep->periodic; in is_ep_periodic()
431 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_unmap_dma() argument
436 usb_gadget_unmap_request(&hsotg->gadget, req, hs_ep->map_dir); in dwc2_hsotg_unmap_dma()
504 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_write_fifo() argument
507 bool periodic = is_ep_periodic(hs_ep); in dwc2_hsotg_write_fifo()
510 int to_write = hs_ep->size_loaded; in dwc2_hsotg_write_fifo()
516 to_write -= (buf_pos - hs_ep->last_load); in dwc2_hsotg_write_fifo()
523 u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index)); in dwc2_hsotg_write_fifo()
538 if (hs_ep->fifo_load != 0) { in dwc2_hsotg_write_fifo()
545 hs_ep->size_loaded, hs_ep->fifo_load, hs_ep->fifo_size); in dwc2_hsotg_write_fifo()
548 size_done = hs_ep->size_loaded - size_left; in dwc2_hsotg_write_fifo()
551 can_write = hs_ep->fifo_load - size_done; in dwc2_hsotg_write_fifo()
555 can_write = hs_ep->fifo_size - can_write; in dwc2_hsotg_write_fifo()
563 } else if (hsotg->dedicated_fifos && hs_ep->index != 0) { in dwc2_hsotg_write_fifo()
565 DTXFSTS(hs_ep->fifo_index)); in dwc2_hsotg_write_fifo()
583 max_transfer = hs_ep->ep.maxpacket * hs_ep->mc; in dwc2_hsotg_write_fifo()
647 hs_ep->total_data += to_write; in dwc2_hsotg_write_fifo()
650 hs_ep->fifo_load += to_write; in dwc2_hsotg_write_fifo()
655 dwc2_writel_rep(hsotg, EPFIFO(hs_ep->index), data, to_write); in dwc2_hsotg_write_fifo()
667 static unsigned int get_ep_limit(struct dwc2_hsotg_ep *hs_ep) in get_ep_limit() argument
669 int index = hs_ep->index; in get_ep_limit()
678 if (hs_ep->dir_in) in get_ep_limit()
693 if ((maxpkt * hs_ep->ep.maxpacket) < maxsize) in get_ep_limit()
694 maxsize = maxpkt * hs_ep->ep.maxpacket; in get_ep_limit()
725 static unsigned int dwc2_gadget_get_chain_limit(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_get_chain_limit() argument
727 const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc; in dwc2_gadget_get_chain_limit()
728 int is_isoc = hs_ep->isochronous; in dwc2_gadget_get_chain_limit()
730 u32 mps = hs_ep->ep.maxpacket; in dwc2_gadget_get_chain_limit()
731 int dir_in = hs_ep->dir_in; in dwc2_gadget_get_chain_limit()
734 maxsize = (hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_LIMIT : in dwc2_gadget_get_chain_limit()
741 if (hs_ep->index) in dwc2_gadget_get_chain_limit()
765 static u32 dwc2_gadget_get_desc_params(struct dwc2_hsotg_ep *hs_ep, u32 *mask) in dwc2_gadget_get_desc_params() argument
767 const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc; in dwc2_gadget_get_desc_params()
768 u32 mps = hs_ep->ep.maxpacket; in dwc2_gadget_get_desc_params()
769 int dir_in = hs_ep->dir_in; in dwc2_gadget_get_desc_params()
772 if (!hs_ep->index && !dir_in) { in dwc2_gadget_get_desc_params()
775 } else if (hs_ep->isochronous) { in dwc2_gadget_get_desc_params()
792 if (hs_ep->index) in dwc2_gadget_get_desc_params()
801 static void dwc2_gadget_fill_nonisoc_xfer_ddma_one(struct dwc2_hsotg_ep *hs_ep, in dwc2_gadget_fill_nonisoc_xfer_ddma_one() argument
807 int dir_in = hs_ep->dir_in; in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
808 u32 mps = hs_ep->ep.maxpacket; in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
814 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
816 hs_ep->desc_count = (len / maxsize) + in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
819 hs_ep->desc_count = 1; in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
821 for (i = 0; i < hs_ep->desc_count; ++i) { in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
827 if (!hs_ep->index && !dir_in) in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
842 ((hs_ep->send_zlp && true_last) ? in dwc2_gadget_fill_nonisoc_xfer_ddma_one()
867 static void dwc2_gadget_config_nonisoc_xfer_ddma(struct dwc2_hsotg_ep *hs_ep, in dwc2_gadget_config_nonisoc_xfer_ddma() argument
872 struct dwc2_dma_desc *desc = hs_ep->desc_list; in dwc2_gadget_config_nonisoc_xfer_ddma()
877 if (hs_ep->req) in dwc2_gadget_config_nonisoc_xfer_ddma()
878 ureq = &hs_ep->req->req; in dwc2_gadget_config_nonisoc_xfer_ddma()
882 dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc, in dwc2_gadget_config_nonisoc_xfer_ddma()
889 dwc2_gadget_fill_nonisoc_xfer_ddma_one(hs_ep, &desc, in dwc2_gadget_config_nonisoc_xfer_ddma()
892 desc_count += hs_ep->desc_count; in dwc2_gadget_config_nonisoc_xfer_ddma()
895 hs_ep->desc_count = desc_count; in dwc2_gadget_config_nonisoc_xfer_ddma()
909 static int dwc2_gadget_fill_isoc_desc(struct dwc2_hsotg_ep *hs_ep, in dwc2_gadget_fill_isoc_desc() argument
913 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_fill_isoc_desc()
918 dwc2_gadget_get_desc_params(hs_ep, &mask); in dwc2_gadget_fill_isoc_desc()
920 index = hs_ep->next_desc; in dwc2_gadget_fill_isoc_desc()
921 desc = &hs_ep->desc_list[index]; in dwc2_gadget_fill_isoc_desc()
931 if (hs_ep->next_desc) in dwc2_gadget_fill_isoc_desc()
932 hs_ep->desc_list[index - 1].status &= ~DEV_DMA_L; in dwc2_gadget_fill_isoc_desc()
935 __func__, hs_ep->index, hs_ep->dir_in ? "in" : "out", index); in dwc2_gadget_fill_isoc_desc()
944 if (hs_ep->dir_in) { in dwc2_gadget_fill_isoc_desc()
946 pid = DIV_ROUND_UP(len, hs_ep->ep.maxpacket); in dwc2_gadget_fill_isoc_desc()
951 ((len % hs_ep->ep.maxpacket) ? in dwc2_gadget_fill_isoc_desc()
953 ((hs_ep->target_frame << in dwc2_gadget_fill_isoc_desc()
962 if (hs_ep->dir_in) in dwc2_gadget_fill_isoc_desc()
963 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_fill_isoc_desc()
966 hs_ep->next_desc++; in dwc2_gadget_fill_isoc_desc()
967 if (hs_ep->next_desc >= MAX_DMA_DESC_NUM_HS_ISOC) in dwc2_gadget_fill_isoc_desc()
968 hs_ep->next_desc = 0; in dwc2_gadget_fill_isoc_desc()
980 static void dwc2_gadget_start_isoc_ddma(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_start_isoc_ddma() argument
982 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_start_isoc_ddma()
984 int index = hs_ep->index; in dwc2_gadget_start_isoc_ddma()
992 if (list_empty(&hs_ep->queue)) { in dwc2_gadget_start_isoc_ddma()
993 hs_ep->target_frame = TARGET_FRAME_INITIAL; in dwc2_gadget_start_isoc_ddma()
1000 desc = &hs_ep->desc_list[i]; in dwc2_gadget_start_isoc_ddma()
1006 hs_ep->next_desc = 0; in dwc2_gadget_start_isoc_ddma()
1007 list_for_each_entry_safe(hs_req, treq, &hs_ep->queue, queue) { in dwc2_gadget_start_isoc_ddma()
1014 ret = dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr, in dwc2_gadget_start_isoc_ddma()
1020 hs_ep->compl_desc = 0; in dwc2_gadget_start_isoc_ddma()
1021 depctl = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); in dwc2_gadget_start_isoc_ddma()
1022 dma_reg = hs_ep->dir_in ? DIEPDMA(index) : DOEPDMA(index); in dwc2_gadget_start_isoc_ddma()
1025 dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg); in dwc2_gadget_start_isoc_ddma()
1032 static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep);
1034 struct dwc2_hsotg_ep *hs_ep,
1049 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_start_req() argument
1054 int index = hs_ep->index; in dwc2_hsotg_start_req()
1055 int dir_in = hs_ep->dir_in; in dwc2_hsotg_start_req()
1066 if (hs_ep->req && !continuing) { in dwc2_hsotg_start_req()
1070 } else if (hs_ep->req != hs_req && continuing) { in dwc2_hsotg_start_req()
1084 hs_ep->dir_in ? "in" : "out"); in dwc2_hsotg_start_req()
1099 maxreq = get_ep_limit(hs_ep); in dwc2_hsotg_start_req()
1101 maxreq = dwc2_gadget_get_chain_limit(hs_ep); in dwc2_hsotg_start_req()
1104 int round = maxreq % hs_ep->ep.maxpacket; in dwc2_hsotg_start_req()
1117 packets = DIV_ROUND_UP(length, hs_ep->ep.maxpacket); in dwc2_hsotg_start_req()
1122 if (hs_ep->isochronous) in dwc2_hsotg_start_req()
1135 if ((ureq->length >= hs_ep->ep.maxpacket) && in dwc2_hsotg_start_req()
1136 !(ureq->length % hs_ep->ep.maxpacket)) in dwc2_hsotg_start_req()
1137 hs_ep->send_zlp = 1; in dwc2_hsotg_start_req()
1147 hs_ep->req = hs_req; in dwc2_hsotg_start_req()
1151 u32 mps = hs_ep->ep.maxpacket; in dwc2_hsotg_start_req()
1165 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, ureq->dma + offset, in dwc2_hsotg_start_req()
1169 dwc2_writel(hsotg, hs_ep->desc_list_dma, dma_reg); in dwc2_hsotg_start_req()
1172 __func__, (u32)hs_ep->desc_list_dma, dma_reg); in dwc2_hsotg_start_req()
1190 if (hs_ep->isochronous) { in dwc2_hsotg_start_req()
1191 if (!dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_hsotg_start_req()
1192 if (hs_ep->interval == 1) { in dwc2_hsotg_start_req()
1193 if (hs_ep->target_frame & 0x1) in dwc2_hsotg_start_req()
1200 hs_req->req.frame_number = hs_ep->target_frame; in dwc2_hsotg_start_req()
1202 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA); in dwc2_hsotg_start_req()
1223 hs_ep->size_loaded = length; in dwc2_hsotg_start_req()
1224 hs_ep->last_load = ureq->actual; in dwc2_hsotg_start_req()
1228 hs_ep->fifo_load = 0; in dwc2_hsotg_start_req()
1230 dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req); in dwc2_hsotg_start_req()
1248 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 1); in dwc2_hsotg_start_req()
1264 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_map_dma() argument
1269 hs_ep->map_dir = hs_ep->dir_in; in dwc2_hsotg_map_dma()
1270 ret = usb_gadget_map_request(&hsotg->gadget, req, hs_ep->dir_in); in dwc2_hsotg_map_dma()
1284 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_handle_unaligned_buf_start() argument
1296 hs_ep->ep.name, req_buf, hs_req->req.length); in dwc2_hsotg_handle_unaligned_buf_start()
1310 if (hs_ep->dir_in) in dwc2_hsotg_handle_unaligned_buf_start()
1317 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_handle_unaligned_buf_complete() argument
1325 hs_ep->ep.name, hs_req->req.status, hs_req->req.actual); in dwc2_hsotg_handle_unaligned_buf_complete()
1328 if (!hs_ep->dir_in && !hs_req->req.status) in dwc2_hsotg_handle_unaligned_buf_complete()
1346 static bool dwc2_gadget_target_frame_elapsed(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_target_frame_elapsed() argument
1348 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_target_frame_elapsed()
1349 u32 target_frame = hs_ep->target_frame; in dwc2_gadget_target_frame_elapsed()
1351 bool frame_overrun = hs_ep->frame_overrun; in dwc2_gadget_target_frame_elapsed()
1376 struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_set_ep0_desc_chain() argument
1381 hs_ep->desc_list = hsotg->setup_desc[0]; in dwc2_gadget_set_ep0_desc_chain()
1382 hs_ep->desc_list_dma = hsotg->setup_desc_dma[0]; in dwc2_gadget_set_ep0_desc_chain()
1386 hs_ep->desc_list = hsotg->ctrl_in_desc; in dwc2_gadget_set_ep0_desc_chain()
1387 hs_ep->desc_list_dma = hsotg->ctrl_in_desc_dma; in dwc2_gadget_set_ep0_desc_chain()
1390 hs_ep->desc_list = hsotg->ctrl_out_desc; in dwc2_gadget_set_ep0_desc_chain()
1391 hs_ep->desc_list_dma = hsotg->ctrl_out_desc_dma; in dwc2_gadget_set_ep0_desc_chain()
1406 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_queue() local
1407 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_queue()
1435 if (hs_ep->isochronous && in dwc2_hsotg_ep_queue()
1436 req->length > (hs_ep->mc * hs_ep->ep.maxpacket)) { in dwc2_hsotg_ep_queue()
1444 if (using_desc_dma(hs) && hs_ep->isochronous) { in dwc2_hsotg_ep_queue()
1445 maxsize = dwc2_gadget_get_desc_params(hs_ep, &mask); in dwc2_hsotg_ep_queue()
1446 if (hs_ep->dir_in && req->length > maxsize) { in dwc2_hsotg_ep_queue()
1452 if (!hs_ep->dir_in && req->length > hs_ep->ep.maxpacket) { in dwc2_hsotg_ep_queue()
1454 req->length, hs_ep->ep.maxpacket); in dwc2_hsotg_ep_queue()
1459 ret = dwc2_hsotg_handle_unaligned_buf_start(hs, hs_ep, hs_req); in dwc2_hsotg_ep_queue()
1465 ret = dwc2_hsotg_map_dma(hs, hs_ep, req); in dwc2_hsotg_ep_queue()
1470 if (using_desc_dma(hs) && !hs_ep->index) { in dwc2_hsotg_ep_queue()
1471 ret = dwc2_gadget_set_ep0_desc_chain(hs, hs_ep); in dwc2_hsotg_ep_queue()
1476 first = list_empty(&hs_ep->queue); in dwc2_hsotg_ep_queue()
1477 list_add_tail(&hs_req->queue, &hs_ep->queue); in dwc2_hsotg_ep_queue()
1485 if (using_desc_dma(hs) && hs_ep->isochronous) { in dwc2_hsotg_ep_queue()
1486 if (hs_ep->target_frame != TARGET_FRAME_INITIAL) { in dwc2_hsotg_ep_queue()
1493 dwc2_gadget_fill_isoc_desc(hs_ep, dma_addr, in dwc2_hsotg_ep_queue()
1500 if (!hs_ep->index && !req->length && !hs_ep->dir_in && in dwc2_hsotg_ep_queue()
1502 hs_ep->dir_in = 1; in dwc2_hsotg_ep_queue()
1505 if (!hs_ep->isochronous) { in dwc2_hsotg_ep_queue()
1506 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false); in dwc2_hsotg_ep_queue()
1512 while (dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_hsotg_ep_queue()
1513 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_hsotg_ep_queue()
1520 if (hs_ep->target_frame != TARGET_FRAME_INITIAL) in dwc2_hsotg_ep_queue()
1521 dwc2_hsotg_start_req(hs, hs_ep, hs_req, false); in dwc2_hsotg_ep_queue()
1529 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_queue_lock() local
1530 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_queue_lock()
1560 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_complete_oursetup() local
1561 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_complete_oursetup()
1732 static struct dwc2_hsotg_req *get_ep_head(struct dwc2_hsotg_ep *hs_ep) in get_ep_head() argument
1734 return list_first_entry_or_null(&hs_ep->queue, struct dwc2_hsotg_req, in get_ep_head()
1746 static void dwc2_gadget_start_next_request(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_start_next_request() argument
1748 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_start_next_request()
1749 int dir_in = hs_ep->dir_in; in dwc2_gadget_start_next_request()
1752 if (!list_empty(&hs_ep->queue)) { in dwc2_gadget_start_next_request()
1753 hs_req = get_ep_head(hs_ep); in dwc2_gadget_start_next_request()
1754 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, false); in dwc2_gadget_start_next_request()
1757 if (!hs_ep->isochronous) in dwc2_gadget_start_next_request()
2015 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_complete_setup() local
2016 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_complete_setup()
2071 struct dwc2_hsotg_ep *hs_ep) in dwc2_hsotg_program_zlp() argument
2074 u8 index = hs_ep->index; in dwc2_hsotg_program_zlp()
2075 u32 epctl_reg = hs_ep->dir_in ? DIEPCTL(index) : DOEPCTL(index); in dwc2_hsotg_program_zlp()
2076 u32 epsiz_reg = hs_ep->dir_in ? DIEPTSIZ(index) : DOEPTSIZ(index); in dwc2_hsotg_program_zlp()
2078 if (hs_ep->dir_in) in dwc2_hsotg_program_zlp()
2086 dma_addr_t dma = hs_ep->desc_list_dma; in dwc2_hsotg_program_zlp()
2089 dwc2_gadget_set_ep0_desc_chain(hsotg, hs_ep); in dwc2_hsotg_program_zlp()
2091 dwc2_gadget_config_nonisoc_xfer_ddma(hs_ep, dma, 0); in dwc2_hsotg_program_zlp()
2119 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_complete_request() argument
2129 hs_ep, hs_ep->ep.name, hs_req, result, hs_req->req.complete); in dwc2_hsotg_complete_request()
2140 dwc2_hsotg_unmap_dma(hsotg, hs_ep, hs_req); in dwc2_hsotg_complete_request()
2142 dwc2_hsotg_handle_unaligned_buf_complete(hsotg, hs_ep, hs_req); in dwc2_hsotg_complete_request()
2144 hs_ep->req = NULL; in dwc2_hsotg_complete_request()
2154 usb_gadget_giveback_request(&hs_ep->ep, &hs_req->req); in dwc2_hsotg_complete_request()
2159 if (using_desc_dma(hsotg) && hs_ep->isochronous) in dwc2_hsotg_complete_request()
2168 if (!hs_ep->req && result >= 0) in dwc2_hsotg_complete_request()
2169 dwc2_gadget_start_next_request(hs_ep); in dwc2_hsotg_complete_request()
2181 static void dwc2_gadget_complete_isoc_request_ddma(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_complete_isoc_request_ddma() argument
2183 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_complete_isoc_request_ddma()
2189 desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status; in dwc2_gadget_complete_isoc_request_ddma()
2195 hs_req = get_ep_head(hs_ep); in dwc2_gadget_complete_isoc_request_ddma()
2205 mask = hs_ep->dir_in ? DEV_DMA_ISOC_TX_NBYTES_MASK : in dwc2_gadget_complete_isoc_request_ddma()
2213 if (!hs_ep->dir_in && ureq->length & 0x3) in dwc2_gadget_complete_isoc_request_ddma()
2222 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); in dwc2_gadget_complete_isoc_request_ddma()
2224 hs_ep->compl_desc++; in dwc2_gadget_complete_isoc_request_ddma()
2225 if (hs_ep->compl_desc > (MAX_DMA_DESC_NUM_HS_ISOC - 1)) in dwc2_gadget_complete_isoc_request_ddma()
2226 hs_ep->compl_desc = 0; in dwc2_gadget_complete_isoc_request_ddma()
2227 desc_sts = hs_ep->desc_list[hs_ep->compl_desc].status; in dwc2_gadget_complete_isoc_request_ddma()
2240 static void dwc2_gadget_handle_isoc_bna(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_handle_isoc_bna() argument
2242 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_handle_isoc_bna()
2244 if (!hs_ep->dir_in) in dwc2_gadget_handle_isoc_bna()
2246 dwc2_hsotg_complete_request(hsotg, hs_ep, get_ep_head(hs_ep), 0); in dwc2_gadget_handle_isoc_bna()
2248 hs_ep->target_frame = TARGET_FRAME_INITIAL; in dwc2_gadget_handle_isoc_bna()
2249 hs_ep->next_desc = 0; in dwc2_gadget_handle_isoc_bna()
2250 hs_ep->compl_desc = 0; in dwc2_gadget_handle_isoc_bna()
2265 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[ep_idx]; in dwc2_hsotg_rx_data() local
2266 struct dwc2_hsotg_req *hs_req = hs_ep->req; in dwc2_hsotg_rx_data()
2303 hs_ep->total_data += to_read; in dwc2_hsotg_rx_data()
2343 static unsigned int dwc2_gadget_get_xfersize_ddma(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_get_xfersize_ddma() argument
2345 const struct usb_endpoint_descriptor *ep_desc = hs_ep->ep.desc; in dwc2_gadget_get_xfersize_ddma()
2346 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_get_xfersize_ddma()
2349 struct dwc2_dma_desc *desc = hs_ep->desc_list; in dwc2_gadget_get_xfersize_ddma()
2352 u32 mps = hs_ep->ep.maxpacket; in dwc2_gadget_get_xfersize_ddma()
2353 int dir_in = hs_ep->dir_in; in dwc2_gadget_get_xfersize_ddma()
2359 if (hs_ep->index) in dwc2_gadget_get_xfersize_ddma()
2363 for (i = 0; i < hs_ep->desc_count; ++i) { in dwc2_gadget_get_xfersize_ddma()
2393 struct dwc2_hsotg_ep *hs_ep = hsotg->eps_out[epnum]; in dwc2_hsotg_handle_outdone() local
2394 struct dwc2_hsotg_req *hs_req = hs_ep->req; in dwc2_hsotg_handle_outdone()
2406 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); in dwc2_hsotg_handle_outdone()
2412 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); in dwc2_hsotg_handle_outdone()
2426 size_done = hs_ep->size_loaded - size_left; in dwc2_hsotg_handle_outdone()
2427 size_done += hs_ep->last_load; in dwc2_hsotg_handle_outdone()
2434 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true); in dwc2_hsotg_handle_outdone()
2457 if (!using_desc_dma(hsotg) && hs_ep->isochronous) { in dwc2_hsotg_handle_outdone()
2458 req->frame_number = hs_ep->target_frame; in dwc2_hsotg_handle_outdone()
2459 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_hsotg_handle_outdone()
2462 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, result); in dwc2_hsotg_handle_outdone()
2585 struct dwc2_hsotg_ep *hs_ep; in dwc2_hsotg_set_ep_maxpacket() local
2588 hs_ep = index_to_ep(hsotg, ep, dir_in); in dwc2_hsotg_set_ep_maxpacket()
2589 if (!hs_ep) in dwc2_hsotg_set_ep_maxpacket()
2599 hs_ep->ep.maxpacket = mps_bytes; in dwc2_hsotg_set_ep_maxpacket()
2600 hs_ep->mc = 1; in dwc2_hsotg_set_ep_maxpacket()
2604 hs_ep->mc = mc; in dwc2_hsotg_set_ep_maxpacket()
2607 hs_ep->ep.maxpacket = mps; in dwc2_hsotg_set_ep_maxpacket()
2653 struct dwc2_hsotg_ep *hs_ep) in dwc2_hsotg_trytx() argument
2655 struct dwc2_hsotg_req *hs_req = hs_ep->req; in dwc2_hsotg_trytx()
2657 if (!hs_ep->dir_in || !hs_req) { in dwc2_hsotg_trytx()
2662 if (hs_ep->index != 0) in dwc2_hsotg_trytx()
2663 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, in dwc2_hsotg_trytx()
2664 hs_ep->dir_in, 0); in dwc2_hsotg_trytx()
2670 hs_ep->index); in dwc2_hsotg_trytx()
2671 return dwc2_hsotg_write_fifo(hsotg, hs_ep, hs_req); in dwc2_hsotg_trytx()
2686 struct dwc2_hsotg_ep *hs_ep) in dwc2_hsotg_complete_in() argument
2688 struct dwc2_hsotg_req *hs_req = hs_ep->req; in dwc2_hsotg_complete_in()
2689 u32 epsize = dwc2_readl(hsotg, DIEPTSIZ(hs_ep->index)); in dwc2_hsotg_complete_in()
2698 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_STATUS_IN) { in dwc2_hsotg_complete_in()
2705 hs_ep->dir_in = 0; in dwc2_hsotg_complete_in()
2707 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); in dwc2_hsotg_complete_in()
2733 size_left = dwc2_gadget_get_xfersize_ddma(hs_ep); in dwc2_hsotg_complete_in()
2741 size_done = hs_ep->size_loaded - size_left; in dwc2_hsotg_complete_in()
2742 size_done += hs_ep->last_load; in dwc2_hsotg_complete_in()
2754 dwc2_hsotg_start_req(hsotg, hs_ep, hs_req, true); in dwc2_hsotg_complete_in()
2759 if (hs_ep->send_zlp) { in dwc2_hsotg_complete_in()
2760 hs_ep->send_zlp = 0; in dwc2_hsotg_complete_in()
2762 dwc2_hsotg_program_zlp(hsotg, hs_ep); in dwc2_hsotg_complete_in()
2768 if (hs_ep->index == 0 && hsotg->ep0_state == DWC2_EP0_DATA_IN) { in dwc2_hsotg_complete_in()
2775 if (!using_desc_dma(hsotg) && hs_ep->isochronous) { in dwc2_hsotg_complete_in()
2776 hs_req->req.frame_number = hs_ep->target_frame; in dwc2_hsotg_complete_in()
2777 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_hsotg_complete_in()
2780 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, 0); in dwc2_hsotg_complete_in()
2824 static void dwc2_gadget_handle_ep_disabled(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_handle_ep_disabled() argument
2826 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_handle_ep_disabled()
2828 unsigned char idx = hs_ep->index; in dwc2_gadget_handle_ep_disabled()
2829 int dir_in = hs_ep->dir_in; in dwc2_gadget_handle_ep_disabled()
2838 dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index); in dwc2_gadget_handle_ep_disabled()
2854 if (!hs_ep->isochronous) in dwc2_gadget_handle_ep_disabled()
2857 if (list_empty(&hs_ep->queue)) { in dwc2_gadget_handle_ep_disabled()
2859 __func__, hs_ep); in dwc2_gadget_handle_ep_disabled()
2864 hs_req = get_ep_head(hs_ep); in dwc2_gadget_handle_ep_disabled()
2866 hs_req->req.frame_number = hs_ep->target_frame; in dwc2_gadget_handle_ep_disabled()
2868 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, in dwc2_gadget_handle_ep_disabled()
2871 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_handle_ep_disabled()
2874 } while (dwc2_gadget_target_frame_elapsed(hs_ep)); in dwc2_gadget_handle_ep_disabled()
2940 struct dwc2_hsotg_ep *hs_ep);
2956 static void dwc2_gadget_handle_nak(struct dwc2_hsotg_ep *hs_ep) in dwc2_gadget_handle_nak() argument
2958 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_gadget_handle_nak()
2960 int dir_in = hs_ep->dir_in; in dwc2_gadget_handle_nak()
2963 if (!dir_in || !hs_ep->isochronous) in dwc2_gadget_handle_nak()
2966 if (hs_ep->target_frame == TARGET_FRAME_INITIAL) { in dwc2_gadget_handle_nak()
2969 hs_ep->target_frame = hsotg->frame_number; in dwc2_gadget_handle_nak()
2970 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_handle_nak()
2979 hs_ep->target_frame &= ~hs_ep->interval + 1; in dwc2_gadget_handle_nak()
2984 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_handle_nak()
2985 dwc2_gadget_dec_frame_num_by_one(hs_ep); in dwc2_gadget_handle_nak()
2988 dwc2_gadget_start_isoc_ddma(hs_ep); in dwc2_gadget_handle_nak()
2992 hs_ep->target_frame = hsotg->frame_number; in dwc2_gadget_handle_nak()
2993 if (hs_ep->interval > 1) { in dwc2_gadget_handle_nak()
2995 DIEPCTL(hs_ep->index)); in dwc2_gadget_handle_nak()
2996 if (hs_ep->target_frame & 0x1) in dwc2_gadget_handle_nak()
3001 dwc2_writel(hsotg, ctrl, DIEPCTL(hs_ep->index)); in dwc2_gadget_handle_nak()
3008 ctrl = dwc2_readl(hsotg, DIEPCTL(hs_ep->index)); in dwc2_gadget_handle_nak()
3010 dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep); in dwc2_gadget_handle_nak()
3012 dwc2_hsotg_txfifo_flush(hsotg, hs_ep->fifo_index); in dwc2_gadget_handle_nak()
3014 while (dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_gadget_handle_nak()
3015 hs_req = get_ep_head(hs_ep); in dwc2_gadget_handle_nak()
3017 hs_req->req.frame_number = hs_ep->target_frame; in dwc2_gadget_handle_nak()
3019 dwc2_hsotg_complete_request(hsotg, hs_ep, hs_req, -ENODATA); in dwc2_gadget_handle_nak()
3022 dwc2_gadget_incr_frame_num(hs_ep); in dwc2_gadget_handle_nak()
3027 if (!hs_ep->req) in dwc2_gadget_handle_nak()
3028 dwc2_gadget_start_next_request(hs_ep); in dwc2_gadget_handle_nak()
3042 struct dwc2_hsotg_ep *hs_ep = index_to_ep(hsotg, idx, dir_in); in dwc2_hsotg_epint() local
3053 if (!hs_ep) { in dwc2_hsotg_epint()
3072 if (using_desc_dma(hsotg) && idx == 0 && !hs_ep->dir_in && in dwc2_hsotg_epint()
3083 if (using_desc_dma(hsotg) && hs_ep->isochronous) { in dwc2_hsotg_epint()
3084 dwc2_gadget_complete_isoc_request_ddma(hs_ep); in dwc2_hsotg_epint()
3091 if (!hs_ep->isochronous || !(ints & DXEPINT_NAKINTRPT)) in dwc2_hsotg_epint()
3092 dwc2_hsotg_complete_in(hsotg, hs_ep); in dwc2_hsotg_epint()
3094 if (idx == 0 && !hs_ep->req) in dwc2_hsotg_epint()
3101 if (!hs_ep->isochronous || !(ints & DXEPINT_OUTTKNEPDIS)) in dwc2_hsotg_epint()
3107 dwc2_gadget_handle_ep_disabled(hs_ep); in dwc2_hsotg_epint()
3110 dwc2_gadget_handle_out_token_ep_disabled(hs_ep); in dwc2_hsotg_epint()
3113 dwc2_gadget_handle_nak(hs_ep); in dwc2_hsotg_epint()
3165 if (hs_ep->isochronous) in dwc2_hsotg_epint()
3166 dwc2_gadget_handle_isoc_bna(hs_ep); in dwc2_hsotg_epint()
3169 if (dir_in && !hs_ep->isochronous) { in dwc2_hsotg_epint()
3188 dwc2_hsotg_trytx(hsotg, hs_ep); in dwc2_hsotg_epint()
3619 struct dwc2_hsotg_ep *hs_ep; in dwc2_gadget_handle_incomplete_isoc_in() local
3629 hs_ep = hsotg->eps_in[idx]; in dwc2_gadget_handle_incomplete_isoc_in()
3631 if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous) in dwc2_gadget_handle_incomplete_isoc_in()
3636 dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_gadget_handle_incomplete_isoc_in()
3666 struct dwc2_hsotg_ep *hs_ep; in dwc2_gadget_handle_incomplete_isoc_out() local
3675 hs_ep = hsotg->eps_out[idx]; in dwc2_gadget_handle_incomplete_isoc_out()
3677 if ((BIT(idx) & ~daintmsk) || !hs_ep->isochronous) in dwc2_gadget_handle_incomplete_isoc_out()
3682 dwc2_gadget_target_frame_elapsed(hs_ep)) { in dwc2_gadget_handle_incomplete_isoc_out()
3844 struct dwc2_hsotg_ep *hs_ep; in dwc2_hsotg_irq() local
3855 hs_ep = hsotg->eps_out[idx]; in dwc2_hsotg_irq()
3863 if ((epctrl & DXEPCTL_EPENA) && hs_ep->isochronous) { in dwc2_hsotg_irq()
3871 if (hs_ep->halted) { in dwc2_hsotg_irq()
3915 struct dwc2_hsotg_ep *hs_ep) in dwc2_hsotg_ep_stop_xfr() argument
3920 epctrl_reg = hs_ep->dir_in ? DIEPCTL(hs_ep->index) : in dwc2_hsotg_ep_stop_xfr()
3921 DOEPCTL(hs_ep->index); in dwc2_hsotg_ep_stop_xfr()
3922 epint_reg = hs_ep->dir_in ? DIEPINT(hs_ep->index) : in dwc2_hsotg_ep_stop_xfr()
3923 DOEPINT(hs_ep->index); in dwc2_hsotg_ep_stop_xfr()
3926 hs_ep->name); in dwc2_hsotg_ep_stop_xfr()
3928 if (hs_ep->dir_in) { in dwc2_hsotg_ep_stop_xfr()
3929 if (hsotg->dedicated_fifos || hs_ep->periodic) { in dwc2_hsotg_ep_stop_xfr()
3986 if (hs_ep->dir_in) { in dwc2_hsotg_ep_stop_xfr()
3989 if (hsotg->dedicated_fifos || hs_ep->periodic) in dwc2_hsotg_ep_stop_xfr()
3990 fifo_index = hs_ep->fifo_index; in dwc2_hsotg_ep_stop_xfr()
3998 if (!hsotg->dedicated_fifos && !hs_ep->periodic) in dwc2_hsotg_ep_stop_xfr()
4017 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_enable() local
4018 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_ep_enable()
4020 unsigned int index = hs_ep->index; in dwc2_hsotg_ep_enable()
4044 if (dir_in != hs_ep->dir_in) { in dwc2_hsotg_ep_enable()
4083 if (using_desc_dma(hsotg) && !hs_ep->desc_list) { in dwc2_hsotg_ep_enable()
4084 hs_ep->desc_list = dmam_alloc_coherent(hsotg->dev, in dwc2_hsotg_ep_enable()
4086 &hs_ep->desc_list_dma, GFP_ATOMIC); in dwc2_hsotg_ep_enable()
4087 if (!hs_ep->desc_list) { in dwc2_hsotg_ep_enable()
4105 dwc2_hsotg_set_ep_maxpacket(hsotg, hs_ep->index, mps, mc, dir_in); in dwc2_hsotg_ep_enable()
4108 hs_ep->isochronous = 0; in dwc2_hsotg_ep_enable()
4109 hs_ep->periodic = 0; in dwc2_hsotg_ep_enable()
4110 hs_ep->halted = 0; in dwc2_hsotg_ep_enable()
4111 hs_ep->wedged = 0; in dwc2_hsotg_ep_enable()
4112 hs_ep->interval = desc->bInterval; in dwc2_hsotg_ep_enable()
4118 hs_ep->isochronous = 1; in dwc2_hsotg_ep_enable()
4119 hs_ep->interval = 1 << (desc->bInterval - 1); in dwc2_hsotg_ep_enable()
4120 hs_ep->target_frame = TARGET_FRAME_INITIAL; in dwc2_hsotg_ep_enable()
4121 hs_ep->next_desc = 0; in dwc2_hsotg_ep_enable()
4122 hs_ep->compl_desc = 0; in dwc2_hsotg_ep_enable()
4124 hs_ep->periodic = 1; in dwc2_hsotg_ep_enable()
4142 hs_ep->periodic = 1; in dwc2_hsotg_ep_enable()
4145 hs_ep->interval = 1 << (desc->bInterval - 1); in dwc2_hsotg_ep_enable()
4164 size = hs_ep->ep.maxpacket * hs_ep->mc; in dwc2_hsotg_ep_enable()
4187 hs_ep->fifo_index = fifo_index; in dwc2_hsotg_ep_enable()
4188 hs_ep->fifo_size = fifo_size; in dwc2_hsotg_ep_enable()
4192 if (index && !hs_ep->isochronous) in dwc2_hsotg_ep_enable()
4202 hs_ep->isochronous && dir_in) { in dwc2_hsotg_ep_enable()
4230 if (ret && using_desc_dma(hsotg) && hs_ep->desc_list) { in dwc2_hsotg_ep_enable()
4233 hs_ep->desc_list, hs_ep->desc_list_dma); in dwc2_hsotg_ep_enable()
4234 hs_ep->desc_list = NULL; in dwc2_hsotg_ep_enable()
4246 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_disable() local
4247 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_ep_disable()
4248 int dir_in = hs_ep->dir_in; in dwc2_hsotg_ep_disable()
4249 int index = hs_ep->index; in dwc2_hsotg_ep_disable()
4270 dwc2_hsotg_ep_stop_xfr(hsotg, hs_ep); in dwc2_hsotg_ep_disable()
4280 dwc2_hsotg_ctrl_epint(hsotg, hs_ep->index, hs_ep->dir_in, 0); in dwc2_hsotg_ep_disable()
4283 kill_all_requests(hsotg, hs_ep, -ESHUTDOWN); in dwc2_hsotg_ep_disable()
4285 hsotg->fifo_map &= ~(1 << hs_ep->fifo_index); in dwc2_hsotg_ep_disable()
4286 hs_ep->fifo_index = 0; in dwc2_hsotg_ep_disable()
4287 hs_ep->fifo_size = 0; in dwc2_hsotg_ep_disable()
4294 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_disable_lock() local
4295 struct dwc2_hsotg *hsotg = hs_ep->parent; in dwc2_hsotg_ep_disable_lock()
4330 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_dequeue() local
4331 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_dequeue()
4338 if (!on_list(hs_ep, hs_req)) { in dwc2_hsotg_ep_dequeue()
4344 if (req == &hs_ep->req->req) in dwc2_hsotg_ep_dequeue()
4345 dwc2_hsotg_ep_stop_xfr(hs, hs_ep); in dwc2_hsotg_ep_dequeue()
4347 dwc2_hsotg_complete_request(hs, hs_ep, hs_req, -ECONNRESET); in dwc2_hsotg_ep_dequeue()
4360 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_gadget_ep_set_wedge() local
4361 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_gadget_ep_set_wedge()
4367 hs_ep->wedged = 1; in dwc2_gadget_ep_set_wedge()
4386 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_sethalt() local
4387 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_sethalt()
4388 int index = hs_ep->index; in dwc2_hsotg_ep_sethalt()
4404 if (hs_ep->isochronous) { in dwc2_hsotg_ep_sethalt()
4409 if (!now && value && !list_empty(&hs_ep->queue)) { in dwc2_hsotg_ep_sethalt()
4415 if (hs_ep->dir_in) { in dwc2_hsotg_ep_sethalt()
4425 hs_ep->wedged = 0; in dwc2_hsotg_ep_sethalt()
4445 hs_ep->wedged = 0; in dwc2_hsotg_ep_sethalt()
4454 hs_ep->halted = value; in dwc2_hsotg_ep_sethalt()
4465 struct dwc2_hsotg_ep *hs_ep = our_ep(ep); in dwc2_hsotg_ep_sethalt_lock() local
4466 struct dwc2_hsotg *hs = hs_ep->parent; in dwc2_hsotg_ep_sethalt_lock()
4798 struct dwc2_hsotg_ep *hs_ep, in dwc2_hsotg_initep() argument
4811 hs_ep->dir_in = dir_in; in dwc2_hsotg_initep()
4812 hs_ep->index = epnum; in dwc2_hsotg_initep()
4814 snprintf(hs_ep->name, sizeof(hs_ep->name), "ep%d%s", epnum, dir); in dwc2_hsotg_initep()
4816 INIT_LIST_HEAD(&hs_ep->queue); in dwc2_hsotg_initep()
4817 INIT_LIST_HEAD(&hs_ep->ep.ep_list); in dwc2_hsotg_initep()
4821 list_add_tail(&hs_ep->ep.ep_list, &hsotg->gadget.ep_list); in dwc2_hsotg_initep()
4823 hs_ep->parent = hsotg; in dwc2_hsotg_initep()
4824 hs_ep->ep.name = hs_ep->name; in dwc2_hsotg_initep()
4827 usb_ep_set_maxpacket_limit(&hs_ep->ep, 8); in dwc2_hsotg_initep()
4829 usb_ep_set_maxpacket_limit(&hs_ep->ep, in dwc2_hsotg_initep()
4831 hs_ep->ep.ops = &dwc2_hsotg_ep_ops; in dwc2_hsotg_initep()
4834 hs_ep->ep.caps.type_control = true; in dwc2_hsotg_initep()
4837 hs_ep->ep.caps.type_iso = true; in dwc2_hsotg_initep()
4838 hs_ep->ep.caps.type_bulk = true; in dwc2_hsotg_initep()
4840 hs_ep->ep.caps.type_int = true; in dwc2_hsotg_initep()
4844 hs_ep->ep.caps.dir_in = true; in dwc2_hsotg_initep()
4846 hs_ep->ep.caps.dir_out = true; in dwc2_hsotg_initep()