344 int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)  in cdnsp_ep_enqueue()  argument
346 	struct cdnsp_device *pdev = pep->pdev;  in cdnsp_ep_enqueue()
350 	if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {  in cdnsp_ep_enqueue()
358 	preq->direction = pep->direction;  in cdnsp_ep_enqueue()
359 	preq->epnum = pep->number;  in cdnsp_ep_enqueue()
362 	ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);  in cdnsp_ep_enqueue()
368 	list_add_tail(&preq->list, &pep->pending_list);  in cdnsp_ep_enqueue()
372 	switch (usb_endpoint_type(pep->endpoint.desc)) {  in cdnsp_ep_enqueue()
391 					pep->direction);  in cdnsp_ep_enqueue()
423 int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)  in cdnsp_ep_dequeue()  argument
425 	struct cdnsp_device *pdev = pep->pdev;  in cdnsp_ep_dequeue()
431 	if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)  in cdnsp_ep_dequeue()
432 		ret_stop = cdnsp_cmd_stop_ep(pdev, pep);  in cdnsp_ep_dequeue()
434 	ret_rem = cdnsp_remove_request(pdev, preq, pep);  in cdnsp_ep_dequeue()
488 				       struct cdnsp_ep *pep)  in cdnsp_invalidate_ep_events()  argument
507 		    TRB_TO_EP_ID(data) == (pep->idx + 1)) {  in cdnsp_invalidate_ep_events()
596 			struct cdnsp_ep *pep,  in cdnsp_halt_endpoint()  argument
603 	ret = cdnsp_cmd_stop_ep(pdev, pep);  in cdnsp_halt_endpoint()
608 		if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {  in cdnsp_halt_endpoint()
609 			cdnsp_queue_halt_endpoint(pdev, pep->idx);  in cdnsp_halt_endpoint()
614 		pep->ep_state |= EP_HALTED;  in cdnsp_halt_endpoint()
616 		cdnsp_queue_reset_ep(pdev, pep->idx);  in cdnsp_halt_endpoint()
619 		trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);  in cdnsp_halt_endpoint()
624 		pep->ep_state &= ~EP_HALTED;  in cdnsp_halt_endpoint()
626 		if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))  in cdnsp_halt_endpoint()
627 			cdnsp_ring_doorbell_for_active_rings(pdev, pep);  in cdnsp_halt_endpoint()
629 		pep->ep_state &= ~EP_WEDGE;  in cdnsp_halt_endpoint()
636 					  struct cdnsp_ep *pep)  in cdnsp_update_eps_configuration()  argument
667 	ep_sts = GET_EP_CTX_STATE(pep->out_ctx);  in cdnsp_update_eps_configuration()
675 	trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);  in cdnsp_update_eps_configuration()
765 int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)  in cdnsp_alloc_streams()  argument
767 	unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);  in cdnsp_alloc_streams()
787 	trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);  in cdnsp_alloc_streams()
789 	ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);  in cdnsp_alloc_streams()
793 	cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);  in cdnsp_alloc_streams()
795 	pep->ep_state |= EP_HAS_STREAMS;  in cdnsp_alloc_streams()
796 	pep->stream_info.td_count = 0;  in cdnsp_alloc_streams()
797 	pep->stream_info.first_prime_det = 0;  in cdnsp_alloc_streams()
931 	struct cdnsp_ep *pep;  in cdnsp_gadget_ep_enable()  local
940 	pep = to_cdnsp_ep(ep);  in cdnsp_gadget_ep_enable()
941 	pdev = pep->pdev;  in cdnsp_gadget_ep_enable()
942 	pep->ep_state &= ~EP_UNCONFIGURED;  in cdnsp_gadget_ep_enable()
944 	if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,  in cdnsp_gadget_ep_enable()
945 			  "%s is already enabled\n", pep->name))  in cdnsp_gadget_ep_enable()
957 	pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;  in cdnsp_gadget_ep_enable()
961 			pep->interval = desc->bInterval << 3;  in cdnsp_gadget_ep_enable()
963 			pep->interval = BIT(desc->bInterval - 1) << 3;  in cdnsp_gadget_ep_enable()
967 		if (pep->interval > BIT(12)) {  in cdnsp_gadget_ep_enable()
976 	ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);  in cdnsp_gadget_ep_enable()
984 	ret = cdnsp_update_eps_configuration(pdev, pep);  in cdnsp_gadget_ep_enable()
986 		cdnsp_free_endpoint_rings(pdev, pep);  in cdnsp_gadget_ep_enable()
990 	pep->ep_state |= EP_ENABLED;  in cdnsp_gadget_ep_enable()
991 	pep->ep_state &= ~EP_STOPPED;  in cdnsp_gadget_ep_enable()
994 	trace_cdnsp_ep_enable_end(pep, 0);  in cdnsp_gadget_ep_enable()
1005 	struct cdnsp_ep *pep;  in cdnsp_gadget_ep_disable()  local
1013 	pep = to_cdnsp_ep(ep);  in cdnsp_gadget_ep_disable()
1014 	pdev = pep->pdev;  in cdnsp_gadget_ep_disable()
1018 	if (!(pep->ep_state & EP_ENABLED)) {  in cdnsp_gadget_ep_disable()
1019 		dev_err(pdev->dev, "%s is already disabled\n", pep->name);  in cdnsp_gadget_ep_disable()
1024 	pep->ep_state |= EP_DIS_IN_RROGRESS;  in cdnsp_gadget_ep_disable()
1027 	if (!(pep->ep_state & EP_UNCONFIGURED)) {  in cdnsp_gadget_ep_disable()
1028 		cdnsp_cmd_stop_ep(pdev, pep);  in cdnsp_gadget_ep_disable()
1029 		cdnsp_cmd_flush_ep(pdev, pep);  in cdnsp_gadget_ep_disable()
1033 	while (!list_empty(&pep->pending_list)) {  in cdnsp_gadget_ep_disable()
1034 		preq = next_request(&pep->pending_list);  in cdnsp_gadget_ep_disable()
1035 		cdnsp_ep_dequeue(pep, preq);  in cdnsp_gadget_ep_disable()
1038 	cdnsp_invalidate_ep_events(pdev, pep);  in cdnsp_gadget_ep_disable()
1040 	pep->ep_state &= ~EP_DIS_IN_RROGRESS;  in cdnsp_gadget_ep_disable()
1041 	drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);  in cdnsp_gadget_ep_disable()
1046 	cdnsp_endpoint_zero(pdev, pep);  in cdnsp_gadget_ep_disable()
1048 	if (!(pep->ep_state & EP_UNCONFIGURED))  in cdnsp_gadget_ep_disable()
1049 		ret = cdnsp_update_eps_configuration(pdev, pep);  in cdnsp_gadget_ep_disable()
1051 	cdnsp_free_endpoint_rings(pdev, pep);  in cdnsp_gadget_ep_disable()
1053 	pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);  in cdnsp_gadget_ep_disable()
1054 	pep->ep_state |= EP_STOPPED;  in cdnsp_gadget_ep_disable()
1057 	trace_cdnsp_ep_disable_end(pep, 0);  in cdnsp_gadget_ep_disable()
1066 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);  in cdnsp_gadget_ep_alloc_request()  local
1073 	preq->epnum = pep->number;  in cdnsp_gadget_ep_alloc_request()
1074 	preq->pep = pep;  in cdnsp_gadget_ep_alloc_request()
1096 	struct cdnsp_ep *pep;  in cdnsp_gadget_ep_queue()  local
1103 	pep = to_cdnsp_ep(ep);  in cdnsp_gadget_ep_queue()
1104 	pdev = pep->pdev;  in cdnsp_gadget_ep_queue()
1106 	if (!(pep->ep_state & EP_ENABLED)) {  in cdnsp_gadget_ep_queue()
1108 			pep->name);  in cdnsp_gadget_ep_queue()
1114 	ret = cdnsp_ep_enqueue(pep, preq);  in cdnsp_gadget_ep_queue()
1123 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);  in cdnsp_gadget_ep_dequeue()  local
1124 	struct cdnsp_device *pdev = pep->pdev;  in cdnsp_gadget_ep_dequeue()
1131 	if (!pep->endpoint.desc) {  in cdnsp_gadget_ep_dequeue()
1134 			pep->name);  in cdnsp_gadget_ep_dequeue()
1139 	if (!(pep->ep_state & EP_ENABLED))  in cdnsp_gadget_ep_dequeue()
1143 	ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));  in cdnsp_gadget_ep_dequeue()
1151 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);  in cdnsp_gadget_ep_set_halt()  local
1152 	struct cdnsp_device *pdev = pep->pdev;  in cdnsp_gadget_ep_set_halt()
1159 	preq = next_request(&pep->pending_list);  in cdnsp_gadget_ep_set_halt()
1162 			trace_cdnsp_ep_busy_try_halt_again(pep, 0);  in cdnsp_gadget_ep_set_halt()
1168 	ret = cdnsp_halt_endpoint(pdev, pep, value);  in cdnsp_gadget_ep_set_halt()
1177 	struct cdnsp_ep *pep = to_cdnsp_ep(ep);  in cdnsp_gadget_ep_set_wedge()  local
1178 	struct cdnsp_device *pdev = pep->pdev;  in cdnsp_gadget_ep_set_wedge()
1183 	pep->ep_state |= EP_WEDGE;  in cdnsp_gadget_ep_set_wedge()
1184 	ret = cdnsp_halt_endpoint(pdev, pep, 1);  in cdnsp_gadget_ep_set_wedge()
1212 void cdnsp_gadget_giveback(struct cdnsp_ep *pep,  in cdnsp_gadget_giveback()  argument
1216 	struct cdnsp_device *pdev = pep->pdev;  in cdnsp_gadget_giveback()
1230 		usb_gadget_giveback_request(&pep->endpoint, &preq->request);  in cdnsp_gadget_giveback()
1574 				   struct cdnsp_ep *pep)  in cdnsp_get_ep_buffering()  argument
1581 	if (!pep->direction) {  in cdnsp_get_ep_buffering()
1582 		pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);  in cdnsp_get_ep_buffering()
1583 		pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);  in cdnsp_get_ep_buffering()
1584 		pep->buffering = (pep->buffering + 1) / 2;  in cdnsp_get_ep_buffering()
1585 		pep->buffering_period = (pep->buffering_period + 1) / 2;  in cdnsp_get_ep_buffering()
1594 	reg += pep->number * sizeof(u32) * 2;  in cdnsp_get_ep_buffering()
1596 	pep->buffering = (readl(reg) + 1) / 2;  in cdnsp_get_ep_buffering()
1597 	pep->buffering_period = pep->buffering;  in cdnsp_get_ep_buffering()
1603 	struct cdnsp_ep *pep;  in cdnsp_gadget_init_endpoints()  local
1623 		pep = &pdev->eps[i];  in cdnsp_gadget_init_endpoints()
1624 		pep->pdev = pdev;  in cdnsp_gadget_init_endpoints()
1625 		pep->number = epnum;  in cdnsp_gadget_init_endpoints()
1626 		pep->direction = direction; /* 0 for OUT, 1 for IN. */  in cdnsp_gadget_init_endpoints()
1633 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",  in cdnsp_gadget_init_endpoints()
1636 			pep->idx = 0;  in cdnsp_gadget_init_endpoints()
1637 			usb_ep_set_maxpacket_limit(&pep->endpoint, 512);  in cdnsp_gadget_init_endpoints()
1638 			pep->endpoint.maxburst = 1;  in cdnsp_gadget_init_endpoints()
1639 			pep->endpoint.ops = &cdnsp_gadget_ep0_ops;  in cdnsp_gadget_init_endpoints()
1640 			pep->endpoint.desc = &cdnsp_gadget_ep0_desc;  in cdnsp_gadget_init_endpoints()
1641 			pep->endpoint.comp_desc = NULL;  in cdnsp_gadget_init_endpoints()
1642 			pep->endpoint.caps.type_control = true;  in cdnsp_gadget_init_endpoints()
1643 			pep->endpoint.caps.dir_in = true;  in cdnsp_gadget_init_endpoints()
1644 			pep->endpoint.caps.dir_out = true;  in cdnsp_gadget_init_endpoints()
1646 			pdev->ep0_preq.epnum = pep->number;  in cdnsp_gadget_init_endpoints()
1647 			pdev->ep0_preq.pep = pep;  in cdnsp_gadget_init_endpoints()
1648 			pdev->gadget.ep0 = &pep->endpoint;  in cdnsp_gadget_init_endpoints()
1650 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",  in cdnsp_gadget_init_endpoints()
1651 				 epnum, (pep->direction) ? "in" : "out");  in cdnsp_gadget_init_endpoints()
1653 			pep->idx =  (epnum * 2 + (direction ? 1 : 0)) - 1;  in cdnsp_gadget_init_endpoints()
1654 			usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);  in cdnsp_gadget_init_endpoints()
1656 			pep->endpoint.max_streams = max_streams;  in cdnsp_gadget_init_endpoints()
1657 			pep->endpoint.ops = &cdnsp_gadget_ep_ops;  in cdnsp_gadget_init_endpoints()
1658 			list_add_tail(&pep->endpoint.ep_list,  in cdnsp_gadget_init_endpoints()
1661 			pep->endpoint.caps.type_iso = true;  in cdnsp_gadget_init_endpoints()
1662 			pep->endpoint.caps.type_bulk = true;  in cdnsp_gadget_init_endpoints()
1663 			pep->endpoint.caps.type_int = true;  in cdnsp_gadget_init_endpoints()
1665 			pep->endpoint.caps.dir_in = direction;  in cdnsp_gadget_init_endpoints()
1666 			pep->endpoint.caps.dir_out = !direction;  in cdnsp_gadget_init_endpoints()
1669 		pep->endpoint.name = pep->name;  in cdnsp_gadget_init_endpoints()
1670 		pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);  in cdnsp_gadget_init_endpoints()
1671 		pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);  in cdnsp_gadget_init_endpoints()
1672 		cdnsp_get_ep_buffering(pdev, pep);  in cdnsp_gadget_init_endpoints()
1677 			pep->name, 1024,  in cdnsp_gadget_init_endpoints()
1678 			(pep->endpoint.caps.type_control) ? "yes" : "no",  in cdnsp_gadget_init_endpoints()
1679 			(pep->endpoint.caps.type_int) ? "yes" : "no",  in cdnsp_gadget_init_endpoints()
1680 			(pep->endpoint.caps.type_bulk) ? "yes" : "no",  in cdnsp_gadget_init_endpoints()
1681 			(pep->endpoint.caps.type_iso) ? "yes" : "no",  in cdnsp_gadget_init_endpoints()
1682 			(pep->endpoint.caps.dir_in) ? "yes" : "no",  in cdnsp_gadget_init_endpoints()
1683 			(pep->endpoint.caps.dir_out) ? "yes" : "no");  in cdnsp_gadget_init_endpoints()
1685 		INIT_LIST_HEAD(&pep->pending_list);  in cdnsp_gadget_init_endpoints()
1693 	struct cdnsp_ep *pep;  in cdnsp_gadget_free_endpoints()  local
1697 		pep = &pdev->eps[i];  in cdnsp_gadget_free_endpoints()
1698 		if (pep->number != 0 && pep->out_ctx)  in cdnsp_gadget_free_endpoints()
1699 			list_del(&pep->endpoint.ep_list);  in cdnsp_gadget_free_endpoints()