1 // SPDX-License-Identifier: GPL-2.0
3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - https://www.ti.com
20 #include <linux/dma-mapping.h>
30 #define DWC3_ALIGN_FRAME(d, n)	(((d)->frame_number + ((d)->interval * (n))) \
31 					& ~((d)->interval - 1))
34  * dwc3_gadget_set_test_mode - enables usb2 test modes
39  * success or -EINVAL if wrong Test Selector is passed.
45 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_gadget_set_test_mode()
57 		return -EINVAL;  in dwc3_gadget_set_test_mode()
66  * dwc3_gadget_get_link_state - gets current state of usb link
70  * return the link state on success (>= 0) or -ETIMEDOUT.
76 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);  in dwc3_gadget_get_link_state()
82  * dwc3_gadget_set_link_state - sets usb link to a particular state
87  * return 0 on success or -ETIMEDOUT.
99 		while (--retries) {  in dwc3_gadget_set_link_state()
100 			reg = dwc3_readl(dwc->regs, DWC3_DSTS);  in dwc3_gadget_set_link_state()
108 			return -ETIMEDOUT;  in dwc3_gadget_set_link_state()
111 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_gadget_set_link_state()
115 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);  in dwc3_gadget_set_link_state()
119 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);  in dwc3_gadget_set_link_state()
130 	while (--retries) {  in dwc3_gadget_set_link_state()
131 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);  in dwc3_gadget_set_link_state()
139 	return -ETIMEDOUT;  in dwc3_gadget_set_link_state()
146 	if (dwc->ep0state != EP0_SETUP_PHASE) {  in dwc3_ep0_reset_state()
147 		dir = !!dwc->ep0_expect_in;  in dwc3_ep0_reset_state()
148 		if (dwc->ep0state == EP0_DATA_PHASE)  in dwc3_ep0_reset_state()
149 			dwc3_ep0_end_control_data(dwc, dwc->eps[dir]);  in dwc3_ep0_reset_state()
151 			dwc3_ep0_end_control_data(dwc, dwc->eps[!dir]);  in dwc3_ep0_reset_state()
153 		dwc->eps[0]->trb_enqueue = 0;  in dwc3_ep0_reset_state()
154 		dwc->eps[1]->trb_enqueue = 0;  in dwc3_ep0_reset_state()
161  * dwc3_ep_inc_trb - increment a trb index.
171 	if (*index == (DWC3_TRB_NUM - 1))  in dwc3_ep_inc_trb()
176  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
181 	dwc3_ep_inc_trb(&dep->trb_enqueue);  in dwc3_ep_inc_enq()
185  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
190 	dwc3_ep_inc_trb(&dep->trb_dequeue);  in dwc3_ep_inc_deq()
196 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_del_and_unmap_request()
198 	list_del(&req->list);  in dwc3_gadget_del_and_unmap_request()
199 	req->remaining = 0;  in dwc3_gadget_del_and_unmap_request()
200 	req->needs_extra_trb = false;  in dwc3_gadget_del_and_unmap_request()
201 	req->num_trbs = 0;  in dwc3_gadget_del_and_unmap_request()
203 	if (req->request.status == -EINPROGRESS)  in dwc3_gadget_del_and_unmap_request()
204 		req->request.status = status;  in dwc3_gadget_del_and_unmap_request()
206 	if (req->trb)  in dwc3_gadget_del_and_unmap_request()
207 		usb_gadget_unmap_request_by_dev(dwc->sysdev,  in dwc3_gadget_del_and_unmap_request()
208 				&req->request, req->direction);  in dwc3_gadget_del_and_unmap_request()
210 	req->trb = NULL;  in dwc3_gadget_del_and_unmap_request()
213 	if (dep->number > 1)  in dwc3_gadget_del_and_unmap_request()
214 		pm_runtime_put(dwc->dev);  in dwc3_gadget_del_and_unmap_request()
218  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
224  * function will unmap @req and call its ->complete() callback to notify upper
230 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_giveback()
233 	req->status = DWC3_REQUEST_STATUS_COMPLETED;  in dwc3_gadget_giveback()
235 	spin_unlock(&dwc->lock);  in dwc3_gadget_giveback()
236 	usb_gadget_giveback_request(&dep->endpoint, &req->request);  in dwc3_gadget_giveback()
237 	spin_lock(&dwc->lock);  in dwc3_gadget_giveback()
241  * dwc3_send_gadget_generic_command - issue a generic command for the controller
257 	dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);  in dwc3_send_gadget_generic_command()
258 	dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);  in dwc3_send_gadget_generic_command()
261 		reg = dwc3_readl(dwc->regs, DWC3_DGCMD);  in dwc3_send_gadget_generic_command()
265 				ret = -EINVAL;  in dwc3_send_gadget_generic_command()
268 	} while (--timeout);  in dwc3_send_gadget_generic_command()
271 		ret = -ETIMEDOUT;  in dwc3_send_gadget_generic_command()
272 		status = -ETIMEDOUT;  in dwc3_send_gadget_generic_command()
283  * dwc3_send_gadget_ep_cmd - issue an endpoint command
311 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;  in dwc3_send_gadget_ep_cmd()
312 	struct dwc3		*dwc = dep->dwc;  in dwc3_send_gadget_ep_cmd()
318 	int			ret = -EINVAL;  in dwc3_send_gadget_ep_cmd()
330 	if (dwc->gadget->speed <= USB_SPEED_HIGH ||  in dwc3_send_gadget_ep_cmd()
332 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));  in dwc3_send_gadget_ep_cmd()
344 			dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);  in dwc3_send_gadget_ep_cmd()
354 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);  in dwc3_send_gadget_ep_cmd()
355 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);  in dwc3_send_gadget_ep_cmd()
356 		dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);  in dwc3_send_gadget_ep_cmd()
380 	dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);  in dwc3_send_gadget_ep_cmd()
390 		reg = dwc3_readl(dep->regs, DWC3_DEPCMD);  in dwc3_send_gadget_ep_cmd()
399 				dev_WARN(dwc->dev, "No resource for %s\n",  in dwc3_send_gadget_ep_cmd()
400 					 dep->name);  in dwc3_send_gadget_ep_cmd()
401 				ret = -EINVAL;  in dwc3_send_gadget_ep_cmd()
411 				 * Instead of always returning -EINVAL, let's  in dwc3_send_gadget_ep_cmd()
413 				 * the case by returning -EAGAIN.  in dwc3_send_gadget_ep_cmd()
415 				ret = -EAGAIN;  in dwc3_send_gadget_ep_cmd()
418 				dev_WARN(dwc->dev, "UNKNOWN cmd status\n");  in dwc3_send_gadget_ep_cmd()
423 	} while (--timeout);  in dwc3_send_gadget_ep_cmd()
426 		ret = -ETIMEDOUT;  in dwc3_send_gadget_ep_cmd()
427 		cmd_status = -ETIMEDOUT;  in dwc3_send_gadget_ep_cmd()
435 			dep->flags |= DWC3_EP_TRANSFER_STARTED;  in dwc3_send_gadget_ep_cmd()
437 		if (ret != -ETIMEDOUT)  in dwc3_send_gadget_ep_cmd()
446 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));  in dwc3_send_gadget_ep_cmd()
448 		dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);  in dwc3_send_gadget_ep_cmd()
456 	struct dwc3 *dwc = dep->dwc;  in dwc3_send_clear_stall_ep_cmd()
464 	 * some (non-compliant) hosts may not send ACK TPs for pending  in dwc3_send_clear_stall_ep_cmd()
468 	if (dep->direction &&  in dwc3_send_clear_stall_ep_cmd()
470 	    (dwc->gadget->speed >= USB_SPEED_SUPER))  in dwc3_send_clear_stall_ep_cmd()
481 	u32		offset = (char *) trb - (char *) dep->trb_pool;  in dwc3_trb_dma_offset()
483 	return dep->trb_pool_dma + offset;  in dwc3_trb_dma_offset()
488 	struct dwc3		*dwc = dep->dwc;  in dwc3_alloc_trb_pool()
490 	if (dep->trb_pool)  in dwc3_alloc_trb_pool()
493 	dep->trb_pool = dma_alloc_coherent(dwc->sysdev,  in dwc3_alloc_trb_pool()
495 			&dep->trb_pool_dma, GFP_KERNEL);  in dwc3_alloc_trb_pool()
496 	if (!dep->trb_pool) {  in dwc3_alloc_trb_pool()
497 		dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",  in dwc3_alloc_trb_pool()
498 				dep->name);  in dwc3_alloc_trb_pool()
499 		return -ENOMEM;  in dwc3_alloc_trb_pool()
507 	struct dwc3		*dwc = dep->dwc;  in dwc3_free_trb_pool()
509 	dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,  in dwc3_free_trb_pool()
510 			dep->trb_pool, dep->trb_pool_dma);  in dwc3_free_trb_pool()
512 	dep->trb_pool = NULL;  in dwc3_free_trb_pool()
513 	dep->trb_pool_dma = 0;  in dwc3_free_trb_pool()
521 	if (dep->flags & DWC3_EP_RESOURCE_ALLOCATED)  in dwc3_gadget_set_xfer_resource()
533 	dep->flags |= DWC3_EP_RESOURCE_ALLOCATED;  in dwc3_gadget_set_xfer_resource()
538  * dwc3_gadget_start_config - reset endpoint resources
543  * part of the power-on/soft-reset initialization.
545  * Set resource_index=2 to reset only non-control endpoints' resources. Do this
557 		return -EINVAL;  in dwc3_gadget_start_config()
563 	ret = dwc3_send_gadget_ep_cmd(dwc->eps[0], cmd, &params);  in dwc3_gadget_start_config()
568 	for (i = resource_index; i < dwc->num_eps; i++) {  in dwc3_gadget_start_config()
569 		dep = dwc->eps[i];  in dwc3_gadget_start_config()
573 		dep->flags &= ~DWC3_EP_RESOURCE_ALLOCATED;  in dwc3_gadget_start_config()
584 	struct dwc3 *dwc = dep->dwc;  in dwc3_gadget_set_ep_config()
586 	comp_desc = dep->endpoint.comp_desc;  in dwc3_gadget_set_ep_config()
587 	desc = dep->endpoint.desc;  in dwc3_gadget_set_ep_config()
595 	if (dwc->gadget->speed >= USB_SPEED_SUPER) {  in dwc3_gadget_set_ep_config()
596 		u32 burst = dep->endpoint.maxburst;  in dwc3_gadget_set_ep_config()
598 		params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);  in dwc3_gadget_set_ep_config()
603 		params.param2 |= dep->saved_state;  in dwc3_gadget_set_ep_config()
608 	if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))  in dwc3_gadget_set_ep_config()
615 		dep->stream_capable = true;  in dwc3_gadget_set_ep_config()
627 	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);  in dwc3_gadget_set_ep_config()
633 	if (dep->direction)  in dwc3_gadget_set_ep_config()
634 		params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);  in dwc3_gadget_set_ep_config()
636 	if (desc->bInterval) {  in dwc3_gadget_set_ep_config()
647 		bInterval_m1 = min_t(u8, desc->bInterval - 1, 13);  in dwc3_gadget_set_ep_config()
650 		    dwc->gadget->speed == USB_SPEED_FULL)  in dwc3_gadget_set_ep_config()
651 			dep->interval = desc->bInterval;  in dwc3_gadget_set_ep_config()
653 			dep->interval = 1 << (desc->bInterval - 1);  in dwc3_gadget_set_ep_config()
662  * dwc3_gadget_calc_tx_fifo_size - calculates the txfifo size value
697  * dwc3_gadget_calc_ram_depth - calculates the ram depth for txfifo
707 	is_single_port_ram = DWC3_SPRAM_TYPE(dwc->hwparams.hwparams1);  in dwc3_gadget_calc_ram_depth()
713 	ram_depth = is_single_port_ram ? DWC3_RAM0_DEPTH(dwc->hwparams.hwparams6) :  in dwc3_gadget_calc_ram_depth()
714 			DWC3_RAM1_DEPTH(dwc->hwparams.hwparams7);  in dwc3_gadget_calc_ram_depth()
719 	 * at a non-zero address.  in dwc3_gadget_calc_ram_depth()
724 		/* Check if TXFIFOs start at non-zero addr */  in dwc3_gadget_calc_ram_depth()
725 		reg = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));  in dwc3_gadget_calc_ram_depth()
728 		ram_depth -= (fifo_0_start >> 16);  in dwc3_gadget_calc_ram_depth()
735  * dwc3_gadget_clear_tx_fifos - Clears txfifo allocation
748 	if (!dwc->do_fifo_resize)  in dwc3_gadget_clear_tx_fifos()
752 	dep = dwc->eps[1];  in dwc3_gadget_clear_tx_fifos()
753 	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));  in dwc3_gadget_clear_tx_fifos()
759 	dwc->last_fifo_depth = fifo_depth;  in dwc3_gadget_clear_tx_fifos()
761 	for (num = 3; num < min_t(int, dwc->num_eps, DWC3_ENDPOINTS_NUM); num += 2) {  in dwc3_gadget_clear_tx_fifos()
762 		dep = dwc->eps[num];  in dwc3_gadget_clear_tx_fifos()
768 			dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1)) &  in dwc3_gadget_clear_tx_fifos()
771 		dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num >> 1), size);  in dwc3_gadget_clear_tx_fifos()
772 		dep->flags &= ~DWC3_EP_TXFIFO_RESIZED;  in dwc3_gadget_clear_tx_fifos()
774 	dwc->num_ep_resized = 0;  in dwc3_gadget_clear_tx_fifos()
778  * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
786  * on the configured size for RAM1 - which contains TxFifo -,
800 	struct dwc3 *dwc = dep->dwc;  in dwc3_gadget_resize_tx_fifos()
811 	if (!dwc->do_fifo_resize)  in dwc3_gadget_resize_tx_fifos()
815 	if (!usb_endpoint_dir_in(dep->endpoint.desc) || dep->number <= 1)  in dwc3_gadget_resize_tx_fifos()
819 	if (dep->flags & DWC3_EP_TXFIFO_RESIZED)  in dwc3_gadget_resize_tx_fifos()
824 	if ((dep->endpoint.maxburst > 1 &&  in dwc3_gadget_resize_tx_fifos()
825 	     usb_endpoint_xfer_bulk(dep->endpoint.desc)) ||  in dwc3_gadget_resize_tx_fifos()
826 	    usb_endpoint_xfer_isoc(dep->endpoint.desc))  in dwc3_gadget_resize_tx_fifos()
829 	if (dep->endpoint.maxburst > 6 &&  in dwc3_gadget_resize_tx_fifos()
830 	    (usb_endpoint_xfer_bulk(dep->endpoint.desc) ||  in dwc3_gadget_resize_tx_fifos()
831 	     usb_endpoint_xfer_isoc(dep->endpoint.desc)) && DWC3_IP_IS(DWC31))  in dwc3_gadget_resize_tx_fifos()
832 		num_fifos = dwc->tx_fifo_resize_max_num;  in dwc3_gadget_resize_tx_fifos()
838 	num_in_ep = dwc->max_cfg_eps;  in dwc3_gadget_resize_tx_fifos()
839 	num_in_ep -= dwc->num_ep_resized;  in dwc3_gadget_resize_tx_fifos()
843 	remaining = ram_depth - min_depth - dwc->last_fifo_depth;  in dwc3_gadget_resize_tx_fifos()
850 	fifo_size = (num_fifos - 1) * fifo;  in dwc3_gadget_resize_tx_fifos()
858 	/* Check if TXFIFOs start at non-zero addr */  in dwc3_gadget_resize_tx_fifos()
859 	tmp = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(0));  in dwc3_gadget_resize_tx_fifos()
862 	fifo_size |= (fifo_0_start + (dwc->last_fifo_depth << 16));  in dwc3_gadget_resize_tx_fifos()
864 		dwc->last_fifo_depth += DWC3_GTXFIFOSIZ_TXFDEP(fifo_size);  in dwc3_gadget_resize_tx_fifos()
866 		dwc->last_fifo_depth += DWC31_GTXFIFOSIZ_TXFDEP(fifo_size);  in dwc3_gadget_resize_tx_fifos()
869 	if (dwc->last_fifo_depth >= ram_depth) {  in dwc3_gadget_resize_tx_fifos()
870 		dev_err(dwc->dev, "Fifosize(%d) > RAM size(%d) %s depth:%d\n",  in dwc3_gadget_resize_tx_fifos()
871 			dwc->last_fifo_depth, ram_depth,  in dwc3_gadget_resize_tx_fifos()
872 			dep->endpoint.name, fifo_size);  in dwc3_gadget_resize_tx_fifos()
878 		dwc->last_fifo_depth -= fifo_size;  in dwc3_gadget_resize_tx_fifos()
879 		return -ENOMEM;  in dwc3_gadget_resize_tx_fifos()
882 	dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1), fifo_size);  in dwc3_gadget_resize_tx_fifos()
883 	dep->flags |= DWC3_EP_TXFIFO_RESIZED;  in dwc3_gadget_resize_tx_fifos()
884 	dwc->num_ep_resized++;  in dwc3_gadget_resize_tx_fifos()
890  * __dwc3_gadget_ep_enable - initializes a hw endpoint
899 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;  in __dwc3_gadget_ep_enable()
900 	struct dwc3		*dwc = dep->dwc;  in __dwc3_gadget_ep_enable()
905 	if (!(dep->flags & DWC3_EP_ENABLED)) {  in __dwc3_gadget_ep_enable()
915 	if (!(dep->flags & DWC3_EP_RESOURCE_ALLOCATED)) {  in __dwc3_gadget_ep_enable()
921 	if (!(dep->flags & DWC3_EP_ENABLED)) {  in __dwc3_gadget_ep_enable()
925 		dep->type = usb_endpoint_type(desc);  in __dwc3_gadget_ep_enable()
926 		dep->flags |= DWC3_EP_ENABLED;  in __dwc3_gadget_ep_enable()
928 		reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);  in __dwc3_gadget_ep_enable()
929 		reg |= DWC3_DALEPENA_EP(dep->number);  in __dwc3_gadget_ep_enable()
930 		dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);  in __dwc3_gadget_ep_enable()
932 		dep->trb_dequeue = 0;  in __dwc3_gadget_ep_enable()
933 		dep->trb_enqueue = 0;  in __dwc3_gadget_ep_enable()
939 		memset(dep->trb_pool, 0,  in __dwc3_gadget_ep_enable()
943 		trb_st_hw = &dep->trb_pool[0];  in __dwc3_gadget_ep_enable()
945 		trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];  in __dwc3_gadget_ep_enable()
946 		trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));  in __dwc3_gadget_ep_enable()
947 		trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));  in __dwc3_gadget_ep_enable()
948 		trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;  in __dwc3_gadget_ep_enable()
949 		trb_link->ctrl |= DWC3_TRB_CTRL_HWO;  in __dwc3_gadget_ep_enable()
953 	 * Issue StartTransfer here with no-op TRB so we can always rely on No  in __dwc3_gadget_ep_enable()
964 		trb = &dep->trb_pool[0];  in __dwc3_gadget_ep_enable()
976 		if (dep->stream_capable) {  in __dwc3_gadget_ep_enable()
983 			 * no-op TRB as normal, but end it immediately. As a  in __dwc3_gadget_ep_enable()
1001 			if (!dep->direction ||  in __dwc3_gadget_ep_enable()
1002 			    !(dwc->hwparams.hwparams9 &  in __dwc3_gadget_ep_enable()
1004 				dep->flags |= DWC3_EP_FORCE_RESTART_STREAM;  in __dwc3_gadget_ep_enable()
1021 	if (dep->flags & DWC3_EP_DELAY_STOP)  in dwc3_remove_requests()
1024 	/* - giveback all requests to gadget driver */  in dwc3_remove_requests()
1025 	while (!list_empty(&dep->started_list)) {  in dwc3_remove_requests()
1026 		req = next_request(&dep->started_list);  in dwc3_remove_requests()
1031 	while (!list_empty(&dep->pending_list)) {  in dwc3_remove_requests()
1032 		req = next_request(&dep->pending_list);  in dwc3_remove_requests()
1037 	while (!list_empty(&dep->cancelled_list)) {  in dwc3_remove_requests()
1038 		req = next_request(&dep->cancelled_list);  in dwc3_remove_requests()
1045  * __dwc3_gadget_ep_disable - disables a hw endpoint
1056 	struct dwc3		*dwc = dep->dwc;  in __dwc3_gadget_ep_disable()
1063 	if (dep->flags & DWC3_EP_STALL)  in __dwc3_gadget_ep_disable()
1066 	reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);  in __dwc3_gadget_ep_disable()
1067 	reg &= ~DWC3_DALEPENA_EP(dep->number);  in __dwc3_gadget_ep_disable()
1068 	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);  in __dwc3_gadget_ep_disable()
1070 	dwc3_remove_requests(dwc, dep, -ESHUTDOWN);  in __dwc3_gadget_ep_disable()
1072 	dep->stream_capable = false;  in __dwc3_gadget_ep_disable()
1073 	dep->type = 0;  in __dwc3_gadget_ep_disable()
1080 	if (dep->flags & DWC3_EP_DELAY_STOP)  in __dwc3_gadget_ep_disable()
1082 	dep->flags &= mask;  in __dwc3_gadget_ep_disable()
1084 	/* Clear out the ep descriptors for non-ep0 */  in __dwc3_gadget_ep_disable()
1085 	if (dep->number > 1) {  in __dwc3_gadget_ep_disable()
1086 		dep->endpoint.comp_desc = NULL;  in __dwc3_gadget_ep_disable()
1087 		dep->endpoint.desc = NULL;  in __dwc3_gadget_ep_disable()
1093 /* -------------------------------------------------------------------------- */
1098 	return -EINVAL;  in dwc3_gadget_ep0_enable()
1103 	return -EINVAL;  in dwc3_gadget_ep0_disable()
1106 /* -------------------------------------------------------------------------- */
1116 	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {  in dwc3_gadget_ep_enable()
1118 		return -EINVAL;  in dwc3_gadget_ep_enable()
1121 	if (!desc->wMaxPacketSize) {  in dwc3_gadget_ep_enable()
1123 		return -EINVAL;  in dwc3_gadget_ep_enable()
1127 	dwc = dep->dwc;  in dwc3_gadget_ep_enable()
1129 	if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,  in dwc3_gadget_ep_enable()
1131 					dep->name))  in dwc3_gadget_ep_enable()
1134 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_ep_enable()
1136 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_ep_enable()
1150 		return -EINVAL;  in dwc3_gadget_ep_disable()
1154 	dwc = dep->dwc;  in dwc3_gadget_ep_disable()
1156 	if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),  in dwc3_gadget_ep_disable()
1158 					dep->name))  in dwc3_gadget_ep_disable()
1161 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_ep_disable()
1163 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_ep_disable()
1178 	req->direction	= dep->direction;  in dwc3_gadget_ep_alloc_request()
1179 	req->epnum	= dep->number;  in dwc3_gadget_ep_alloc_request()
1180 	req->dep	= dep;  in dwc3_gadget_ep_alloc_request()
1181 	req->status	= DWC3_REQUEST_STATUS_UNKNOWN;  in dwc3_gadget_ep_alloc_request()
1185 	return &req->request;  in dwc3_gadget_ep_alloc_request()
1198  * dwc3_ep_prev_trb - returns the previous TRB in the ring
1211 		tmp = DWC3_TRB_NUM - 1;  in dwc3_ep_prev_trb()
1213 	return &dep->trb_pool[tmp - 1];  in dwc3_ep_prev_trb()
1222 	 * or empty. It's considered full when there are DWC3_TRB_NUM-1 of TRBs  in dwc3_calc_trbs_left()
1225 	if (dep->trb_enqueue == dep->trb_dequeue) {  in dwc3_calc_trbs_left()
1232 		req = next_request(&dep->started_list);  in dwc3_calc_trbs_left()
1233 		if (req && req->num_trbs)  in dwc3_calc_trbs_left()
1236 		return DWC3_TRB_NUM - 1;  in dwc3_calc_trbs_left()
1239 	trbs_left = dep->trb_dequeue - dep->trb_enqueue;  in dwc3_calc_trbs_left()
1240 	trbs_left &= (DWC3_TRB_NUM - 1);  in dwc3_calc_trbs_left()
1242 	if (dep->trb_dequeue < dep->trb_enqueue)  in dwc3_calc_trbs_left()
1243 		trbs_left--;  in dwc3_calc_trbs_left()
1249  * dwc3_prepare_one_trb - setup one TRB from one request
1265 	unsigned int		stream_id = req->request.stream_id;  in dwc3_prepare_one_trb()
1266 	unsigned int		short_not_ok = req->request.short_not_ok;  in dwc3_prepare_one_trb()
1267 	unsigned int		no_interrupt = req->request.no_interrupt;  in dwc3_prepare_one_trb()
1268 	unsigned int		is_last = req->request.is_last;  in dwc3_prepare_one_trb()
1269 	struct dwc3		*dwc = dep->dwc;  in dwc3_prepare_one_trb()
1270 	struct usb_gadget	*gadget = dwc->gadget;  in dwc3_prepare_one_trb()
1271 	enum usb_device_speed	speed = gadget->speed;  in dwc3_prepare_one_trb()
1274 		dma = dep->dwc->bounce_addr;  in dwc3_prepare_one_trb()
1275 	else if (req->request.num_sgs > 0)  in dwc3_prepare_one_trb()
1276 		dma = sg_dma_address(req->start_sg);  in dwc3_prepare_one_trb()
1278 		dma = req->request.dma;  in dwc3_prepare_one_trb()
1280 	trb = &dep->trb_pool[dep->trb_enqueue];  in dwc3_prepare_one_trb()
1282 	if (!req->trb) {  in dwc3_prepare_one_trb()
1284 		req->trb = trb;  in dwc3_prepare_one_trb()
1285 		req->trb_dma = dwc3_trb_dma_offset(dep, trb);  in dwc3_prepare_one_trb()
1288 	req->num_trbs++;  in dwc3_prepare_one_trb()
1290 	trb->size = DWC3_TRB_SIZE_LENGTH(trb_length);  in dwc3_prepare_one_trb()
1291 	trb->bpl = lower_32_bits(dma);  in dwc3_prepare_one_trb()
1292 	trb->bph = upper_32_bits(dma);  in dwc3_prepare_one_trb()
1294 	switch (usb_endpoint_type(dep->endpoint.desc)) {  in dwc3_prepare_one_trb()
1296 		trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;  in dwc3_prepare_one_trb()
1301 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;  in dwc3_prepare_one_trb()
1317 			 *	- DATA0  in dwc3_prepare_one_trb()
1320 			 *	- DATA1, DATA0  in dwc3_prepare_one_trb()
1323 			 *	- DATA2, DATA1, DATA0  in dwc3_prepare_one_trb()
1326 				struct usb_ep *ep = &dep->endpoint;  in dwc3_prepare_one_trb()
1328 				unsigned int maxp = usb_endpoint_maxp(ep->desc);  in dwc3_prepare_one_trb()
1330 				if (req->request.length <= (2 * maxp))  in dwc3_prepare_one_trb()
1331 					mult--;  in dwc3_prepare_one_trb()
1333 				if (req->request.length <= maxp)  in dwc3_prepare_one_trb()
1334 					mult--;  in dwc3_prepare_one_trb()
1336 				trb->size |= DWC3_TRB_SIZE_PCM1(mult);  in dwc3_prepare_one_trb()
1339 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;  in dwc3_prepare_one_trb()
1343 			trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;  in dwc3_prepare_one_trb()
1348 		trb->ctrl = DWC3_TRBCTL_NORMAL;  in dwc3_prepare_one_trb()
1355 		dev_WARN(dwc->dev, "Unknown endpoint type %d\n",  in dwc3_prepare_one_trb()
1356 				usb_endpoint_type(dep->endpoint.desc));  in dwc3_prepare_one_trb()
1363 	if (usb_endpoint_dir_out(dep->endpoint.desc)) {  in dwc3_prepare_one_trb()
1364 		if (!dep->stream_capable)  in dwc3_prepare_one_trb()
1365 			trb->ctrl |= DWC3_TRB_CTRL_CSP;  in dwc3_prepare_one_trb()
1368 			trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;  in dwc3_prepare_one_trb()
1372 	if (dep->stream_capable && DWC3_MST_CAPABLE(&dwc->hwparams))  in dwc3_prepare_one_trb()
1373 		trb->ctrl |= DWC3_TRB_CTRL_CSP;  in dwc3_prepare_one_trb()
1376 		trb->ctrl |= DWC3_TRB_CTRL_IOC;  in dwc3_prepare_one_trb()
1379 		trb->ctrl |= DWC3_TRB_CTRL_CHN;  in dwc3_prepare_one_trb()
1380 	else if (dep->stream_capable && is_last &&  in dwc3_prepare_one_trb()
1381 		 !DWC3_MST_CAPABLE(&dwc->hwparams))  in dwc3_prepare_one_trb()
1382 		trb->ctrl |= DWC3_TRB_CTRL_LST;  in dwc3_prepare_one_trb()
1384 	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)  in dwc3_prepare_one_trb()
1385 		trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);  in dwc3_prepare_one_trb()
1395 	 * However there is a possibility of CPU re-ordering here which can cause  in dwc3_prepare_one_trb()
1397 	 * Add a write memory barrier to prevent CPU re-ordering.  in dwc3_prepare_one_trb()
1400 	trb->ctrl |= DWC3_TRB_CTRL_HWO;  in dwc3_prepare_one_trb()
1409 	unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);  in dwc3_needs_extra_trb()
1410 	unsigned int rem = req->request.length % maxp;  in dwc3_needs_extra_trb()
1412 	if ((req->request.length && req->request.zero && !rem &&  in dwc3_needs_extra_trb()
1413 			!usb_endpoint_xfer_isoc(dep->endpoint.desc)) ||  in dwc3_needs_extra_trb()
1414 			(!req->direction && rem))  in dwc3_needs_extra_trb()
1421  * dwc3_prepare_last_sg - prepare TRBs for the last SG entry
1433 	unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);  in dwc3_prepare_last_sg()
1434 	unsigned int rem = req->request.length % maxp;  in dwc3_prepare_last_sg()
1443 	req->needs_extra_trb = num_trbs > 1;  in dwc3_prepare_last_sg()
1446 	if (req->direction || req->request.length)  in dwc3_prepare_last_sg()
1448 				req->needs_extra_trb, node, false, false);  in dwc3_prepare_last_sg()
1451 	if ((!req->direction && !req->request.length) || req->needs_extra_trb)  in dwc3_prepare_last_sg()
1453 				req->direction ? 0 : maxp - rem,  in dwc3_prepare_last_sg()
1462 	struct scatterlist *sg = req->start_sg;  in dwc3_prepare_trbs_sg()
1465 	unsigned int length = req->request.length;  in dwc3_prepare_trbs_sg()
1466 	unsigned int remaining = req->num_pending_sgs;  in dwc3_prepare_trbs_sg()
1467 	unsigned int num_queued_sgs = req->request.num_mapped_sgs - remaining;  in dwc3_prepare_trbs_sg()
1468 	unsigned int num_trbs = req->num_trbs;  in dwc3_prepare_trbs_sg()
1475 	for_each_sg(req->request.sg, s, num_queued_sgs, i)  in dwc3_prepare_trbs_sg()
1476 		length -= sg_dma_len(s);  in dwc3_prepare_trbs_sg()
1486 		length -= trb_length;  in dwc3_prepare_trbs_sg()
1495 		if ((i == remaining - 1) || !length)  in dwc3_prepare_trbs_sg()
1517 				list_for_each_entry(r, &dep->started_list, list) {  in dwc3_prepare_trbs_sg()
1518 					if (r != req && !r->request.no_interrupt)  in dwc3_prepare_trbs_sg()
1538 			req->start_sg = sg_next(s);  in dwc3_prepare_trbs_sg()
1540 		req->num_queued_sgs++;  in dwc3_prepare_trbs_sg()
1541 		req->num_pending_sgs--;  in dwc3_prepare_trbs_sg()
1549 			req->num_pending_sgs = 0;  in dwc3_prepare_trbs_sg()
1557 	return req->num_trbs - num_trbs;  in dwc3_prepare_trbs_sg()
1563 	return dwc3_prepare_last_sg(dep, req, req->request.length, 0);  in dwc3_prepare_trbs_linear()
1567  * dwc3_prepare_trbs - setup TRBs from requests
1593 	list_for_each_entry(req, &dep->started_list, list) {  in dwc3_prepare_trbs()
1594 		if (req->num_pending_sgs > 0) {  in dwc3_prepare_trbs()
1596 			if (!ret || req->num_pending_sgs)  in dwc3_prepare_trbs()
1608 		if (dep->stream_capable && req->request.is_last &&  in dwc3_prepare_trbs()
1609 		    !DWC3_MST_CAPABLE(&dep->dwc->hwparams))  in dwc3_prepare_trbs()
1613 	list_for_each_entry_safe(req, n, &dep->pending_list, list) {  in dwc3_prepare_trbs()
1614 		struct dwc3	*dwc = dep->dwc;  in dwc3_prepare_trbs()
1616 		ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,  in dwc3_prepare_trbs()
1617 						    dep->direction);  in dwc3_prepare_trbs()
1621 		req->sg			= req->request.sg;  in dwc3_prepare_trbs()
1622 		req->start_sg		= req->sg;  in dwc3_prepare_trbs()
1623 		req->num_queued_sgs	= 0;  in dwc3_prepare_trbs()
1624 		req->num_pending_sgs	= req->request.num_mapped_sgs;  in dwc3_prepare_trbs()
1626 		if (req->num_pending_sgs > 0) {  in dwc3_prepare_trbs()
1628 			if (req->num_pending_sgs)  in dwc3_prepare_trbs()
1642 		if (dep->stream_capable && req->request.is_last &&  in dwc3_prepare_trbs()
1643 		    !DWC3_MST_CAPABLE(&dwc->hwparams))  in dwc3_prepare_trbs()
1669 	starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);  in __dwc3_gadget_kick_transfer()
1678 	req = next_request(&dep->started_list);  in __dwc3_gadget_kick_transfer()
1680 		dep->flags |= DWC3_EP_PENDING_REQUEST;  in __dwc3_gadget_kick_transfer()
1687 		params.param0 = upper_32_bits(req->trb_dma);  in __dwc3_gadget_kick_transfer()
1688 		params.param1 = lower_32_bits(req->trb_dma);  in __dwc3_gadget_kick_transfer()
1691 		if (dep->stream_capable)  in __dwc3_gadget_kick_transfer()
1692 			cmd |= DWC3_DEPCMD_PARAM(req->request.stream_id);  in __dwc3_gadget_kick_transfer()
1694 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc))  in __dwc3_gadget_kick_transfer()
1695 			cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);  in __dwc3_gadget_kick_transfer()
1698 			DWC3_DEPCMD_PARAM(dep->resource_index);  in __dwc3_gadget_kick_transfer()
1705 		if (ret == -EAGAIN)  in __dwc3_gadget_kick_transfer()
1710 		list_for_each_entry_safe(req, tmp, &dep->started_list, list)  in __dwc3_gadget_kick_transfer()
1714 		if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))  in __dwc3_gadget_kick_transfer()
1720 	if (dep->stream_capable && req->request.is_last &&  in __dwc3_gadget_kick_transfer()
1721 	    !DWC3_MST_CAPABLE(&dep->dwc->hwparams))  in __dwc3_gadget_kick_transfer()
1722 		dep->flags |= DWC3_EP_WAIT_TRANSFER_COMPLETE;  in __dwc3_gadget_kick_transfer()
1731 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);  in __dwc3_gadget_get_frame()
1736  * __dwc3_stop_active_transfer - stop the current active transfer
1755 	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);  in __dwc3_stop_active_transfer()
1764 	if (ret == -ETIMEDOUT && dep->dwc->ep0state != EP0_SETUP_PHASE) {  in __dwc3_stop_active_transfer()
1765 		dep->flags |= DWC3_EP_DELAY_STOP;  in __dwc3_stop_active_transfer()
1769 	dep->resource_index = 0;  in __dwc3_stop_active_transfer()
1772 		dep->flags &= ~DWC3_EP_TRANSFER_STARTED;  in __dwc3_stop_active_transfer()
1774 		dep->flags |= DWC3_EP_END_TRANSFER_PENDING;  in __dwc3_stop_active_transfer()
1776 	dep->flags &= ~DWC3_EP_DELAY_STOP;  in __dwc3_stop_active_transfer()
1781  * dwc3_gadget_start_isoc_quirk - workaround invalid frame number
1784  * This function tests for the correct combination of BIT[15:14] from the 16-bit
1788  * In DWC_usb31 version 1.70a-ea06 and prior, for highspeed and fullspeed
1789  * isochronous IN, BIT[15:14] of the 16-bit microframe number reported by the
1793  * internal 16-bit microframe, the START TRANSFER command will pass and the
1796  * other conditions, the START TRANSFER command will fail with bus-expiry.
1801  * (or 2 seconds). 4 seconds into the future will result in a bus-expiry status.
1804  * command status will result in a 2-second delay start. The smaller BIT[15:14]
1828 	while (dep->combo_num < 2) {  in dwc3_gadget_start_isoc_quirk()
1835 		 * 4 uframes in the future with BIT[15:14] as dep->combo_num  in dwc3_gadget_start_isoc_quirk()
1837 		test_frame_number = dep->frame_number & DWC3_FRNUMBER_MASK;  in dwc3_gadget_start_isoc_quirk()
1838 		test_frame_number |= dep->combo_num << 14;  in dwc3_gadget_start_isoc_quirk()
1839 		test_frame_number += max_t(u32, 4, dep->interval);  in dwc3_gadget_start_isoc_quirk()
1841 		params.param0 = upper_32_bits(dep->dwc->bounce_addr);  in dwc3_gadget_start_isoc_quirk()
1842 		params.param1 = lower_32_bits(dep->dwc->bounce_addr);  in dwc3_gadget_start_isoc_quirk()
1848 		/* Redo if some other failure beside bus-expiry is received */  in dwc3_gadget_start_isoc_quirk()
1849 		if (cmd_status && cmd_status != -EAGAIN) {  in dwc3_gadget_start_isoc_quirk()
1850 			dep->start_cmd_status = 0;  in dwc3_gadget_start_isoc_quirk()
1851 			dep->combo_num = 0;  in dwc3_gadget_start_isoc_quirk()
1856 		if (dep->combo_num == 0)  in dwc3_gadget_start_isoc_quirk()
1857 			dep->start_cmd_status = cmd_status;  in dwc3_gadget_start_isoc_quirk()
1859 		dep->combo_num++;  in dwc3_gadget_start_isoc_quirk()
1872 	test0 = (dep->start_cmd_status == 0);  in dwc3_gadget_start_isoc_quirk()
1876 		dep->combo_num = 1;  in dwc3_gadget_start_isoc_quirk()
1878 		dep->combo_num = 2;  in dwc3_gadget_start_isoc_quirk()
1880 		dep->combo_num = 3;  in dwc3_gadget_start_isoc_quirk()
1882 		dep->combo_num = 0;  in dwc3_gadget_start_isoc_quirk()
1884 	dep->frame_number &= DWC3_FRNUMBER_MASK;  in dwc3_gadget_start_isoc_quirk()
1885 	dep->frame_number |= dep->combo_num << 14;  in dwc3_gadget_start_isoc_quirk()
1886 	dep->frame_number += max_t(u32, 4, dep->interval);  in dwc3_gadget_start_isoc_quirk()
1889 	dep->start_cmd_status = 0;  in dwc3_gadget_start_isoc_quirk()
1890 	dep->combo_num = 0;  in dwc3_gadget_start_isoc_quirk()
1897 	const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;  in __dwc3_gadget_start_isoc()
1898 	struct dwc3 *dwc = dep->dwc;  in __dwc3_gadget_start_isoc()
1902 	if (list_empty(&dep->pending_list) &&  in __dwc3_gadget_start_isoc()
1903 	    list_empty(&dep->started_list)) {  in __dwc3_gadget_start_isoc()
1904 		dep->flags |= DWC3_EP_PENDING_REQUEST;  in __dwc3_gadget_start_isoc()
1905 		return -EAGAIN;  in __dwc3_gadget_start_isoc()
1908 	if (!dwc->dis_start_transfer_quirk &&  in __dwc3_gadget_start_isoc()
1911 		if (dwc->gadget->speed <= USB_SPEED_HIGH && dep->direction)  in __dwc3_gadget_start_isoc()
1915 	if (desc->bInterval <= 14 &&  in __dwc3_gadget_start_isoc()
1916 	    dwc->gadget->speed >= USB_SPEED_HIGH) {  in __dwc3_gadget_start_isoc()
1919 				(dep->frame_number & DWC3_FRNUMBER_MASK);  in __dwc3_gadget_start_isoc()
1930 		dep->frame_number = (dep->frame_number & ~DWC3_FRNUMBER_MASK) |  in __dwc3_gadget_start_isoc()
1933 			dep->frame_number += BIT(14);  in __dwc3_gadget_start_isoc()
1940 		if (desc->bInterval < 3)  in __dwc3_gadget_start_isoc()
1941 			future_interval += 3 - desc->bInterval;  in __dwc3_gadget_start_isoc()
1943 		dep->frame_number = DWC3_ALIGN_FRAME(dep, future_interval);  in __dwc3_gadget_start_isoc()
1946 		if (ret != -EAGAIN)  in __dwc3_gadget_start_isoc()
1951 	 * After a number of unsuccessful start attempts due to bus-expiry  in __dwc3_gadget_start_isoc()
1955 	if (ret == -EAGAIN)  in __dwc3_gadget_start_isoc()
1963 	struct dwc3		*dwc = dep->dwc;  in __dwc3_gadget_ep_queue()
1965 	if (!dep->endpoint.desc || !dwc->pullups_connected || !dwc->connected) {  in __dwc3_gadget_ep_queue()
1966 		dev_dbg(dwc->dev, "%s: can't queue to disabled endpoint\n",  in __dwc3_gadget_ep_queue()
1967 				dep->name);  in __dwc3_gadget_ep_queue()
1968 		return -ESHUTDOWN;  in __dwc3_gadget_ep_queue()
1971 	if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",  in __dwc3_gadget_ep_queue()
1972 				&req->request, req->dep->name))  in __dwc3_gadget_ep_queue()
1973 		return -EINVAL;  in __dwc3_gadget_ep_queue()
1975 	if (WARN(req->status < DWC3_REQUEST_STATUS_COMPLETED,  in __dwc3_gadget_ep_queue()
1977 				dep->name, &req->request))  in __dwc3_gadget_ep_queue()
1978 		return -EINVAL;  in __dwc3_gadget_ep_queue()
1980 	pm_runtime_get(dwc->dev);  in __dwc3_gadget_ep_queue()
1982 	req->request.actual	= 0;  in __dwc3_gadget_ep_queue()
1983 	req->request.status	= -EINPROGRESS;  in __dwc3_gadget_ep_queue()
1987 	list_add_tail(&req->list, &dep->pending_list);  in __dwc3_gadget_ep_queue()
1988 	req->status = DWC3_REQUEST_STATUS_QUEUED;  in __dwc3_gadget_ep_queue()
1990 	if (dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)  in __dwc3_gadget_ep_queue()
1997 	if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||  in __dwc3_gadget_ep_queue()
1998 	    (dep->flags & DWC3_EP_WEDGE) ||  in __dwc3_gadget_ep_queue()
1999 	    (dep->flags & DWC3_EP_DELAY_STOP) ||  in __dwc3_gadget_ep_queue()
2000 	    (dep->flags & DWC3_EP_STALL)) {  in __dwc3_gadget_ep_queue()
2001 		dep->flags |= DWC3_EP_DELAY_START;  in __dwc3_gadget_ep_queue()
2008 	 * (micro-)frame number.  in __dwc3_gadget_ep_queue()
2013 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in __dwc3_gadget_ep_queue()
2014 		if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {  in __dwc3_gadget_ep_queue()
2015 			if ((dep->flags & DWC3_EP_PENDING_REQUEST))  in __dwc3_gadget_ep_queue()
2032 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep_queue()
2038 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_ep_queue()
2040 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_ep_queue()
2049 	/* If req->trb is not set, then the request has not started */  in dwc3_gadget_ep_skip_trbs()
2050 	if (!req->trb)  in dwc3_gadget_ep_skip_trbs()
2063 	for (i = 0; i < req->num_trbs; i++) {  in dwc3_gadget_ep_skip_trbs()
2066 		trb = &dep->trb_pool[dep->trb_dequeue];  in dwc3_gadget_ep_skip_trbs()
2067 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;  in dwc3_gadget_ep_skip_trbs()
2071 	req->num_trbs = 0;  in dwc3_gadget_ep_skip_trbs()
2077 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep_cleanup_cancelled_requests()
2079 	while (!list_empty(&dep->cancelled_list)) {  in dwc3_gadget_ep_cleanup_cancelled_requests()
2080 		req = next_request(&dep->cancelled_list);  in dwc3_gadget_ep_cleanup_cancelled_requests()
2082 		switch (req->status) {  in dwc3_gadget_ep_cleanup_cancelled_requests()
2084 			dwc3_gadget_giveback(dep, req, -ESHUTDOWN);  in dwc3_gadget_ep_cleanup_cancelled_requests()
2087 			dwc3_gadget_giveback(dep, req, -ECONNRESET);  in dwc3_gadget_ep_cleanup_cancelled_requests()
2090 			dwc3_gadget_giveback(dep, req, -EPIPE);  in dwc3_gadget_ep_cleanup_cancelled_requests()
2093 			dev_err(dwc->dev, "request cancelled with wrong reason:%d\n", req->status);  in dwc3_gadget_ep_cleanup_cancelled_requests()
2094 			dwc3_gadget_giveback(dep, req, -ECONNRESET);  in dwc3_gadget_ep_cleanup_cancelled_requests()
2101 		if (!dep->endpoint.desc)  in dwc3_gadget_ep_cleanup_cancelled_requests()
2113 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep_dequeue()
2120 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_ep_dequeue()
2122 	list_for_each_entry(r, &dep->cancelled_list, list) {  in dwc3_gadget_ep_dequeue()
2127 	list_for_each_entry(r, &dep->pending_list, list) {  in dwc3_gadget_ep_dequeue()
2136 			if (dep->number > 1)  in dwc3_gadget_ep_dequeue()
2137 				dwc3_gadget_giveback(dep, req, -ECONNRESET);  in dwc3_gadget_ep_dequeue()
2144 	list_for_each_entry(r, &dep->started_list, list) {  in dwc3_gadget_ep_dequeue()
2155 			list_for_each_entry_safe(r, t, &dep->started_list, list)  in dwc3_gadget_ep_dequeue()
2159 			dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;  in dwc3_gadget_ep_dequeue()
2165 	dev_err(dwc->dev, "request %pK was not queued to %s\n",  in dwc3_gadget_ep_dequeue()
2166 		request, ep->name);  in dwc3_gadget_ep_dequeue()
2167 	ret = -EINVAL;  in dwc3_gadget_ep_dequeue()
2169 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_ep_dequeue()
2177 	struct dwc3				*dwc = dep->dwc;  in __dwc3_gadget_ep_set_halt()
2182 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {  in __dwc3_gadget_ep_set_halt()
2183 		dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);  in __dwc3_gadget_ep_set_halt()
2184 		return -EINVAL;  in __dwc3_gadget_ep_set_halt()
2195 		if (dep->number > 1)  in __dwc3_gadget_ep_set_halt()
2196 			trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);  in __dwc3_gadget_ep_set_halt()
2198 			trb = &dwc->ep0_trb[dep->trb_enqueue];  in __dwc3_gadget_ep_set_halt()
2200 		transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;  in __dwc3_gadget_ep_set_halt()
2201 		started = !list_empty(&dep->started_list);  in __dwc3_gadget_ep_set_halt()
2203 		if (!protocol && ((dep->direction && transfer_in_flight) ||  in __dwc3_gadget_ep_set_halt()
2204 				(!dep->direction && started))) {  in __dwc3_gadget_ep_set_halt()
2205 			return -EAGAIN;  in __dwc3_gadget_ep_set_halt()
2211 			dev_err(dwc->dev, "failed to set STALL on %s\n",  in __dwc3_gadget_ep_set_halt()
2212 					dep->name);  in __dwc3_gadget_ep_set_halt()
2214 			dep->flags |= DWC3_EP_STALL;  in __dwc3_gadget_ep_set_halt()
2221 		if (dep->number <= 1) {  in __dwc3_gadget_ep_set_halt()
2222 			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);  in __dwc3_gadget_ep_set_halt()
2228 		list_for_each_entry_safe(req, tmp, &dep->started_list, list)  in __dwc3_gadget_ep_set_halt()
2231 		if (dep->flags & DWC3_EP_END_TRANSFER_PENDING ||  in __dwc3_gadget_ep_set_halt()
2232 		    (dep->flags & DWC3_EP_DELAY_STOP)) {  in __dwc3_gadget_ep_set_halt()
2233 			dep->flags |= DWC3_EP_PENDING_CLEAR_STALL;  in __dwc3_gadget_ep_set_halt()
2235 				dwc->clear_stall_protocol = dep->number;  in __dwc3_gadget_ep_set_halt()
2244 			dev_err(dwc->dev, "failed to clear STALL on %s\n",  in __dwc3_gadget_ep_set_halt()
2245 					dep->name);  in __dwc3_gadget_ep_set_halt()
2249 		dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);  in __dwc3_gadget_ep_set_halt()
2251 		if ((dep->flags & DWC3_EP_DELAY_START) &&  in __dwc3_gadget_ep_set_halt()
2252 		    !usb_endpoint_xfer_isoc(dep->endpoint.desc))  in __dwc3_gadget_ep_set_halt()
2255 		dep->flags &= ~DWC3_EP_DELAY_START;  in __dwc3_gadget_ep_set_halt()
2264 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep_set_halt()
2270 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_ep_set_halt()
2272 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_ep_set_halt()
2280 	struct dwc3			*dwc = dep->dwc;  in dwc3_gadget_ep_set_wedge()
2284 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_ep_set_wedge()
2285 	dep->flags |= DWC3_EP_WEDGE;  in dwc3_gadget_ep_set_wedge()
2287 	if (dep->number == 0 || dep->number == 1)  in dwc3_gadget_ep_set_wedge()
2291 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_ep_set_wedge()
2296 /* -------------------------------------------------------------------------- */
2326 /* -------------------------------------------------------------------------- */
2335 	reg = dwc3_readl(dwc->regs, DWC3_DEVTEN);  in dwc3_gadget_enable_linksts_evts()
2341 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);  in dwc3_gadget_enable_linksts_evts()
2366 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);  in __dwc3_gadget_wakeup()
2379 		return -EINVAL;  in __dwc3_gadget_wakeup()
2387 		dev_err(dwc->dev, "failed to put link in Recovery\n");  in __dwc3_gadget_wakeup()
2395 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in __dwc3_gadget_wakeup()
2397 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);  in __dwc3_gadget_wakeup()
2410 	while (retries--) {  in __dwc3_gadget_wakeup()
2411 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);  in __dwc3_gadget_wakeup()
2419 		dev_err(dwc->dev, "failed to send remote wakeup\n");  in __dwc3_gadget_wakeup()
2420 		return -EINVAL;  in __dwc3_gadget_wakeup()
2432 	if (!dwc->wakeup_configured) {  in dwc3_gadget_wakeup()
2433 		dev_err(dwc->dev, "remote wakeup not configured\n");  in dwc3_gadget_wakeup()
2434 		return -EINVAL;  in dwc3_gadget_wakeup()
2437 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_wakeup()
2438 	if (!dwc->gadget->wakeup_armed) {  in dwc3_gadget_wakeup()
2439 		dev_err(dwc->dev, "not armed for remote wakeup\n");  in dwc3_gadget_wakeup()
2440 		spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_wakeup()
2441 		return -EINVAL;  in dwc3_gadget_wakeup()
2445 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_wakeup()
2459 	if (!dwc->wakeup_configured) {  in dwc3_gadget_func_wakeup()
2460 		dev_err(dwc->dev, "remote wakeup not configured\n");  in dwc3_gadget_func_wakeup()
2461 		return -EINVAL;  in dwc3_gadget_func_wakeup()
2464 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_func_wakeup()
2473 			spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_func_wakeup()
2474 			return -EINVAL;  in dwc3_gadget_func_wakeup()
2477 		dwc->suspended = false;  in dwc3_gadget_func_wakeup()
2478 		dwc->link_state = DWC3_LINK_STATE_U0;  in dwc3_gadget_func_wakeup()
2485 		dev_err(dwc->dev, "function remote wakeup failed, ret:%d\n", ret);  in dwc3_gadget_func_wakeup()
2487 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_func_wakeup()
2497 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_set_remote_wakeup()
2498 	dwc->wakeup_configured = !!set;  in dwc3_gadget_set_remote_wakeup()
2499 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_set_remote_wakeup()
2510 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_set_selfpowered()
2511 	g->is_selfpowered = !!is_selfpowered;  in dwc3_gadget_set_selfpowered()
2512 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_set_selfpowered()
2521 	for (epnum = 2; epnum < dwc->num_eps; epnum++) {  in dwc3_stop_active_transfers()
2524 		dep = dwc->eps[epnum];  in dwc3_stop_active_transfers()
2528 		dwc3_remove_requests(dwc, dep, -ESHUTDOWN);  in dwc3_stop_active_transfers()
2534 	enum usb_ssp_rate	ssp_rate = dwc->gadget_ssp_rate;  in __dwc3_gadget_set_ssp_rate()
2538 		ssp_rate = dwc->max_ssp_rate;  in __dwc3_gadget_set_ssp_rate()
2540 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);  in __dwc3_gadget_set_ssp_rate()
2546 	else if (dwc->max_ssp_rate != USB_SSP_GEN_1x2)  in __dwc3_gadget_set_ssp_rate()
2550 	    dwc->max_ssp_rate != USB_SSP_GEN_2x1)  in __dwc3_gadget_set_ssp_rate()
2553 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);  in __dwc3_gadget_set_ssp_rate()
2561 	speed = dwc->gadget_max_speed;  in __dwc3_gadget_set_speed()
2562 	if (speed == USB_SPEED_UNKNOWN || speed > dwc->maximum_speed)  in __dwc3_gadget_set_speed()
2563 		speed = dwc->maximum_speed;  in __dwc3_gadget_set_speed()
2571 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);  in __dwc3_gadget_set_speed()
2577 	 * bit if we try to force the IP to USB2-only mode.  in __dwc3_gadget_set_speed()
2588 	    !dwc->dis_metastability_quirk) {  in __dwc3_gadget_set_speed()
2608 			dev_err(dwc->dev, "invalid speed (%d)\n", speed);  in __dwc3_gadget_set_speed()
2622 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);  in __dwc3_gadget_set_speed()
2631 	if (pm_runtime_suspended(dwc->dev))  in dwc3_gadget_run_stop()
2647 	reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));  in dwc3_gadget_run_stop()
2659 		dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);  in dwc3_gadget_run_stop()
2661 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_gadget_run_stop()
2673 		dwc->pullups_connected = true;  in dwc3_gadget_run_stop()
2677 		dwc->pullups_connected = false;  in dwc3_gadget_run_stop()
2684 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);  in dwc3_gadget_run_stop()
2686 	} while (--timeout && !(!is_on ^ !reg));  in dwc3_gadget_run_stop()
2689 		reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));  in dwc3_gadget_run_stop()
2691 		dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);  in dwc3_gadget_run_stop()
2695 		return -ETIMEDOUT;  in dwc3_gadget_run_stop()
2709 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_soft_disconnect()
2710 	if (!dwc->pullups_connected) {  in dwc3_gadget_soft_disconnect()
2711 		spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_soft_disconnect()
2715 	dwc->connected = false;  in dwc3_gadget_soft_disconnect()
2721 	if (dwc->delayed_status)  in dwc3_gadget_soft_disconnect()
2726 	 * Section 4.1.8 Table 4-7, it states that for a device-initiated  in dwc3_gadget_soft_disconnect()
2732 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_soft_disconnect()
2741 	if (dwc->ep0state != EP0_SETUP_PHASE) {  in dwc3_gadget_soft_disconnect()
2742 		reinit_completion(&dwc->ep0_in_setup);  in dwc3_gadget_soft_disconnect()
2744 		ret = wait_for_completion_timeout(&dwc->ep0_in_setup,  in dwc3_gadget_soft_disconnect()
2747 			dev_warn(dwc->dev, "wait for SETUP phase timed out\n");  in dwc3_gadget_soft_disconnect()
2748 			spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_soft_disconnect()
2750 			spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_soft_disconnect()
2769 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_soft_disconnect()
2771 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_soft_disconnect()
2783 	 * device-initiated disconnect requires a core soft reset  in dwc3_gadget_soft_connect()
2802 	dwc->softconnect = is_on;  in dwc3_gadget_pullup()
2810 		pm_runtime_barrier(dwc->dev);  in dwc3_gadget_pullup()
2811 		if (pm_runtime_suspended(dwc->dev))  in dwc3_gadget_pullup()
2820 	ret = pm_runtime_get_sync(dwc->dev);  in dwc3_gadget_pullup()
2822 		pm_runtime_put(dwc->dev);  in dwc3_gadget_pullup()
2824 			pm_runtime_set_suspended(dwc->dev);  in dwc3_gadget_pullup()
2828 	if (dwc->pullups_connected == is_on) {  in dwc3_gadget_pullup()
2829 		pm_runtime_put(dwc->dev);  in dwc3_gadget_pullup()
2833 	synchronize_irq(dwc->irq_gadget);  in dwc3_gadget_pullup()
2840 	pm_runtime_put(dwc->dev);  in dwc3_gadget_pullup()
2861 	/* On 2.30a and above this bit enables U3/L2-L1 Suspend Events */  in dwc3_gadget_enable_irq()
2865 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);  in dwc3_gadget_enable_irq()
2871 	dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);  in dwc3_gadget_disable_irq()
2878  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
2891  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
2905 	ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);  in dwc3_gadget_setup_nump()
2908 	nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;  in dwc3_gadget_setup_nump()
2912 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);  in dwc3_gadget_setup_nump()
2915 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);  in dwc3_gadget_setup_nump()
2925 	 * Use IMOD if enabled via dwc->imod_interval. Otherwise, if  in __dwc3_gadget_start()
2928 	if (dwc->imod_interval) {  in __dwc3_gadget_start()
2929 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);  in __dwc3_gadget_start()
2930 		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);  in __dwc3_gadget_start()
2932 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);  in __dwc3_gadget_start()
2942 	reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);  in __dwc3_gadget_start()
2948 	dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);  in __dwc3_gadget_start()
2959 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);  in __dwc3_gadget_start()
2961 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);  in __dwc3_gadget_start()
2964 	if (DWC3_MST_CAPABLE(&dwc->hwparams)) {  in __dwc3_gadget_start()
2965 		reg = dwc3_readl(dwc->regs, DWC3_DCFG1);  in __dwc3_gadget_start()
2967 		dwc3_writel(dwc->regs, DWC3_DCFG1, reg);  in __dwc3_gadget_start()
2975 		dev_err(dwc->dev, "failed to config endpoints\n");  in __dwc3_gadget_start()
2979 	dep = dwc->eps[0];  in __dwc3_gadget_start()
2980 	dep->flags = 0;  in __dwc3_gadget_start()
2983 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);  in __dwc3_gadget_start()
2987 	dep = dwc->eps[1];  in __dwc3_gadget_start()
2988 	dep->flags = 0;  in __dwc3_gadget_start()
2991 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);  in __dwc3_gadget_start()
2996 	dwc->ep0state = EP0_SETUP_PHASE;  in __dwc3_gadget_start()
2997 	dwc->ep0_bounced = false;  in __dwc3_gadget_start()
2998 	dwc->link_state = DWC3_LINK_STATE_SS_DIS;  in __dwc3_gadget_start()
2999 	dwc->delayed_status = false;  in __dwc3_gadget_start()
3008 	__dwc3_gadget_ep_disable(dwc->eps[0]);  in __dwc3_gadget_start()
3022 	irq = dwc->irq_gadget;  in dwc3_gadget_start()
3024 			IRQF_SHARED, "dwc3", dwc->ev_buf);  in dwc3_gadget_start()
3026 		dev_err(dwc->dev, "failed to request irq #%d --> %d\n",  in dwc3_gadget_start()
3031 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_start()
3032 	dwc->gadget_driver	= driver;  in dwc3_gadget_start()
3033 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_start()
3035 	if (dwc->sys_wakeup)  in dwc3_gadget_start()
3036 		device_wakeup_enable(dwc->sysdev);  in dwc3_gadget_start()
3044 	__dwc3_gadget_ep_disable(dwc->eps[0]);  in __dwc3_gadget_stop()
3045 	__dwc3_gadget_ep_disable(dwc->eps[1]);  in __dwc3_gadget_stop()
3053 	if (dwc->sys_wakeup)  in dwc3_gadget_stop()
3054 		device_wakeup_disable(dwc->sysdev);  in dwc3_gadget_stop()
3056 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_stop()
3057 	dwc->gadget_driver	= NULL;  in dwc3_gadget_stop()
3058 	dwc->max_cfg_eps = 0;  in dwc3_gadget_stop()
3059 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_stop()
3061 	free_irq(dwc->irq_gadget, dwc->ev_buf);  in dwc3_gadget_stop()
3071 	params->besl_baseline = USB_DEFAULT_BESL_UNSPECIFIED;  in dwc3_gadget_config_params()
3072 	params->besl_deep = USB_DEFAULT_BESL_UNSPECIFIED;  in dwc3_gadget_config_params()
3075 	if (!dwc->dis_enblslpm_quirk) {  in dwc3_gadget_config_params()
3085 		params->besl_baseline = 1;  in dwc3_gadget_config_params()
3086 		if (dwc->is_utmi_l1_suspend)  in dwc3_gadget_config_params()
3087 			params->besl_deep =  in dwc3_gadget_config_params()
3088 				clamp_t(u8, dwc->hird_threshold, 2, 15);  in dwc3_gadget_config_params()
3092 	if (dwc->dis_u1_entry_quirk)  in dwc3_gadget_config_params()
3093 		params->bU1devExitLat = 0;  in dwc3_gadget_config_params()
3095 		params->bU1devExitLat = DWC3_DEFAULT_U1_DEV_EXIT_LAT;  in dwc3_gadget_config_params()
3098 	if (dwc->dis_u2_entry_quirk)  in dwc3_gadget_config_params()
3099 		params->bU2DevExitLat = 0;  in dwc3_gadget_config_params()
3101 		params->bU2DevExitLat =  in dwc3_gadget_config_params()
3111 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_set_speed()
3112 	dwc->gadget_max_speed = speed;  in dwc3_gadget_set_speed()
3113 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_set_speed()
3122 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_set_ssp_rate()
3123 	dwc->gadget_max_speed = USB_SPEED_SUPER_PLUS;  in dwc3_gadget_set_ssp_rate()
3124 	dwc->gadget_ssp_rate = rate;  in dwc3_gadget_set_ssp_rate()
3125 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_set_ssp_rate()
3134 	if (dwc->usb2_phy)  in dwc3_gadget_vbus_draw()
3135 		return usb_phy_set_power(dwc->usb2_phy, mA);  in dwc3_gadget_vbus_draw()
3137 	if (!dwc->usb_psy)  in dwc3_gadget_vbus_draw()
3138 		return -EOPNOTSUPP;  in dwc3_gadget_vbus_draw()
3141 	ret = power_supply_set_property(dwc->usb_psy, POWER_SUPPLY_PROP_INPUT_CURRENT_LIMIT, &val);  in dwc3_gadget_vbus_draw()
3147  * dwc3_gadget_check_config - ensure dwc3 can support the USB configuration
3164 	if (!dwc->do_fifo_resize)  in dwc3_gadget_check_config()
3167 	list_for_each_entry(ep, &g->ep_list, ep_list) {  in dwc3_gadget_check_config()
3169 		if (ep->claimed && (ep->address & USB_DIR_IN))  in dwc3_gadget_check_config()
3173 	if (ep_num <= dwc->max_cfg_eps)  in dwc3_gadget_check_config()
3177 	dwc->max_cfg_eps = ep_num;  in dwc3_gadget_check_config()
3179 	fifo_size = dwc3_gadget_calc_tx_fifo_size(dwc, dwc->max_cfg_eps);  in dwc3_gadget_check_config()
3181 	fifo_size += dwc->max_cfg_eps;  in dwc3_gadget_check_config()
3186 		return -ENOMEM;  in dwc3_gadget_check_config()
3196 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_async_callbacks()
3197 	dwc->async_callbacks = enable;  in dwc3_gadget_async_callbacks()
3198 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_async_callbacks()
3218 /* -------------------------------------------------------------------------- */
3222 	struct dwc3 *dwc = dep->dwc;  in dwc3_gadget_init_control_endpoint()
3224 	usb_ep_set_maxpacket_limit(&dep->endpoint, 512);  in dwc3_gadget_init_control_endpoint()
3225 	dep->endpoint.maxburst = 1;  in dwc3_gadget_init_control_endpoint()
3226 	dep->endpoint.ops = &dwc3_gadget_ep0_ops;  in dwc3_gadget_init_control_endpoint()
3227 	if (!dep->direction)  in dwc3_gadget_init_control_endpoint()
3228 		dwc->gadget->ep0 = &dep->endpoint;  in dwc3_gadget_init_control_endpoint()
3230 	dep->endpoint.caps.type_control = true;  in dwc3_gadget_init_control_endpoint()
3237 	struct dwc3 *dwc = dep->dwc;  in dwc3_gadget_init_in_endpoint()
3247 	size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));  in dwc3_gadget_init_in_endpoint()
3258 	 * maxpacket = mdwidth * (fifo_size - 1);  in dwc3_gadget_init_in_endpoint()
3262 	 * maxpacket = mdwidth * ((fifo_size - 1) - 1) - mdwidth;  in dwc3_gadget_init_in_endpoint()
3265 		maxpacket = mdwidth * (size - 1);  in dwc3_gadget_init_in_endpoint()
3267 		maxpacket = mdwidth * ((size - 1) - 1) - mdwidth;  in dwc3_gadget_init_in_endpoint()
3271 	usb_ep_set_maxpacket_limit(&dep->endpoint, size);  in dwc3_gadget_init_in_endpoint()
3273 	dep->endpoint.max_streams = 16;  in dwc3_gadget_init_in_endpoint()
3274 	dep->endpoint.ops = &dwc3_gadget_ep_ops;  in dwc3_gadget_init_in_endpoint()
3275 	list_add_tail(&dep->endpoint.ep_list,  in dwc3_gadget_init_in_endpoint()
3276 			&dwc->gadget->ep_list);  in dwc3_gadget_init_in_endpoint()
3277 	dep->endpoint.caps.type_iso = true;  in dwc3_gadget_init_in_endpoint()
3278 	dep->endpoint.caps.type_bulk = true;  in dwc3_gadget_init_in_endpoint()
3279 	dep->endpoint.caps.type_int = true;  in dwc3_gadget_init_in_endpoint()
3286 	struct dwc3 *dwc = dep->dwc;  in dwc3_gadget_init_out_endpoint()
3296 	size = dwc3_readl(dwc->regs, DWC3_GRXFIFOSIZ(0));  in dwc3_gadget_init_out_endpoint()
3313 	size -= (3 * 8) + 16;  in dwc3_gadget_init_out_endpoint()
3319 	usb_ep_set_maxpacket_limit(&dep->endpoint, size);  in dwc3_gadget_init_out_endpoint()
3320 	dep->endpoint.max_streams = 16;  in dwc3_gadget_init_out_endpoint()
3321 	dep->endpoint.ops = &dwc3_gadget_ep_ops;  in dwc3_gadget_init_out_endpoint()
3322 	list_add_tail(&dep->endpoint.ep_list,  in dwc3_gadget_init_out_endpoint()
3323 			&dwc->gadget->ep_list);  in dwc3_gadget_init_out_endpoint()
3324 	dep->endpoint.caps.type_iso = true;  in dwc3_gadget_init_out_endpoint()
3325 	dep->endpoint.caps.type_bulk = true;  in dwc3_gadget_init_out_endpoint()
3326 	dep->endpoint.caps.type_int = true;  in dwc3_gadget_init_out_endpoint()
3340 		return -ENOMEM;  in dwc3_gadget_init_endpoint()
3342 	dep->dwc = dwc;  in dwc3_gadget_init_endpoint()
3343 	dep->number = epnum;  in dwc3_gadget_init_endpoint()
3344 	dep->direction = direction;  in dwc3_gadget_init_endpoint()
3345 	dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);  in dwc3_gadget_init_endpoint()
3346 	dwc->eps[epnum] = dep;  in dwc3_gadget_init_endpoint()
3347 	dep->combo_num = 0;  in dwc3_gadget_init_endpoint()
3348 	dep->start_cmd_status = 0;  in dwc3_gadget_init_endpoint()
3350 	snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,  in dwc3_gadget_init_endpoint()
3353 	dep->endpoint.name = dep->name;  in dwc3_gadget_init_endpoint()
3355 	if (!(dep->number > 1)) {  in dwc3_gadget_init_endpoint()
3356 		dep->endpoint.desc = &dwc3_gadget_ep0_desc;  in dwc3_gadget_init_endpoint()
3357 		dep->endpoint.comp_desc = NULL;  in dwc3_gadget_init_endpoint()
3370 	dep->endpoint.caps.dir_in = direction;  in dwc3_gadget_init_endpoint()
3371 	dep->endpoint.caps.dir_out = !direction;  in dwc3_gadget_init_endpoint()
3373 	INIT_LIST_HEAD(&dep->pending_list);  in dwc3_gadget_init_endpoint()
3374 	INIT_LIST_HEAD(&dep->started_list);  in dwc3_gadget_init_endpoint()
3375 	INIT_LIST_HEAD(&dep->cancelled_list);  in dwc3_gadget_init_endpoint()
3386 	INIT_LIST_HEAD(&dwc->gadget->ep_list);  in dwc3_gadget_init_endpoints()
3405 		dep = dwc->eps[epnum];  in dwc3_gadget_free_endpoints()
3410 		 * bi-directional USB endpoint 0.  in dwc3_gadget_free_endpoints()
3419 			list_del(&dep->endpoint.ep_list);  in dwc3_gadget_free_endpoints()
3427 /* -------------------------------------------------------------------------- */
3438 	req->num_trbs--;  in dwc3_gadget_ep_reclaim_completed_trb()
3450 	if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))  in dwc3_gadget_ep_reclaim_completed_trb()
3451 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;  in dwc3_gadget_ep_reclaim_completed_trb()
3455 	 * have the Isoc-First type. Track and report its interval frame number.  in dwc3_gadget_ep_reclaim_completed_trb()
3457 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&  in dwc3_gadget_ep_reclaim_completed_trb()
3458 	    (trb->ctrl & DWC3_TRBCTL_ISOCHRONOUS_FIRST)) {  in dwc3_gadget_ep_reclaim_completed_trb()
3461 		frame_number = DWC3_TRB_CTRL_GET_SID_SOFN(trb->ctrl);  in dwc3_gadget_ep_reclaim_completed_trb()
3462 		frame_number &= ~(dep->interval - 1);  in dwc3_gadget_ep_reclaim_completed_trb()
3463 		req->request.frame_number = frame_number;  in dwc3_gadget_ep_reclaim_completed_trb()
3469 	 * TRB. Don't add it to req->remaining calculation.  in dwc3_gadget_ep_reclaim_completed_trb()
3471 	if (trb->bpl == lower_32_bits(dep->dwc->bounce_addr) &&  in dwc3_gadget_ep_reclaim_completed_trb()
3472 	    trb->bph == upper_32_bits(dep->dwc->bounce_addr)) {  in dwc3_gadget_ep_reclaim_completed_trb()
3473 		trb->ctrl &= ~DWC3_TRB_CTRL_HWO;  in dwc3_gadget_ep_reclaim_completed_trb()
3477 	count = trb->size & DWC3_TRB_SIZE_MASK;  in dwc3_gadget_ep_reclaim_completed_trb()
3478 	req->remaining += count;  in dwc3_gadget_ep_reclaim_completed_trb()
3480 	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)  in dwc3_gadget_ep_reclaim_completed_trb()
3483 	if (event->status & DEPEVT_STATUS_SHORT && !chain)  in dwc3_gadget_ep_reclaim_completed_trb()
3486 	if ((trb->ctrl & DWC3_TRB_CTRL_ISP_IMI) &&  in dwc3_gadget_ep_reclaim_completed_trb()
3487 	    DWC3_TRB_SIZE_TRBSTS(trb->size) == DWC3_TRBSTS_MISSED_ISOC)  in dwc3_gadget_ep_reclaim_completed_trb()
3490 	if ((trb->ctrl & DWC3_TRB_CTRL_IOC) ||  in dwc3_gadget_ep_reclaim_completed_trb()
3491 	    (trb->ctrl & DWC3_TRB_CTRL_LST))  in dwc3_gadget_ep_reclaim_completed_trb()
3501 	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];  in dwc3_gadget_ep_reclaim_trb_sg()
3502 	struct scatterlist *sg = req->sg;  in dwc3_gadget_ep_reclaim_trb_sg()
3504 	unsigned int num_queued = req->num_queued_sgs;  in dwc3_gadget_ep_reclaim_trb_sg()
3509 		trb = &dep->trb_pool[dep->trb_dequeue];  in dwc3_gadget_ep_reclaim_trb_sg()
3511 		req->sg = sg_next(s);  in dwc3_gadget_ep_reclaim_trb_sg()
3512 		req->num_queued_sgs--;  in dwc3_gadget_ep_reclaim_trb_sg()
3527 	struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];  in dwc3_gadget_ep_reclaim_trb_linear()
3535 	return req->num_pending_sgs == 0 && req->num_queued_sgs == 0;  in dwc3_gadget_ep_request_completed()
3545 	if (req->request.num_mapped_sgs)  in dwc3_gadget_ep_cleanup_completed_request()
3552 	req->request.actual = req->request.length - req->remaining;  in dwc3_gadget_ep_cleanup_completed_request()
3557 	if (req->needs_extra_trb) {  in dwc3_gadget_ep_cleanup_completed_request()
3560 		req->needs_extra_trb = false;  in dwc3_gadget_ep_cleanup_completed_request()
3569 	if (req->request.no_interrupt) {  in dwc3_gadget_ep_cleanup_completed_request()
3572 		trb = dwc3_ep_prev_trb(dep, dep->trb_dequeue);  in dwc3_gadget_ep_cleanup_completed_request()
3573 		switch (DWC3_TRB_SIZE_TRBSTS(trb->size)) {  in dwc3_gadget_ep_cleanup_completed_request()
3576 			request_status = -EXDEV;  in dwc3_gadget_ep_cleanup_completed_request()
3602 	while (!list_empty(&dep->started_list)) {  in dwc3_gadget_ep_cleanup_completed_requests()
3605 		req = next_request(&dep->started_list);  in dwc3_gadget_ep_cleanup_completed_requests()
3614 		if (!dep->endpoint.desc)  in dwc3_gadget_ep_cleanup_completed_requests()
3622 	struct dwc3		*dwc = dep->dwc;  in dwc3_gadget_ep_should_continue()
3624 	if (!dep->endpoint.desc || !dwc->pullups_connected ||  in dwc3_gadget_ep_should_continue()
3625 	    !dwc->connected)  in dwc3_gadget_ep_should_continue()
3628 	if (!list_empty(&dep->pending_list))  in dwc3_gadget_ep_should_continue()
3635 	req = next_request(&dep->started_list);  in dwc3_gadget_ep_should_continue()
3645 	dep->frame_number = event->parameters;  in dwc3_gadget_endpoint_frame_from_event()
3651 	struct dwc3		*dwc = dep->dwc;  in dwc3_gadget_endpoint_trbs_complete()
3656 	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)  in dwc3_gadget_endpoint_trbs_complete()
3659 	if (!dep->endpoint.desc)  in dwc3_gadget_endpoint_trbs_complete()
3662 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&  in dwc3_gadget_endpoint_trbs_complete()
3663 		list_empty(&dep->started_list) &&  in dwc3_gadget_endpoint_trbs_complete()
3664 		(list_empty(&dep->pending_list) || status == -EXDEV))  in dwc3_gadget_endpoint_trbs_complete()
3672 	 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.  in dwc3_gadget_endpoint_trbs_complete()
3680 			dep = dwc->eps[i];  in dwc3_gadget_endpoint_trbs_complete()
3684 			if (!(dep->flags & DWC3_EP_ENABLED))  in dwc3_gadget_endpoint_trbs_complete()
3687 			if (!list_empty(&dep->started_list))  in dwc3_gadget_endpoint_trbs_complete()
3691 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_gadget_endpoint_trbs_complete()
3692 		reg |= dwc->u1u2;  in dwc3_gadget_endpoint_trbs_complete()
3693 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);  in dwc3_gadget_endpoint_trbs_complete()
3695 		dwc->u1u2 = 0;  in dwc3_gadget_endpoint_trbs_complete()
3706 	if (!dep->endpoint.desc)  in dwc3_gadget_endpoint_transfer_in_progress()
3709 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc))  in dwc3_gadget_endpoint_transfer_in_progress()
3712 	if (event->status & DEPEVT_STATUS_BUSERR)  in dwc3_gadget_endpoint_transfer_in_progress()
3713 		status = -ECONNRESET;  in dwc3_gadget_endpoint_transfer_in_progress()
3715 	if (event->status & DEPEVT_STATUS_MISSED_ISOC)  in dwc3_gadget_endpoint_transfer_in_progress()
3716 		status = -EXDEV;  in dwc3_gadget_endpoint_transfer_in_progress()
3726 	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;  in dwc3_gadget_endpoint_transfer_complete()
3728 	if (event->status & DEPEVT_STATUS_BUSERR)  in dwc3_gadget_endpoint_transfer_complete()
3729 		status = -ECONNRESET;  in dwc3_gadget_endpoint_transfer_complete()
3732 		dep->flags &= ~DWC3_EP_WAIT_TRANSFER_COMPLETE;  in dwc3_gadget_endpoint_transfer_complete()
3748 	if (dep->flags & DWC3_EP_END_TRANSFER_PENDING)  in dwc3_gadget_endpoint_transfer_not_ready()
3757 	u8 cmd = DEPEVT_PARAMETER_CMD(event->parameters);  in dwc3_gadget_endpoint_command_complete()
3767 	if (dep->stream_capable)  in dwc3_gadget_endpoint_command_complete()
3768 		dep->flags |= DWC3_EP_IGNORE_NEXT_NOSTREAM;  in dwc3_gadget_endpoint_command_complete()
3770 	dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;  in dwc3_gadget_endpoint_command_complete()
3771 	dep->flags &= ~DWC3_EP_TRANSFER_STARTED;  in dwc3_gadget_endpoint_command_complete()
3774 	if (dep->flags & DWC3_EP_PENDING_CLEAR_STALL) {  in dwc3_gadget_endpoint_command_complete()
3775 		struct dwc3 *dwc = dep->dwc;  in dwc3_gadget_endpoint_command_complete()
3777 		dep->flags &= ~DWC3_EP_PENDING_CLEAR_STALL;  in dwc3_gadget_endpoint_command_complete()
3779 			struct usb_ep *ep0 = &dwc->eps[0]->endpoint;  in dwc3_gadget_endpoint_command_complete()
3781 			dev_err(dwc->dev, "failed to clear STALL on %s\n", dep->name);  in dwc3_gadget_endpoint_command_complete()
3782 			if (dwc->delayed_status)  in dwc3_gadget_endpoint_command_complete()
3787 		dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);  in dwc3_gadget_endpoint_command_complete()
3788 		if (dwc->clear_stall_protocol == dep->number)  in dwc3_gadget_endpoint_command_complete()
3792 	if ((dep->flags & DWC3_EP_DELAY_START) &&  in dwc3_gadget_endpoint_command_complete()
3793 	    !usb_endpoint_xfer_isoc(dep->endpoint.desc))  in dwc3_gadget_endpoint_command_complete()
3796 	dep->flags &= ~DWC3_EP_DELAY_START;  in dwc3_gadget_endpoint_command_complete()
3802 	struct dwc3 *dwc = dep->dwc;  in dwc3_gadget_endpoint_stream_event()
3804 	if (event->status == DEPEVT_STREAMEVT_FOUND) {  in dwc3_gadget_endpoint_stream_event()
3805 		dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;  in dwc3_gadget_endpoint_stream_event()
3810 	switch (event->parameters) {  in dwc3_gadget_endpoint_stream_event()
3819 		if (dep->flags & DWC3_EP_FORCE_RESTART_STREAM) {  in dwc3_gadget_endpoint_stream_event()
3820 			if (dep->flags & DWC3_EP_FIRST_STREAM_PRIMED)  in dwc3_gadget_endpoint_stream_event()
3821 				dep->flags &= ~DWC3_EP_FORCE_RESTART_STREAM;  in dwc3_gadget_endpoint_stream_event()
3823 				dep->flags |= DWC3_EP_FIRST_STREAM_PRIMED;  in dwc3_gadget_endpoint_stream_event()
3828 		if ((dep->flags & DWC3_EP_IGNORE_NEXT_NOSTREAM) ||  in dwc3_gadget_endpoint_stream_event()
3829 		    !(dep->flags & DWC3_EP_FORCE_RESTART_STREAM) ||  in dwc3_gadget_endpoint_stream_event()
3830 		    (!DWC3_MST_CAPABLE(&dwc->hwparams) &&  in dwc3_gadget_endpoint_stream_event()
3831 		     !(dep->flags & DWC3_EP_WAIT_TRANSFER_COMPLETE)))  in dwc3_gadget_endpoint_stream_event()
3851 			dwc3_send_gadget_generic_command(dwc, cmd, dep->number);  in dwc3_gadget_endpoint_stream_event()
3853 			dep->flags |= DWC3_EP_DELAY_START;  in dwc3_gadget_endpoint_stream_event()
3861 	dep->flags &= ~DWC3_EP_IGNORE_NEXT_NOSTREAM;  in dwc3_gadget_endpoint_stream_event()
3868 	u8			epnum = event->endpoint_number;  in dwc3_endpoint_interrupt()
3870 	dep = dwc->eps[epnum];  in dwc3_endpoint_interrupt()
3872 		dev_warn(dwc->dev, "spurious event, endpoint %u is not allocated\n", epnum);  in dwc3_endpoint_interrupt()
3876 	if (!(dep->flags & DWC3_EP_ENABLED)) {  in dwc3_endpoint_interrupt()
3877 		if ((epnum > 1) && !(dep->flags & DWC3_EP_TRANSFER_STARTED))  in dwc3_endpoint_interrupt()
3881 		if ((event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT) &&  in dwc3_endpoint_interrupt()
3882 			!(epnum <= 1 && event->endpoint_event == DWC3_DEPEVT_XFERCOMPLETE))  in dwc3_endpoint_interrupt()
3891 	switch (event->endpoint_event) {  in dwc3_endpoint_interrupt()
3910 		dev_err(dwc->dev, "unknown endpoint event %d\n", event->endpoint_event);  in dwc3_endpoint_interrupt()
3917 	if (dwc->async_callbacks && dwc->gadget_driver->disconnect) {  in dwc3_disconnect_gadget()
3918 		spin_unlock(&dwc->lock);  in dwc3_disconnect_gadget()
3919 		dwc->gadget_driver->disconnect(dwc->gadget);  in dwc3_disconnect_gadget()
3920 		spin_lock(&dwc->lock);  in dwc3_disconnect_gadget()
3926 	if (dwc->async_callbacks && dwc->gadget_driver->suspend) {  in dwc3_suspend_gadget()
3927 		spin_unlock(&dwc->lock);  in dwc3_suspend_gadget()
3928 		dwc->gadget_driver->suspend(dwc->gadget);  in dwc3_suspend_gadget()
3929 		spin_lock(&dwc->lock);  in dwc3_suspend_gadget()
3935 	if (dwc->async_callbacks && dwc->gadget_driver->resume) {  in dwc3_resume_gadget()
3936 		spin_unlock(&dwc->lock);  in dwc3_resume_gadget()
3937 		dwc->gadget_driver->resume(dwc->gadget);  in dwc3_resume_gadget()
3938 		spin_lock(&dwc->lock);  in dwc3_resume_gadget()
3944 	if (!dwc->gadget_driver)  in dwc3_reset_gadget()
3947 	if (dwc->async_callbacks && dwc->gadget->speed != USB_SPEED_UNKNOWN) {  in dwc3_reset_gadget()
3948 		spin_unlock(&dwc->lock);  in dwc3_reset_gadget()
3949 		usb_gadget_udc_reset(dwc->gadget, dwc->gadget_driver);  in dwc3_reset_gadget()
3950 		spin_lock(&dwc->lock);  in dwc3_reset_gadget()
3957 	struct dwc3 *dwc = dep->dwc;  in dwc3_stop_active_transfer()
3965 	if (dep->number <= 1 && dwc->ep0state != EP0_DATA_PHASE)  in dwc3_stop_active_transfer()
3968 	if (interrupt && (dep->flags & DWC3_EP_DELAY_STOP))  in dwc3_stop_active_transfer()
3971 	if (!(dep->flags & DWC3_EP_TRANSFER_STARTED) ||  in dwc3_stop_active_transfer()
3972 	    (dep->flags & DWC3_EP_END_TRANSFER_PENDING))  in dwc3_stop_active_transfer()
3982 	if (dwc->ep0state != EP0_SETUP_PHASE && !dwc->delayed_status) {  in dwc3_stop_active_transfer()
3983 		dep->flags |= DWC3_EP_DELAY_STOP;  in dwc3_stop_active_transfer()
4029 		dep = dwc->eps[epnum];  in dwc3_clear_stall_all_ep()
4033 		if (!(dep->flags & DWC3_EP_STALL))  in dwc3_clear_stall_all_ep()
4036 		dep->flags &= ~DWC3_EP_STALL;  in dwc3_clear_stall_all_ep()
4047 	dwc->suspended = false;  in dwc3_gadget_disconnect_interrupt()
4051 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_gadget_disconnect_interrupt()
4056 	dwc->connected = false;  in dwc3_gadget_disconnect_interrupt()
4060 	dwc->gadget->speed = USB_SPEED_UNKNOWN;  in dwc3_gadget_disconnect_interrupt()
4061 	dwc->setup_packet_pending = false;  in dwc3_gadget_disconnect_interrupt()
4062 	dwc->gadget->wakeup_armed = false;  in dwc3_gadget_disconnect_interrupt()
4064 	usb_gadget_set_state(dwc->gadget, USB_STATE_NOTATTACHED);  in dwc3_gadget_disconnect_interrupt()
4071 	 * interrupt to set dwc->connected to FALSE.  in dwc3_gadget_disconnect_interrupt()
4073 	pm_request_idle(dwc->dev);  in dwc3_gadget_disconnect_interrupt()
4080 	dwc->suspended = false;  in dwc3_gadget_reset_interrupt()
4089 	dwc->connected = false;  in dwc3_gadget_reset_interrupt()
4118 		if (dwc->setup_packet_pending)  in dwc3_gadget_reset_interrupt()
4133 	 * Section 4.1.2 Table 4-2, it states that during a USB reset, the SW  in dwc3_gadget_reset_interrupt()
4138 	dwc->connected = true;  in dwc3_gadget_reset_interrupt()
4140 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_gadget_reset_interrupt()
4143 	dwc->test_mode = false;  in dwc3_gadget_reset_interrupt()
4144 	dwc->gadget->wakeup_armed = false;  in dwc3_gadget_reset_interrupt()
4149 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);  in dwc3_gadget_reset_interrupt()
4151 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);  in dwc3_gadget_reset_interrupt()
4162 	if (!dwc->softconnect)  in dwc3_gadget_conndone_interrupt()
4165 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);  in dwc3_gadget_conndone_interrupt()
4167 	dwc->speed = speed;  in dwc3_gadget_conndone_interrupt()
4172 	dwc->gadget->ssp_rate = USB_SSP_GEN_UNKNOWN;  in dwc3_gadget_conndone_interrupt()
4186 		dwc->gadget->ep0->maxpacket = 512;  in dwc3_gadget_conndone_interrupt()
4187 		dwc->gadget->speed = USB_SPEED_SUPER_PLUS;  in dwc3_gadget_conndone_interrupt()
4190 			dwc->gadget->ssp_rate = USB_SSP_GEN_2x2;  in dwc3_gadget_conndone_interrupt()
4192 			dwc->gadget->ssp_rate = USB_SSP_GEN_2x1;  in dwc3_gadget_conndone_interrupt()
4212 		dwc->gadget->ep0->maxpacket = 512;  in dwc3_gadget_conndone_interrupt()
4213 		dwc->gadget->speed = USB_SPEED_SUPER;  in dwc3_gadget_conndone_interrupt()
4216 			dwc->gadget->speed = USB_SPEED_SUPER_PLUS;  in dwc3_gadget_conndone_interrupt()
4217 			dwc->gadget->ssp_rate = USB_SSP_GEN_1x2;  in dwc3_gadget_conndone_interrupt()
4222 		dwc->gadget->ep0->maxpacket = 64;  in dwc3_gadget_conndone_interrupt()
4223 		dwc->gadget->speed = USB_SPEED_HIGH;  in dwc3_gadget_conndone_interrupt()
4227 		dwc->gadget->ep0->maxpacket = 64;  in dwc3_gadget_conndone_interrupt()
4228 		dwc->gadget->speed = USB_SPEED_FULL;  in dwc3_gadget_conndone_interrupt()
4232 	dwc->eps[1]->endpoint.maxpacket = dwc->gadget->ep0->maxpacket;  in dwc3_gadget_conndone_interrupt()
4234 	/* Enable USB2 LPM Capability */  in dwc3_gadget_conndone_interrupt()
4237 	    !dwc->usb2_gadget_lpm_disable &&  in dwc3_gadget_conndone_interrupt()
4240 		reg = dwc3_readl(dwc->regs, DWC3_DCFG);  in dwc3_gadget_conndone_interrupt()
4242 		dwc3_writel(dwc->regs, DWC3_DCFG, reg);  in dwc3_gadget_conndone_interrupt()
4244 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_gadget_conndone_interrupt()
4247 		reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold |  in dwc3_gadget_conndone_interrupt()
4248 					    (dwc->is_utmi_l1_suspend << 4));  in dwc3_gadget_conndone_interrupt()
4251 		 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and  in dwc3_gadget_conndone_interrupt()
4253 		 * BESL value in the LPM token is less than or equal to LPM  in dwc3_gadget_conndone_interrupt()
4254 		 * NYET threshold.  in dwc3_gadget_conndone_interrupt()
4256 		WARN_ONCE(DWC3_VER_IS_PRIOR(DWC3, 240A) && dwc->has_lpm_erratum,  in dwc3_gadget_conndone_interrupt()
4257 				"LPM Erratum not available on dwc3 revisions < 2.40a\n");  in dwc3_gadget_conndone_interrupt()
4259 		if (dwc->has_lpm_erratum && !DWC3_VER_IS_PRIOR(DWC3, 240A)) {  in dwc3_gadget_conndone_interrupt()
4261 			reg |= DWC3_DCTL_NYET_THRES(dwc->lpm_nyet_threshold);  in dwc3_gadget_conndone_interrupt()
4266 		if (dwc->usb2_gadget_lpm_disable) {  in dwc3_gadget_conndone_interrupt()
4267 			reg = dwc3_readl(dwc->regs, DWC3_DCFG);  in dwc3_gadget_conndone_interrupt()
4269 			dwc3_writel(dwc->regs, DWC3_DCFG, reg);  in dwc3_gadget_conndone_interrupt()
4272 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_gadget_conndone_interrupt()
4277 	dep = dwc->eps[0];  in dwc3_gadget_conndone_interrupt()
4280 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);  in dwc3_gadget_conndone_interrupt()
4284 	dep = dwc->eps[1];  in dwc3_gadget_conndone_interrupt()
4287 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);  in dwc3_gadget_conndone_interrupt()
4302 	dwc->suspended = false;  in dwc3_gadget_wakeup_interrupt()
4309 	if (dwc->async_callbacks && dwc->gadget_driver->resume) {  in dwc3_gadget_wakeup_interrupt()
4310 		spin_unlock(&dwc->lock);  in dwc3_gadget_wakeup_interrupt()
4311 		dwc->gadget_driver->resume(dwc->gadget);  in dwc3_gadget_wakeup_interrupt()
4312 		spin_lock(&dwc->lock);  in dwc3_gadget_wakeup_interrupt()
4315 	dwc->link_state = evtinfo & DWC3_LINK_STATE_MASK;  in dwc3_gadget_wakeup_interrupt()
4327 	 * host-initiated U3 exit.  in dwc3_gadget_linksts_change_interrupt()
4338 	 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation  in dwc3_gadget_linksts_change_interrupt()
4341 	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);  in dwc3_gadget_linksts_change_interrupt()
4344 		if ((dwc->link_state == DWC3_LINK_STATE_U3) &&  in dwc3_gadget_linksts_change_interrupt()
4365 	 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us  in dwc3_gadget_linksts_change_interrupt()
4373 			switch (dwc->link_state) {  in dwc3_gadget_linksts_change_interrupt()
4376 				reg = dwc3_readl(dwc->regs, DWC3_DCTL);  in dwc3_gadget_linksts_change_interrupt()
4382 				if (!dwc->u1u2)  in dwc3_gadget_linksts_change_interrupt()
4383 					dwc->u1u2 = reg & u1u2;  in dwc3_gadget_linksts_change_interrupt()
4398 		if (dwc->gadget->wakeup_armed) {  in dwc3_gadget_linksts_change_interrupt()
4401 			dwc->suspended = false;  in dwc3_gadget_linksts_change_interrupt()
4405 		if (dwc->speed == USB_SPEED_SUPER)  in dwc3_gadget_linksts_change_interrupt()
4420 	dwc->link_state = next;  in dwc3_gadget_linksts_change_interrupt()
4428 	if (!dwc->suspended && next == DWC3_LINK_STATE_U3) {  in dwc3_gadget_suspend_interrupt()
4429 		dwc->suspended = true;  in dwc3_gadget_suspend_interrupt()
4433 	dwc->link_state = next;  in dwc3_gadget_suspend_interrupt()
4439 	switch (event->type) {  in dwc3_gadget_interrupt()
4450 		dwc3_gadget_wakeup_interrupt(dwc, event->event_info);  in dwc3_gadget_interrupt()
4453 		dev_WARN_ONCE(dwc->dev, true, "unexpected hibernation event\n");  in dwc3_gadget_interrupt()
4456 		dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);  in dwc3_gadget_interrupt()
4461 			dwc3_gadget_suspend_interrupt(dwc, event->event_info);  in dwc3_gadget_interrupt()
4469 		dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);  in dwc3_gadget_interrupt()
4476 	trace_dwc3_event(event->raw, dwc);  in dwc3_process_event_entry()
4478 	if (!event->type.is_devspec)  in dwc3_process_event_entry()
4479 		dwc3_endpoint_interrupt(dwc, &event->depevt);  in dwc3_process_event_entry()
4480 	else if (event->type.type == DWC3_EVENT_TYPE_DEV)  in dwc3_process_event_entry()
4481 		dwc3_gadget_interrupt(dwc, &event->devt);  in dwc3_process_event_entry()
4483 		dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);  in dwc3_process_event_entry()
4488 	struct dwc3 *dwc = evt->dwc;  in dwc3_process_event_buf()
4492 	left = evt->count;  in dwc3_process_event_buf()
4494 	if (!(evt->flags & DWC3_EVENT_PENDING))  in dwc3_process_event_buf()
4500 		event.raw = *(u32 *) (evt->cache + evt->lpos);  in dwc3_process_event_buf()
4513 		evt->lpos = (evt->lpos + 4) % evt->length;  in dwc3_process_event_buf()
4514 		left -= 4;  in dwc3_process_event_buf()
4517 	evt->count = 0;  in dwc3_process_event_buf()
4521 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),  in dwc3_process_event_buf()
4522 		    DWC3_GEVNTSIZ_SIZE(evt->length));  in dwc3_process_event_buf()
4524 	evt->flags &= ~DWC3_EVENT_PENDING;  in dwc3_process_event_buf()
4531 	if (dwc->imod_interval) {  in dwc3_process_event_buf()
4532 		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);  in dwc3_process_event_buf()
4533 		dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);  in dwc3_process_event_buf()
4542 	struct dwc3 *dwc = evt->dwc;  in dwc3_thread_interrupt()
4547 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_thread_interrupt()
4549 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_thread_interrupt()
4557 	struct dwc3 *dwc = evt->dwc;  in dwc3_check_event_buf()
4561 	if (pm_runtime_suspended(dwc->dev)) {  in dwc3_check_event_buf()
4562 		dwc->pending_events = true;  in dwc3_check_event_buf()
4568 		pm_runtime_get(dwc->dev);  in dwc3_check_event_buf()
4569 		disable_irq_nosync(dwc->irq_gadget);  in dwc3_check_event_buf()
4574 	 * With PCIe legacy interrupt, test shows that top-half irq handler can  in dwc3_check_event_buf()
4575 	 * be called again after HW interrupt deassertion. Check if bottom-half  in dwc3_check_event_buf()
4579 	if (evt->flags & DWC3_EVENT_PENDING)  in dwc3_check_event_buf()
4582 	count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));  in dwc3_check_event_buf()
4587 	if (count > evt->length) {  in dwc3_check_event_buf()
4588 		dev_err_ratelimited(dwc->dev, "invalid count(%u) > evt->length(%u)\n",  in dwc3_check_event_buf()
4589 			count, evt->length);  in dwc3_check_event_buf()
4593 	evt->count = count;  in dwc3_check_event_buf()
4594 	evt->flags |= DWC3_EVENT_PENDING;  in dwc3_check_event_buf()
4597 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0),  in dwc3_check_event_buf()
4598 		    DWC3_GEVNTSIZ_INTMASK | DWC3_GEVNTSIZ_SIZE(evt->length));  in dwc3_check_event_buf()
4600 	amount = min(count, evt->length - evt->lpos);  in dwc3_check_event_buf()
4601 	memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);  in dwc3_check_event_buf()
4604 		memcpy(evt->cache, evt->buf, count - amount);  in dwc3_check_event_buf()
4606 	dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);  in dwc3_check_event_buf()
4620 	struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);  in dwc3_gadget_get_irq()
4627 	if (irq == -EPROBE_DEFER)  in dwc3_gadget_get_irq()
4634 	if (irq == -EPROBE_DEFER)  in dwc3_gadget_get_irq()
4651  * dwc3_gadget_init - initializes gadget related registers
4668 	dwc->irq_gadget = irq;  in dwc3_gadget_init()
4670 	dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,  in dwc3_gadget_init()
4671 					  sizeof(*dwc->ep0_trb) * 2,  in dwc3_gadget_init()
4672 					  &dwc->ep0_trb_addr, GFP_KERNEL);  in dwc3_gadget_init()
4673 	if (!dwc->ep0_trb) {  in dwc3_gadget_init()
4674 		dev_err(dwc->dev, "failed to allocate ep0 trb\n");  in dwc3_gadget_init()
4675 		ret = -ENOMEM;  in dwc3_gadget_init()
4679 	dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);  in dwc3_gadget_init()
4680 	if (!dwc->setup_buf) {  in dwc3_gadget_init()
4681 		ret = -ENOMEM;  in dwc3_gadget_init()
4685 	dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,  in dwc3_gadget_init()
4686 			&dwc->bounce_addr, GFP_KERNEL);  in dwc3_gadget_init()
4687 	if (!dwc->bounce) {  in dwc3_gadget_init()
4688 		ret = -ENOMEM;  in dwc3_gadget_init()
4692 	init_completion(&dwc->ep0_in_setup);  in dwc3_gadget_init()
4693 	dwc->gadget = kzalloc(sizeof(struct usb_gadget), GFP_KERNEL);  in dwc3_gadget_init()
4694 	if (!dwc->gadget) {  in dwc3_gadget_init()
4695 		ret = -ENOMEM;  in dwc3_gadget_init()
4700 	usb_initialize_gadget(dwc->dev, dwc->gadget, dwc_gadget_release);  in dwc3_gadget_init()
4701 	dev				= &dwc->gadget->dev;  in dwc3_gadget_init()
4702 	dev->platform_data		= dwc;  in dwc3_gadget_init()
4703 	dwc->gadget->ops		= &dwc3_gadget_ops;  in dwc3_gadget_init()
4704 	dwc->gadget->speed		= USB_SPEED_UNKNOWN;  in dwc3_gadget_init()
4705 	dwc->gadget->ssp_rate		= USB_SSP_GEN_UNKNOWN;  in dwc3_gadget_init()
4706 	dwc->gadget->sg_supported	= true;  in dwc3_gadget_init()
4707 	dwc->gadget->name		= "dwc3-gadget";  in dwc3_gadget_init()
4708 	dwc->gadget->lpm_capable	= !dwc->usb2_gadget_lpm_disable;  in dwc3_gadget_init()
4709 	dwc->gadget->wakeup_capable	= true;  in dwc3_gadget_init()
4725 	 * composite.c that we are USB 2.0 + LPM ECN.  in dwc3_gadget_init()
4728 	    !dwc->dis_metastability_quirk)  in dwc3_gadget_init()
4729 		dev_info(dwc->dev, "changing max_speed on rev %08x\n",  in dwc3_gadget_init()
4730 				dwc->revision);  in dwc3_gadget_init()
4732 	dwc->gadget->max_speed		= dwc->maximum_speed;  in dwc3_gadget_init()
4733 	dwc->gadget->max_ssp_rate	= dwc->max_ssp_rate;  in dwc3_gadget_init()
4740 	ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);  in dwc3_gadget_init()
4744 	ret = usb_add_gadget(dwc->gadget);  in dwc3_gadget_init()
4746 		dev_err(dwc->dev, "failed to add gadget\n");  in dwc3_gadget_init()
4750 	if (DWC3_IP_IS(DWC32) && dwc->maximum_speed == USB_SPEED_SUPER_PLUS)  in dwc3_gadget_init()
4751 		dwc3_gadget_set_ssp_rate(dwc->gadget, dwc->max_ssp_rate);  in dwc3_gadget_init()
4753 		dwc3_gadget_set_speed(dwc->gadget, dwc->maximum_speed);  in dwc3_gadget_init()
4756 	if (dwc->sys_wakeup)  in dwc3_gadget_init()
4757 		device_wakeup_disable(dwc->sysdev);  in dwc3_gadget_init()
4764 	usb_put_gadget(dwc->gadget);  in dwc3_gadget_init()
4765 	dwc->gadget = NULL;  in dwc3_gadget_init()
4767 	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,  in dwc3_gadget_init()
4768 			dwc->bounce_addr);  in dwc3_gadget_init()
4771 	kfree(dwc->setup_buf);  in dwc3_gadget_init()
4774 	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,  in dwc3_gadget_init()
4775 			dwc->ep0_trb, dwc->ep0_trb_addr);  in dwc3_gadget_init()
4781 /* -------------------------------------------------------------------------- */
4785 	if (!dwc->gadget)  in dwc3_gadget_exit()
4789 	usb_del_gadget(dwc->gadget);  in dwc3_gadget_exit()
4791 	usb_put_gadget(dwc->gadget);  in dwc3_gadget_exit()
4792 	dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,  in dwc3_gadget_exit()
4793 			  dwc->bounce_addr);  in dwc3_gadget_exit()
4794 	kfree(dwc->setup_buf);  in dwc3_gadget_exit()
4795 	dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,  in dwc3_gadget_exit()
4796 			  dwc->ep0_trb, dwc->ep0_trb_addr);  in dwc3_gadget_exit()
4808 	spin_lock_irqsave(&dwc->lock, flags);  in dwc3_gadget_suspend()
4809 	if (dwc->gadget_driver)  in dwc3_gadget_suspend()
4811 	spin_unlock_irqrestore(&dwc->lock, flags);  in dwc3_gadget_suspend()
4821 	if (dwc->softconnect)  in dwc3_gadget_suspend()
4829 	if (!dwc->gadget_driver || !dwc->softconnect)  in dwc3_gadget_resume()