xref: /openbmc/u-boot/drivers/usb/dwc3/gadget.c (revision 522e0354)
1 // SPDX-License-Identifier: GPL-2.0
2 /**
3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
4  *
5  * Copyright (C) 2015 Texas Instruments Incorporated - http://www.ti.com
6  *
7  * Authors: Felipe Balbi <balbi@ti.com>,
8  *	    Sebastian Andrzej Siewior <bigeasy@linutronix.de>
9  *
10  * Taken from Linux Kernel v3.19-rc1 (drivers/usb/dwc3/gadget.c) and ported
11  * to uboot.
12  *
13  * commit 8e74475b0e : usb: dwc3: gadget: use udc-core's reset notifier
14  */
15 
16 #include <common.h>
17 #include <malloc.h>
18 #include <asm/dma-mapping.h>
19 #include <linux/bug.h>
20 #include <linux/list.h>
21 
22 #include <linux/usb/ch9.h>
23 #include <linux/usb/gadget.h>
24 
25 #include "core.h"
26 #include "gadget.h"
27 #include "io.h"
28 
29 #include "linux-compat.h"
30 
31 /**
32  * dwc3_gadget_set_test_mode - Enables USB2 Test Modes
33  * @dwc: pointer to our context structure
34  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
35  *
36  * Caller should take care of locking. This function will
37  * return 0 on success or -EINVAL if wrong Test Selector
38  * is passed
39  */
dwc3_gadget_set_test_mode(struct dwc3 * dwc,int mode)40 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
41 {
42 	u32		reg;
43 
44 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
45 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
46 
47 	switch (mode) {
48 	case TEST_J:
49 	case TEST_K:
50 	case TEST_SE0_NAK:
51 	case TEST_PACKET:
52 	case TEST_FORCE_EN:
53 		reg |= mode << 1;
54 		break;
55 	default:
56 		return -EINVAL;
57 	}
58 
59 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
60 
61 	return 0;
62 }
63 
64 /**
65  * dwc3_gadget_get_link_state - Gets current state of USB Link
66  * @dwc: pointer to our context structure
67  *
68  * Caller should take care of locking. This function will
69  * return the link state on success (>= 0) or -ETIMEDOUT.
70  */
dwc3_gadget_get_link_state(struct dwc3 * dwc)71 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
72 {
73 	u32		reg;
74 
75 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
76 
77 	return DWC3_DSTS_USBLNKST(reg);
78 }
79 
80 /**
81  * dwc3_gadget_set_link_state - Sets USB Link to a particular State
82  * @dwc: pointer to our context structure
83  * @state: the state to put link into
84  *
85  * Caller should take care of locking. This function will
86  * return 0 on success or -ETIMEDOUT.
87  */
dwc3_gadget_set_link_state(struct dwc3 * dwc,enum dwc3_link_state state)88 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
89 {
90 	int		retries = 10000;
91 	u32		reg;
92 
93 	/*
94 	 * Wait until device controller is ready. Only applies to 1.94a and
95 	 * later RTL.
96 	 */
97 	if (dwc->revision >= DWC3_REVISION_194A) {
98 		while (--retries) {
99 			reg = dwc3_readl(dwc->regs, DWC3_DSTS);
100 			if (reg & DWC3_DSTS_DCNRD)
101 				udelay(5);
102 			else
103 				break;
104 		}
105 
106 		if (retries <= 0)
107 			return -ETIMEDOUT;
108 	}
109 
110 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
111 	reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
112 
113 	/* set requested state */
114 	reg |= DWC3_DCTL_ULSTCHNGREQ(state);
115 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
116 
117 	/*
118 	 * The following code is racy when called from dwc3_gadget_wakeup,
119 	 * and is not needed, at least on newer versions
120 	 */
121 	if (dwc->revision >= DWC3_REVISION_194A)
122 		return 0;
123 
124 	/* wait for a change in DSTS */
125 	retries = 10000;
126 	while (--retries) {
127 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
128 
129 		if (DWC3_DSTS_USBLNKST(reg) == state)
130 			return 0;
131 
132 		udelay(5);
133 	}
134 
135 	dev_vdbg(dwc->dev, "link state change request timed out\n");
136 
137 	return -ETIMEDOUT;
138 }
139 
140 /**
141  * dwc3_gadget_resize_tx_fifos - reallocate fifo spaces for current use-case
142  * @dwc: pointer to our context structure
143  *
144  * This function will a best effort FIFO allocation in order
145  * to improve FIFO usage and throughput, while still allowing
146  * us to enable as many endpoints as possible.
147  *
148  * Keep in mind that this operation will be highly dependent
149  * on the configured size for RAM1 - which contains TxFifo -,
150  * the amount of endpoints enabled on coreConsultant tool, and
151  * the width of the Master Bus.
152  *
153  * In the ideal world, we would always be able to satisfy the
154  * following equation:
155  *
156  * ((512 + 2 * MDWIDTH-Bytes) + (Number of IN Endpoints - 1) * \
157  * (3 * (1024 + MDWIDTH-Bytes) + MDWIDTH-Bytes)) / MDWIDTH-Bytes
158  *
159  * Unfortunately, due to many variables that's not always the case.
160  */
dwc3_gadget_resize_tx_fifos(struct dwc3 * dwc)161 int dwc3_gadget_resize_tx_fifos(struct dwc3 *dwc)
162 {
163 	int		last_fifo_depth = 0;
164 	int		fifo_size;
165 	int		mdwidth;
166 	int		num;
167 
168 	if (!dwc->needs_fifo_resize)
169 		return 0;
170 
171 	mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
172 
173 	/* MDWIDTH is represented in bits, we need it in bytes */
174 	mdwidth >>= 3;
175 
176 	/*
177 	 * FIXME For now we will only allocate 1 wMaxPacketSize space
178 	 * for each enabled endpoint, later patches will come to
179 	 * improve this algorithm so that we better use the internal
180 	 * FIFO space
181 	 */
182 	for (num = 0; num < dwc->num_in_eps; num++) {
183 		/* bit0 indicates direction; 1 means IN ep */
184 		struct dwc3_ep	*dep = dwc->eps[(num << 1) | 1];
185 		int		mult = 1;
186 		int		tmp;
187 
188 		if (!(dep->flags & DWC3_EP_ENABLED))
189 			continue;
190 
191 		if (usb_endpoint_xfer_bulk(dep->endpoint.desc)
192 				|| usb_endpoint_xfer_isoc(dep->endpoint.desc))
193 			mult = 3;
194 
195 		/*
196 		 * REVISIT: the following assumes we will always have enough
197 		 * space available on the FIFO RAM for all possible use cases.
198 		 * Make sure that's true somehow and change FIFO allocation
199 		 * accordingly.
200 		 *
201 		 * If we have Bulk or Isochronous endpoints, we want
202 		 * them to be able to be very, very fast. So we're giving
203 		 * those endpoints a fifo_size which is enough for 3 full
204 		 * packets
205 		 */
206 		tmp = mult * (dep->endpoint.maxpacket + mdwidth);
207 		tmp += mdwidth;
208 
209 		fifo_size = DIV_ROUND_UP(tmp, mdwidth);
210 
211 		fifo_size |= (last_fifo_depth << 16);
212 
213 		dev_vdbg(dwc->dev, "%s: Fifo Addr %04x Size %d\n",
214 				dep->name, last_fifo_depth, fifo_size & 0xffff);
215 
216 		dwc3_writel(dwc->regs, DWC3_GTXFIFOSIZ(num), fifo_size);
217 
218 		last_fifo_depth += (fifo_size & 0xffff);
219 	}
220 
221 	return 0;
222 }
223 
dwc3_gadget_giveback(struct dwc3_ep * dep,struct dwc3_request * req,int status)224 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
225 		int status)
226 {
227 	struct dwc3			*dwc = dep->dwc;
228 
229 	if (req->queued) {
230 		dep->busy_slot++;
231 		/*
232 		 * Skip LINK TRB. We can't use req->trb and check for
233 		 * DWC3_TRBCTL_LINK_TRB because it points the TRB we
234 		 * just completed (not the LINK TRB).
235 		 */
236 		if (((dep->busy_slot & DWC3_TRB_MASK) ==
237 			DWC3_TRB_NUM- 1) &&
238 			usb_endpoint_xfer_isoc(dep->endpoint.desc))
239 			dep->busy_slot++;
240 		req->queued = false;
241 	}
242 
243 	list_del(&req->list);
244 	req->trb = NULL;
245 	dwc3_flush_cache((uintptr_t)req->request.dma, req->request.length);
246 
247 	if (req->request.status == -EINPROGRESS)
248 		req->request.status = status;
249 
250 	if (dwc->ep0_bounced && dep->number == 0)
251 		dwc->ep0_bounced = false;
252 	else
253 		usb_gadget_unmap_request(&dwc->gadget, &req->request,
254 				req->direction);
255 
256 	dev_dbg(dwc->dev, "request %p from %s completed %d/%d ===> %d\n",
257 			req, dep->name, req->request.actual,
258 			req->request.length, status);
259 
260 	spin_unlock(&dwc->lock);
261 	usb_gadget_giveback_request(&dep->endpoint, &req->request);
262 	spin_lock(&dwc->lock);
263 }
264 
dwc3_send_gadget_generic_command(struct dwc3 * dwc,unsigned cmd,u32 param)265 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
266 {
267 	u32		timeout = 500;
268 	u32		reg;
269 
270 	dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
271 	dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
272 
273 	do {
274 		reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
275 		if (!(reg & DWC3_DGCMD_CMDACT)) {
276 			dev_vdbg(dwc->dev, "Command Complete --> %d\n",
277 					DWC3_DGCMD_STATUS(reg));
278 			return 0;
279 		}
280 
281 		/*
282 		 * We can't sleep here, because it's also called from
283 		 * interrupt context.
284 		 */
285 		timeout--;
286 		if (!timeout)
287 			return -ETIMEDOUT;
288 		udelay(1);
289 	} while (1);
290 }
291 
dwc3_send_gadget_ep_cmd(struct dwc3 * dwc,unsigned ep,unsigned cmd,struct dwc3_gadget_ep_cmd_params * params)292 int dwc3_send_gadget_ep_cmd(struct dwc3 *dwc, unsigned ep,
293 		unsigned cmd, struct dwc3_gadget_ep_cmd_params *params)
294 {
295 	u32			timeout = 500;
296 	u32			reg;
297 
298 	dwc3_writel(dwc->regs, DWC3_DEPCMDPAR0(ep), params->param0);
299 	dwc3_writel(dwc->regs, DWC3_DEPCMDPAR1(ep), params->param1);
300 	dwc3_writel(dwc->regs, DWC3_DEPCMDPAR2(ep), params->param2);
301 
302 	dwc3_writel(dwc->regs, DWC3_DEPCMD(ep), cmd | DWC3_DEPCMD_CMDACT);
303 	do {
304 		reg = dwc3_readl(dwc->regs, DWC3_DEPCMD(ep));
305 		if (!(reg & DWC3_DEPCMD_CMDACT)) {
306 			dev_vdbg(dwc->dev, "Command Complete --> %d\n",
307 					DWC3_DEPCMD_STATUS(reg));
308 			return 0;
309 		}
310 
311 		/*
312 		 * We can't sleep here, because it is also called from
313 		 * interrupt context.
314 		 */
315 		timeout--;
316 		if (!timeout)
317 			return -ETIMEDOUT;
318 
319 		udelay(1);
320 	} while (1);
321 }
322 
dwc3_trb_dma_offset(struct dwc3_ep * dep,struct dwc3_trb * trb)323 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
324 		struct dwc3_trb *trb)
325 {
326 	u32		offset = (char *) trb - (char *) dep->trb_pool;
327 
328 	return dep->trb_pool_dma + offset;
329 }
330 
dwc3_alloc_trb_pool(struct dwc3_ep * dep)331 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
332 {
333 	if (dep->trb_pool)
334 		return 0;
335 
336 	if (dep->number == 0 || dep->number == 1)
337 		return 0;
338 
339 	dep->trb_pool = dma_alloc_coherent(sizeof(struct dwc3_trb) *
340 					   DWC3_TRB_NUM,
341 					   (unsigned long *)&dep->trb_pool_dma);
342 	if (!dep->trb_pool) {
343 		dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
344 				dep->name);
345 		return -ENOMEM;
346 	}
347 
348 	return 0;
349 }
350 
dwc3_free_trb_pool(struct dwc3_ep * dep)351 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
352 {
353 	dma_free_coherent(dep->trb_pool);
354 
355 	dep->trb_pool = NULL;
356 	dep->trb_pool_dma = 0;
357 }
358 
dwc3_gadget_start_config(struct dwc3 * dwc,struct dwc3_ep * dep)359 static int dwc3_gadget_start_config(struct dwc3 *dwc, struct dwc3_ep *dep)
360 {
361 	struct dwc3_gadget_ep_cmd_params params;
362 	u32			cmd;
363 
364 	memset(&params, 0x00, sizeof(params));
365 
366 	if (dep->number != 1) {
367 		cmd = DWC3_DEPCMD_DEPSTARTCFG;
368 		/* XferRscIdx == 0 for ep0 and 2 for the remaining */
369 		if (dep->number > 1) {
370 			if (dwc->start_config_issued)
371 				return 0;
372 			dwc->start_config_issued = true;
373 			cmd |= DWC3_DEPCMD_PARAM(2);
374 		}
375 
376 		return dwc3_send_gadget_ep_cmd(dwc, 0, cmd, &params);
377 	}
378 
379 	return 0;
380 }
381 
dwc3_gadget_set_ep_config(struct dwc3 * dwc,struct dwc3_ep * dep,const struct usb_endpoint_descriptor * desc,const struct usb_ss_ep_comp_descriptor * comp_desc,bool ignore,bool restore)382 static int dwc3_gadget_set_ep_config(struct dwc3 *dwc, struct dwc3_ep *dep,
383 		const struct usb_endpoint_descriptor *desc,
384 		const struct usb_ss_ep_comp_descriptor *comp_desc,
385 		bool ignore, bool restore)
386 {
387 	struct dwc3_gadget_ep_cmd_params params;
388 
389 	memset(&params, 0x00, sizeof(params));
390 
391 	params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
392 		| DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
393 
394 	/* Burst size is only needed in SuperSpeed mode */
395 	if (dwc->gadget.speed == USB_SPEED_SUPER) {
396 		u32 burst = dep->endpoint.maxburst - 1;
397 
398 		params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst);
399 	}
400 
401 	if (ignore)
402 		params.param0 |= DWC3_DEPCFG_IGN_SEQ_NUM;
403 
404 	if (restore) {
405 		params.param0 |= DWC3_DEPCFG_ACTION_RESTORE;
406 		params.param2 |= dep->saved_state;
407 	}
408 
409 	params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN
410 		| DWC3_DEPCFG_XFER_NOT_READY_EN;
411 
412 	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
413 		params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
414 			| DWC3_DEPCFG_STREAM_EVENT_EN;
415 		dep->stream_capable = true;
416 	}
417 
418 	if (!usb_endpoint_xfer_control(desc))
419 		params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
420 
421 	/*
422 	 * We are doing 1:1 mapping for endpoints, meaning
423 	 * Physical Endpoints 2 maps to Logical Endpoint 2 and
424 	 * so on. We consider the direction bit as part of the physical
425 	 * endpoint number. So USB endpoint 0x81 is 0x03.
426 	 */
427 	params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
428 
429 	/*
430 	 * We must use the lower 16 TX FIFOs even though
431 	 * HW might have more
432 	 */
433 	if (dep->direction)
434 		params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
435 
436 	if (desc->bInterval) {
437 		params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
438 		dep->interval = 1 << (desc->bInterval - 1);
439 	}
440 
441 	return dwc3_send_gadget_ep_cmd(dwc, dep->number,
442 			DWC3_DEPCMD_SETEPCONFIG, &params);
443 }
444 
dwc3_gadget_set_xfer_resource(struct dwc3 * dwc,struct dwc3_ep * dep)445 static int dwc3_gadget_set_xfer_resource(struct dwc3 *dwc, struct dwc3_ep *dep)
446 {
447 	struct dwc3_gadget_ep_cmd_params params;
448 
449 	memset(&params, 0x00, sizeof(params));
450 
451 	params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
452 
453 	return dwc3_send_gadget_ep_cmd(dwc, dep->number,
454 			DWC3_DEPCMD_SETTRANSFRESOURCE, &params);
455 }
456 
457 /**
458  * __dwc3_gadget_ep_enable - Initializes a HW endpoint
459  * @dep: endpoint to be initialized
460  * @desc: USB Endpoint Descriptor
461  *
462  * Caller should take care of locking
463  */
__dwc3_gadget_ep_enable(struct dwc3_ep * dep,const struct usb_endpoint_descriptor * desc,const struct usb_ss_ep_comp_descriptor * comp_desc,bool ignore,bool restore)464 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep,
465 		const struct usb_endpoint_descriptor *desc,
466 		const struct usb_ss_ep_comp_descriptor *comp_desc,
467 		bool ignore, bool restore)
468 {
469 	struct dwc3		*dwc = dep->dwc;
470 	u32			reg;
471 	int			ret;
472 
473 	dev_vdbg(dwc->dev, "Enabling %s\n", dep->name);
474 
475 	if (!(dep->flags & DWC3_EP_ENABLED)) {
476 		ret = dwc3_gadget_start_config(dwc, dep);
477 		if (ret)
478 			return ret;
479 	}
480 
481 	ret = dwc3_gadget_set_ep_config(dwc, dep, desc, comp_desc, ignore,
482 			restore);
483 	if (ret)
484 		return ret;
485 
486 	if (!(dep->flags & DWC3_EP_ENABLED)) {
487 		struct dwc3_trb	*trb_st_hw;
488 		struct dwc3_trb	*trb_link;
489 
490 		ret = dwc3_gadget_set_xfer_resource(dwc, dep);
491 		if (ret)
492 			return ret;
493 
494 		dep->endpoint.desc = desc;
495 		dep->comp_desc = comp_desc;
496 		dep->type = usb_endpoint_type(desc);
497 		dep->flags |= DWC3_EP_ENABLED;
498 
499 		reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
500 		reg |= DWC3_DALEPENA_EP(dep->number);
501 		dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
502 
503 		if (!usb_endpoint_xfer_isoc(desc))
504 			return 0;
505 
506 		/* Link TRB for ISOC. The HWO bit is never reset */
507 		trb_st_hw = &dep->trb_pool[0];
508 
509 		trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
510 		memset(trb_link, 0, sizeof(*trb_link));
511 
512 		trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
513 		trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
514 		trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
515 		trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
516 	}
517 
518 	return 0;
519 }
520 
521 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force);
dwc3_remove_requests(struct dwc3 * dwc,struct dwc3_ep * dep)522 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
523 {
524 	struct dwc3_request		*req;
525 
526 	if (!list_empty(&dep->req_queued)) {
527 		dwc3_stop_active_transfer(dwc, dep->number, true);
528 
529 		/* - giveback all requests to gadget driver */
530 		while (!list_empty(&dep->req_queued)) {
531 			req = next_request(&dep->req_queued);
532 
533 			dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
534 		}
535 	}
536 
537 	while (!list_empty(&dep->request_list)) {
538 		req = next_request(&dep->request_list);
539 
540 		dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
541 	}
542 }
543 
544 /**
545  * __dwc3_gadget_ep_disable - Disables a HW endpoint
546  * @dep: the endpoint to disable
547  *
548  * This function also removes requests which are currently processed ny the
549  * hardware and those which are not yet scheduled.
550  * Caller should take care of locking.
551  */
__dwc3_gadget_ep_disable(struct dwc3_ep * dep)552 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
553 {
554 	struct dwc3		*dwc = dep->dwc;
555 	u32			reg;
556 
557 	dwc3_remove_requests(dwc, dep);
558 
559 	/* make sure HW endpoint isn't stalled */
560 	if (dep->flags & DWC3_EP_STALL)
561 		__dwc3_gadget_ep_set_halt(dep, 0, false);
562 
563 	reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
564 	reg &= ~DWC3_DALEPENA_EP(dep->number);
565 	dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
566 
567 	dep->stream_capable = false;
568 	dep->endpoint.desc = NULL;
569 	dep->comp_desc = NULL;
570 	dep->type = 0;
571 	dep->flags = 0;
572 
573 	return 0;
574 }
575 
576 /* -------------------------------------------------------------------------- */
577 
dwc3_gadget_ep0_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)578 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
579 		const struct usb_endpoint_descriptor *desc)
580 {
581 	return -EINVAL;
582 }
583 
dwc3_gadget_ep0_disable(struct usb_ep * ep)584 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
585 {
586 	return -EINVAL;
587 }
588 
589 /* -------------------------------------------------------------------------- */
590 
dwc3_gadget_ep_enable(struct usb_ep * ep,const struct usb_endpoint_descriptor * desc)591 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
592 		const struct usb_endpoint_descriptor *desc)
593 {
594 	struct dwc3_ep			*dep;
595 	unsigned long			flags;
596 	int				ret;
597 
598 	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
599 		pr_debug("dwc3: invalid parameters\n");
600 		return -EINVAL;
601 	}
602 
603 	if (!desc->wMaxPacketSize) {
604 		pr_debug("dwc3: missing wMaxPacketSize\n");
605 		return -EINVAL;
606 	}
607 
608 	dep = to_dwc3_ep(ep);
609 
610 	if (dep->flags & DWC3_EP_ENABLED) {
611 		WARN(true, "%s is already enabled\n",
612 				dep->name);
613 		return 0;
614 	}
615 
616 	switch (usb_endpoint_type(desc)) {
617 	case USB_ENDPOINT_XFER_CONTROL:
618 		strlcat(dep->name, "-control", sizeof(dep->name));
619 		break;
620 	case USB_ENDPOINT_XFER_ISOC:
621 		strlcat(dep->name, "-isoc", sizeof(dep->name));
622 		break;
623 	case USB_ENDPOINT_XFER_BULK:
624 		strlcat(dep->name, "-bulk", sizeof(dep->name));
625 		break;
626 	case USB_ENDPOINT_XFER_INT:
627 		strlcat(dep->name, "-int", sizeof(dep->name));
628 		break;
629 	default:
630 		dev_err(dwc->dev, "invalid endpoint transfer type\n");
631 	}
632 
633 	spin_lock_irqsave(&dwc->lock, flags);
634 	ret = __dwc3_gadget_ep_enable(dep, desc, ep->comp_desc, false, false);
635 	spin_unlock_irqrestore(&dwc->lock, flags);
636 
637 	return ret;
638 }
639 
dwc3_gadget_ep_disable(struct usb_ep * ep)640 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
641 {
642 	struct dwc3_ep			*dep;
643 	unsigned long			flags;
644 	int				ret;
645 
646 	if (!ep) {
647 		pr_debug("dwc3: invalid parameters\n");
648 		return -EINVAL;
649 	}
650 
651 	dep = to_dwc3_ep(ep);
652 
653 	if (!(dep->flags & DWC3_EP_ENABLED)) {
654 		WARN(true, "%s is already disabled\n",
655 				dep->name);
656 		return 0;
657 	}
658 
659 	snprintf(dep->name, sizeof(dep->name), "ep%d%s",
660 			dep->number >> 1,
661 			(dep->number & 1) ? "in" : "out");
662 
663 	spin_lock_irqsave(&dwc->lock, flags);
664 	ret = __dwc3_gadget_ep_disable(dep);
665 	spin_unlock_irqrestore(&dwc->lock, flags);
666 
667 	return ret;
668 }
669 
dwc3_gadget_ep_alloc_request(struct usb_ep * ep,gfp_t gfp_flags)670 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
671 	gfp_t gfp_flags)
672 {
673 	struct dwc3_request		*req;
674 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
675 
676 	req = kzalloc(sizeof(*req), gfp_flags);
677 	if (!req)
678 		return NULL;
679 
680 	req->epnum	= dep->number;
681 	req->dep	= dep;
682 
683 	return &req->request;
684 }
685 
dwc3_gadget_ep_free_request(struct usb_ep * ep,struct usb_request * request)686 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
687 		struct usb_request *request)
688 {
689 	struct dwc3_request		*req = to_dwc3_request(request);
690 
691 	kfree(req);
692 }
693 
694 /**
695  * dwc3_prepare_one_trb - setup one TRB from one request
696  * @dep: endpoint for which this request is prepared
697  * @req: dwc3_request pointer
698  */
dwc3_prepare_one_trb(struct dwc3_ep * dep,struct dwc3_request * req,dma_addr_t dma,unsigned length,unsigned last,unsigned chain,unsigned node)699 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
700 		struct dwc3_request *req, dma_addr_t dma,
701 		unsigned length, unsigned last, unsigned chain, unsigned node)
702 {
703 	struct dwc3_trb		*trb;
704 
705 	dev_vdbg(dwc->dev, "%s: req %p dma %08llx length %d%s%s\n",
706 			dep->name, req, (unsigned long long) dma,
707 			length, last ? " last" : "",
708 			chain ? " chain" : "");
709 
710 
711 	trb = &dep->trb_pool[dep->free_slot & DWC3_TRB_MASK];
712 
713 	if (!req->trb) {
714 		dwc3_gadget_move_request_queued(req);
715 		req->trb = trb;
716 		req->trb_dma = dwc3_trb_dma_offset(dep, trb);
717 		req->start_slot = dep->free_slot & DWC3_TRB_MASK;
718 	}
719 
720 	dep->free_slot++;
721 	/* Skip the LINK-TRB on ISOC */
722 	if (((dep->free_slot & DWC3_TRB_MASK) == DWC3_TRB_NUM - 1) &&
723 			usb_endpoint_xfer_isoc(dep->endpoint.desc))
724 		dep->free_slot++;
725 
726 	trb->size = DWC3_TRB_SIZE_LENGTH(length);
727 	trb->bpl = lower_32_bits(dma);
728 	trb->bph = upper_32_bits(dma);
729 
730 	switch (usb_endpoint_type(dep->endpoint.desc)) {
731 	case USB_ENDPOINT_XFER_CONTROL:
732 		trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
733 		break;
734 
735 	case USB_ENDPOINT_XFER_ISOC:
736 		if (!node)
737 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
738 		else
739 			trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
740 		break;
741 
742 	case USB_ENDPOINT_XFER_BULK:
743 	case USB_ENDPOINT_XFER_INT:
744 		trb->ctrl = DWC3_TRBCTL_NORMAL;
745 		break;
746 	default:
747 		/*
748 		 * This is only possible with faulty memory because we
749 		 * checked it already :)
750 		 */
751 		BUG();
752 	}
753 
754 	if (!req->request.no_interrupt && !chain)
755 		trb->ctrl |= DWC3_TRB_CTRL_IOC;
756 
757 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
758 		trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
759 		trb->ctrl |= DWC3_TRB_CTRL_CSP;
760 	} else if (last) {
761 		trb->ctrl |= DWC3_TRB_CTRL_LST;
762 	}
763 
764 	if (chain)
765 		trb->ctrl |= DWC3_TRB_CTRL_CHN;
766 
767 	if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
768 		trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(req->request.stream_id);
769 
770 	trb->ctrl |= DWC3_TRB_CTRL_HWO;
771 
772 	dwc3_flush_cache((uintptr_t)dma, length);
773 	dwc3_flush_cache((uintptr_t)trb, sizeof(*trb));
774 }
775 
776 /*
777  * dwc3_prepare_trbs - setup TRBs from requests
778  * @dep: endpoint for which requests are being prepared
779  * @starting: true if the endpoint is idle and no requests are queued.
780  *
781  * The function goes through the requests list and sets up TRBs for the
782  * transfers. The function returns once there are no more TRBs available or
783  * it runs out of requests.
784  */
dwc3_prepare_trbs(struct dwc3_ep * dep,bool starting)785 static void dwc3_prepare_trbs(struct dwc3_ep *dep, bool starting)
786 {
787 	struct dwc3_request	*req, *n;
788 	u32			trbs_left;
789 	u32			max;
790 
791 	BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
792 
793 	/* the first request must not be queued */
794 	trbs_left = (dep->busy_slot - dep->free_slot) & DWC3_TRB_MASK;
795 
796 	/* Can't wrap around on a non-isoc EP since there's no link TRB */
797 	if (!usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
798 		max = DWC3_TRB_NUM - (dep->free_slot & DWC3_TRB_MASK);
799 		if (trbs_left > max)
800 			trbs_left = max;
801 	}
802 
803 	/*
804 	 * If busy & slot are equal than it is either full or empty. If we are
805 	 * starting to process requests then we are empty. Otherwise we are
806 	 * full and don't do anything
807 	 */
808 	if (!trbs_left) {
809 		if (!starting)
810 			return;
811 		trbs_left = DWC3_TRB_NUM;
812 		/*
813 		 * In case we start from scratch, we queue the ISOC requests
814 		 * starting from slot 1. This is done because we use ring
815 		 * buffer and have no LST bit to stop us. Instead, we place
816 		 * IOC bit every TRB_NUM/4. We try to avoid having an interrupt
817 		 * after the first request so we start at slot 1 and have
818 		 * 7 requests proceed before we hit the first IOC.
819 		 * Other transfer types don't use the ring buffer and are
820 		 * processed from the first TRB until the last one. Since we
821 		 * don't wrap around we have to start at the beginning.
822 		 */
823 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
824 			dep->busy_slot = 1;
825 			dep->free_slot = 1;
826 		} else {
827 			dep->busy_slot = 0;
828 			dep->free_slot = 0;
829 		}
830 	}
831 
832 	/* The last TRB is a link TRB, not used for xfer */
833 	if ((trbs_left <= 1) && usb_endpoint_xfer_isoc(dep->endpoint.desc))
834 		return;
835 
836 	list_for_each_entry_safe(req, n, &dep->request_list, list) {
837 		unsigned	length;
838 		dma_addr_t	dma;
839 
840 		dma = req->request.dma;
841 		length = req->request.length;
842 
843 		dwc3_prepare_one_trb(dep, req, dma, length,
844 				     true, false, 0);
845 
846 		break;
847 	}
848 }
849 
__dwc3_gadget_kick_transfer(struct dwc3_ep * dep,u16 cmd_param,int start_new)850 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep, u16 cmd_param,
851 		int start_new)
852 {
853 	struct dwc3_gadget_ep_cmd_params params;
854 	struct dwc3_request		*req;
855 	struct dwc3			*dwc = dep->dwc;
856 	int				ret;
857 	u32				cmd;
858 
859 	if (start_new && (dep->flags & DWC3_EP_BUSY)) {
860 		dev_vdbg(dwc->dev, "%s: endpoint busy\n", dep->name);
861 		return -EBUSY;
862 	}
863 	dep->flags &= ~DWC3_EP_PENDING_REQUEST;
864 
865 	/*
866 	 * If we are getting here after a short-out-packet we don't enqueue any
867 	 * new requests as we try to set the IOC bit only on the last request.
868 	 */
869 	if (start_new) {
870 		if (list_empty(&dep->req_queued))
871 			dwc3_prepare_trbs(dep, start_new);
872 
873 		/* req points to the first request which will be sent */
874 		req = next_request(&dep->req_queued);
875 	} else {
876 		dwc3_prepare_trbs(dep, start_new);
877 
878 		/*
879 		 * req points to the first request where HWO changed from 0 to 1
880 		 */
881 		req = next_request(&dep->req_queued);
882 	}
883 	if (!req) {
884 		dep->flags |= DWC3_EP_PENDING_REQUEST;
885 		return 0;
886 	}
887 
888 	memset(&params, 0, sizeof(params));
889 
890 	if (start_new) {
891 		params.param0 = upper_32_bits(req->trb_dma);
892 		params.param1 = lower_32_bits(req->trb_dma);
893 		cmd = DWC3_DEPCMD_STARTTRANSFER;
894 	} else {
895 		cmd = DWC3_DEPCMD_UPDATETRANSFER;
896 	}
897 
898 	cmd |= DWC3_DEPCMD_PARAM(cmd_param);
899 	ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
900 	if (ret < 0) {
901 		dev_dbg(dwc->dev, "failed to send STARTTRANSFER command\n");
902 
903 		/*
904 		 * FIXME we need to iterate over the list of requests
905 		 * here and stop, unmap, free and del each of the linked
906 		 * requests instead of what we do now.
907 		 */
908 		usb_gadget_unmap_request(&dwc->gadget, &req->request,
909 				req->direction);
910 		list_del(&req->list);
911 		return ret;
912 	}
913 
914 	dep->flags |= DWC3_EP_BUSY;
915 
916 	if (start_new) {
917 		dep->resource_index = dwc3_gadget_ep_get_transfer_index(dwc,
918 				dep->number);
919 		WARN_ON_ONCE(!dep->resource_index);
920 	}
921 
922 	return 0;
923 }
924 
__dwc3_gadget_start_isoc(struct dwc3 * dwc,struct dwc3_ep * dep,u32 cur_uf)925 static void __dwc3_gadget_start_isoc(struct dwc3 *dwc,
926 		struct dwc3_ep *dep, u32 cur_uf)
927 {
928 	u32 uf;
929 
930 	if (list_empty(&dep->request_list)) {
931 		dev_vdbg(dwc->dev, "ISOC ep %s run out for requests.\n",
932 			dep->name);
933 		dep->flags |= DWC3_EP_PENDING_REQUEST;
934 		return;
935 	}
936 
937 	/* 4 micro frames in the future */
938 	uf = cur_uf + dep->interval * 4;
939 
940 	__dwc3_gadget_kick_transfer(dep, uf, 1);
941 }
942 
dwc3_gadget_start_isoc(struct dwc3 * dwc,struct dwc3_ep * dep,const struct dwc3_event_depevt * event)943 static void dwc3_gadget_start_isoc(struct dwc3 *dwc,
944 		struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
945 {
946 	u32 cur_uf, mask;
947 
948 	mask = ~(dep->interval - 1);
949 	cur_uf = event->parameters & mask;
950 
951 	__dwc3_gadget_start_isoc(dwc, dep, cur_uf);
952 }
953 
__dwc3_gadget_ep_queue(struct dwc3_ep * dep,struct dwc3_request * req)954 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
955 {
956 	struct dwc3		*dwc = dep->dwc;
957 	int			ret;
958 
959 	req->request.actual	= 0;
960 	req->request.status	= -EINPROGRESS;
961 	req->direction		= dep->direction;
962 	req->epnum		= dep->number;
963 
964 	/*
965 	 * DWC3 hangs on OUT requests smaller than maxpacket size,
966 	 * so HACK the request length
967 	 */
968 	if (dep->direction == 0 &&
969 	    req->request.length < dep->endpoint.maxpacket)
970 		req->request.length = dep->endpoint.maxpacket;
971 
972 	/*
973 	 * We only add to our list of requests now and
974 	 * start consuming the list once we get XferNotReady
975 	 * IRQ.
976 	 *
977 	 * That way, we avoid doing anything that we don't need
978 	 * to do now and defer it until the point we receive a
979 	 * particular token from the Host side.
980 	 *
981 	 * This will also avoid Host cancelling URBs due to too
982 	 * many NAKs.
983 	 */
984 	ret = usb_gadget_map_request(&dwc->gadget, &req->request,
985 			dep->direction);
986 	if (ret)
987 		return ret;
988 
989 	list_add_tail(&req->list, &dep->request_list);
990 
991 	/*
992 	 * There are a few special cases:
993 	 *
994 	 * 1. XferNotReady with empty list of requests. We need to kick the
995 	 *    transfer here in that situation, otherwise we will be NAKing
996 	 *    forever. If we get XferNotReady before gadget driver has a
997 	 *    chance to queue a request, we will ACK the IRQ but won't be
998 	 *    able to receive the data until the next request is queued.
999 	 *    The following code is handling exactly that.
1000 	 *
1001 	 */
1002 	if (dep->flags & DWC3_EP_PENDING_REQUEST) {
1003 		/*
1004 		 * If xfernotready is already elapsed and it is a case
1005 		 * of isoc transfer, then issue END TRANSFER, so that
1006 		 * you can receive xfernotready again and can have
1007 		 * notion of current microframe.
1008 		 */
1009 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1010 			if (list_empty(&dep->req_queued)) {
1011 				dwc3_stop_active_transfer(dwc, dep->number, true);
1012 				dep->flags = DWC3_EP_ENABLED;
1013 			}
1014 			return 0;
1015 		}
1016 
1017 		ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1018 		if (ret && ret != -EBUSY)
1019 			dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1020 					dep->name);
1021 		return ret;
1022 	}
1023 
1024 	/*
1025 	 * 2. XferInProgress on Isoc EP with an active transfer. We need to
1026 	 *    kick the transfer here after queuing a request, otherwise the
1027 	 *    core may not see the modified TRB(s).
1028 	 */
1029 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1030 			(dep->flags & DWC3_EP_BUSY) &&
1031 			!(dep->flags & DWC3_EP_MISSED_ISOC)) {
1032 		WARN_ON_ONCE(!dep->resource_index);
1033 		ret = __dwc3_gadget_kick_transfer(dep, dep->resource_index,
1034 				false);
1035 		if (ret && ret != -EBUSY)
1036 			dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1037 					dep->name);
1038 		return ret;
1039 	}
1040 
1041 	/*
1042 	 * 4. Stream Capable Bulk Endpoints. We need to start the transfer
1043 	 * right away, otherwise host will not know we have streams to be
1044 	 * handled.
1045 	 */
1046 	if (dep->stream_capable) {
1047 		int	ret;
1048 
1049 		ret = __dwc3_gadget_kick_transfer(dep, 0, true);
1050 		if (ret && ret != -EBUSY) {
1051 			dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1052 					dep->name);
1053 		}
1054 	}
1055 
1056 	return 0;
1057 }
1058 
dwc3_gadget_ep_queue(struct usb_ep * ep,struct usb_request * request,gfp_t gfp_flags)1059 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1060 	gfp_t gfp_flags)
1061 {
1062 	struct dwc3_request		*req = to_dwc3_request(request);
1063 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1064 
1065 	unsigned long			flags;
1066 
1067 	int				ret;
1068 
1069 	spin_lock_irqsave(&dwc->lock, flags);
1070 	if (!dep->endpoint.desc) {
1071 		dev_dbg(dwc->dev, "trying to queue request %p to disabled %s\n",
1072 				request, ep->name);
1073 		ret = -ESHUTDOWN;
1074 		goto out;
1075 	}
1076 
1077 	if (req->dep != dep) {
1078 		WARN(true, "request %p belongs to '%s'\n",
1079 				request, req->dep->name);
1080 		ret = -EINVAL;
1081 		goto out;
1082 	}
1083 
1084 	dev_vdbg(dwc->dev, "queing request %p to %s length %d\n",
1085 			request, ep->name, request->length);
1086 
1087 	ret = __dwc3_gadget_ep_queue(dep, req);
1088 
1089 out:
1090 	spin_unlock_irqrestore(&dwc->lock, flags);
1091 
1092 	return ret;
1093 }
1094 
dwc3_gadget_ep_dequeue(struct usb_ep * ep,struct usb_request * request)1095 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1096 		struct usb_request *request)
1097 {
1098 	struct dwc3_request		*req = to_dwc3_request(request);
1099 	struct dwc3_request		*r = NULL;
1100 
1101 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1102 	struct dwc3			*dwc = dep->dwc;
1103 
1104 	unsigned long			flags;
1105 	int				ret = 0;
1106 
1107 	spin_lock_irqsave(&dwc->lock, flags);
1108 
1109 	list_for_each_entry(r, &dep->request_list, list) {
1110 		if (r == req)
1111 			break;
1112 	}
1113 
1114 	if (r != req) {
1115 		list_for_each_entry(r, &dep->req_queued, list) {
1116 			if (r == req)
1117 				break;
1118 		}
1119 		if (r == req) {
1120 			/* wait until it is processed */
1121 			dwc3_stop_active_transfer(dwc, dep->number, true);
1122 			goto out1;
1123 		}
1124 		dev_err(dwc->dev, "request %p was not queued to %s\n",
1125 				request, ep->name);
1126 		ret = -EINVAL;
1127 		goto out0;
1128 	}
1129 
1130 out1:
1131 	/* giveback the request */
1132 	dwc3_gadget_giveback(dep, req, -ECONNRESET);
1133 
1134 out0:
1135 	spin_unlock_irqrestore(&dwc->lock, flags);
1136 
1137 	return ret;
1138 }
1139 
__dwc3_gadget_ep_set_halt(struct dwc3_ep * dep,int value,int protocol)1140 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1141 {
1142 	struct dwc3_gadget_ep_cmd_params	params;
1143 	struct dwc3				*dwc = dep->dwc;
1144 	int					ret;
1145 
1146 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1147 		dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1148 		return -EINVAL;
1149 	}
1150 
1151 	memset(&params, 0x00, sizeof(params));
1152 
1153 	if (value) {
1154 		if (!protocol && ((dep->direction && dep->flags & DWC3_EP_BUSY) ||
1155 				(!list_empty(&dep->req_queued) ||
1156 				 !list_empty(&dep->request_list)))) {
1157 			dev_dbg(dwc->dev, "%s: pending request, cannot halt\n",
1158 					dep->name);
1159 			return -EAGAIN;
1160 		}
1161 
1162 		ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1163 			DWC3_DEPCMD_SETSTALL, &params);
1164 		if (ret)
1165 			dev_err(dwc->dev, "failed to set STALL on %s\n",
1166 					dep->name);
1167 		else
1168 			dep->flags |= DWC3_EP_STALL;
1169 	} else {
1170 		ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
1171 			DWC3_DEPCMD_CLEARSTALL, &params);
1172 		if (ret)
1173 			dev_err(dwc->dev, "failed to clear STALL on %s\n",
1174 					dep->name);
1175 		else
1176 			dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1177 	}
1178 
1179 	return ret;
1180 }
1181 
dwc3_gadget_ep_set_halt(struct usb_ep * ep,int value)1182 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1183 {
1184 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1185 
1186 	unsigned long			flags;
1187 
1188 	int				ret;
1189 
1190 	spin_lock_irqsave(&dwc->lock, flags);
1191 	ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1192 	spin_unlock_irqrestore(&dwc->lock, flags);
1193 
1194 	return ret;
1195 }
1196 
dwc3_gadget_ep_set_wedge(struct usb_ep * ep)1197 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1198 {
1199 	struct dwc3_ep			*dep = to_dwc3_ep(ep);
1200 	unsigned long			flags;
1201 	int				ret;
1202 
1203 	spin_lock_irqsave(&dwc->lock, flags);
1204 	dep->flags |= DWC3_EP_WEDGE;
1205 
1206 	if (dep->number == 0 || dep->number == 1)
1207 		ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1208 	else
1209 		ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1210 	spin_unlock_irqrestore(&dwc->lock, flags);
1211 
1212 	return ret;
1213 }
1214 
1215 /* -------------------------------------------------------------------------- */
1216 
1217 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1218 	.bLength	= USB_DT_ENDPOINT_SIZE,
1219 	.bDescriptorType = USB_DT_ENDPOINT,
1220 	.bmAttributes	= USB_ENDPOINT_XFER_CONTROL,
1221 };
1222 
1223 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1224 	.enable		= dwc3_gadget_ep0_enable,
1225 	.disable	= dwc3_gadget_ep0_disable,
1226 	.alloc_request	= dwc3_gadget_ep_alloc_request,
1227 	.free_request	= dwc3_gadget_ep_free_request,
1228 	.queue		= dwc3_gadget_ep0_queue,
1229 	.dequeue	= dwc3_gadget_ep_dequeue,
1230 	.set_halt	= dwc3_gadget_ep0_set_halt,
1231 	.set_wedge	= dwc3_gadget_ep_set_wedge,
1232 };
1233 
1234 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1235 	.enable		= dwc3_gadget_ep_enable,
1236 	.disable	= dwc3_gadget_ep_disable,
1237 	.alloc_request	= dwc3_gadget_ep_alloc_request,
1238 	.free_request	= dwc3_gadget_ep_free_request,
1239 	.queue		= dwc3_gadget_ep_queue,
1240 	.dequeue	= dwc3_gadget_ep_dequeue,
1241 	.set_halt	= dwc3_gadget_ep_set_halt,
1242 	.set_wedge	= dwc3_gadget_ep_set_wedge,
1243 };
1244 
1245 /* -------------------------------------------------------------------------- */
1246 
dwc3_gadget_get_frame(struct usb_gadget * g)1247 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1248 {
1249 	struct dwc3		*dwc = gadget_to_dwc(g);
1250 	u32			reg;
1251 
1252 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1253 	return DWC3_DSTS_SOFFN(reg);
1254 }
1255 
dwc3_gadget_wakeup(struct usb_gadget * g)1256 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1257 {
1258 	struct dwc3		*dwc = gadget_to_dwc(g);
1259 
1260 	unsigned long		timeout;
1261 	unsigned long		flags;
1262 
1263 	u32			reg;
1264 
1265 	int			ret = 0;
1266 
1267 	u8			link_state;
1268 	u8			speed;
1269 
1270 	spin_lock_irqsave(&dwc->lock, flags);
1271 
1272 	/*
1273 	 * According to the Databook Remote wakeup request should
1274 	 * be issued only when the device is in early suspend state.
1275 	 *
1276 	 * We can check that via USB Link State bits in DSTS register.
1277 	 */
1278 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1279 
1280 	speed = reg & DWC3_DSTS_CONNECTSPD;
1281 	if (speed == DWC3_DSTS_SUPERSPEED) {
1282 		dev_dbg(dwc->dev, "no wakeup on SuperSpeed\n");
1283 		ret = -EINVAL;
1284 		goto out;
1285 	}
1286 
1287 	link_state = DWC3_DSTS_USBLNKST(reg);
1288 
1289 	switch (link_state) {
1290 	case DWC3_LINK_STATE_RX_DET:	/* in HS, means Early Suspend */
1291 	case DWC3_LINK_STATE_U3:	/* in HS, means SUSPEND */
1292 		break;
1293 	default:
1294 		dev_dbg(dwc->dev, "can't wakeup from link state %d\n",
1295 				link_state);
1296 		ret = -EINVAL;
1297 		goto out;
1298 	}
1299 
1300 	ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1301 	if (ret < 0) {
1302 		dev_err(dwc->dev, "failed to put link in Recovery\n");
1303 		goto out;
1304 	}
1305 
1306 	/* Recent versions do this automatically */
1307 	if (dwc->revision < DWC3_REVISION_194A) {
1308 		/* write zeroes to Link Change Request */
1309 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1310 		reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1311 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1312 	}
1313 
1314 	/* poll until Link State changes to ON */
1315 	timeout = 1000;
1316 
1317 	while (timeout--) {
1318 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1319 
1320 		/* in HS, means ON */
1321 		if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1322 			break;
1323 	}
1324 
1325 	if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1326 		dev_err(dwc->dev, "failed to send remote wakeup\n");
1327 		ret = -EINVAL;
1328 	}
1329 
1330 out:
1331 	spin_unlock_irqrestore(&dwc->lock, flags);
1332 
1333 	return ret;
1334 }
1335 
dwc3_gadget_set_selfpowered(struct usb_gadget * g,int is_selfpowered)1336 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1337 		int is_selfpowered)
1338 {
1339 	struct dwc3		*dwc = gadget_to_dwc(g);
1340 	unsigned long		flags;
1341 
1342 	spin_lock_irqsave(&dwc->lock, flags);
1343 	dwc->is_selfpowered = !!is_selfpowered;
1344 	spin_unlock_irqrestore(&dwc->lock, flags);
1345 
1346 	return 0;
1347 }
1348 
dwc3_gadget_run_stop(struct dwc3 * dwc,int is_on,int suspend)1349 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1350 {
1351 	u32			reg;
1352 	u32			timeout = 500;
1353 
1354 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1355 	if (is_on) {
1356 		if (dwc->revision <= DWC3_REVISION_187A) {
1357 			reg &= ~DWC3_DCTL_TRGTULST_MASK;
1358 			reg |= DWC3_DCTL_TRGTULST_RX_DET;
1359 		}
1360 
1361 		if (dwc->revision >= DWC3_REVISION_194A)
1362 			reg &= ~DWC3_DCTL_KEEP_CONNECT;
1363 		reg |= DWC3_DCTL_RUN_STOP;
1364 
1365 		if (dwc->has_hibernation)
1366 			reg |= DWC3_DCTL_KEEP_CONNECT;
1367 
1368 		dwc->pullups_connected = true;
1369 	} else {
1370 		reg &= ~DWC3_DCTL_RUN_STOP;
1371 
1372 		if (dwc->has_hibernation && !suspend)
1373 			reg &= ~DWC3_DCTL_KEEP_CONNECT;
1374 
1375 		dwc->pullups_connected = false;
1376 	}
1377 
1378 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1379 
1380 	do {
1381 		reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1382 		if (is_on) {
1383 			if (!(reg & DWC3_DSTS_DEVCTRLHLT))
1384 				break;
1385 		} else {
1386 			if (reg & DWC3_DSTS_DEVCTRLHLT)
1387 				break;
1388 		}
1389 		timeout--;
1390 		if (!timeout)
1391 			return -ETIMEDOUT;
1392 		udelay(1);
1393 	} while (1);
1394 
1395 	dev_vdbg(dwc->dev, "gadget %s data soft-%s\n",
1396 			dwc->gadget_driver
1397 			? dwc->gadget_driver->function : "no-function",
1398 			is_on ? "connect" : "disconnect");
1399 
1400 	return 0;
1401 }
1402 
dwc3_gadget_pullup(struct usb_gadget * g,int is_on)1403 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1404 {
1405 	struct dwc3		*dwc = gadget_to_dwc(g);
1406 	unsigned long		flags;
1407 	int			ret;
1408 
1409 	is_on = !!is_on;
1410 
1411 	spin_lock_irqsave(&dwc->lock, flags);
1412 	ret = dwc3_gadget_run_stop(dwc, is_on, false);
1413 	spin_unlock_irqrestore(&dwc->lock, flags);
1414 
1415 	return ret;
1416 }
1417 
dwc3_gadget_enable_irq(struct dwc3 * dwc)1418 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1419 {
1420 	u32			reg;
1421 
1422 	/* Enable all but Start and End of Frame IRQs */
1423 	reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1424 			DWC3_DEVTEN_EVNTOVERFLOWEN |
1425 			DWC3_DEVTEN_CMDCMPLTEN |
1426 			DWC3_DEVTEN_ERRTICERREN |
1427 			DWC3_DEVTEN_WKUPEVTEN |
1428 			DWC3_DEVTEN_ULSTCNGEN |
1429 			DWC3_DEVTEN_CONNECTDONEEN |
1430 			DWC3_DEVTEN_USBRSTEN |
1431 			DWC3_DEVTEN_DISCONNEVTEN);
1432 
1433 	dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1434 }
1435 
dwc3_gadget_disable_irq(struct dwc3 * dwc)1436 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1437 {
1438 	/* mask all interrupts */
1439 	dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1440 }
1441 
dwc3_gadget_start(struct usb_gadget * g,struct usb_gadget_driver * driver)1442 static int dwc3_gadget_start(struct usb_gadget *g,
1443 		struct usb_gadget_driver *driver)
1444 {
1445 	struct dwc3		*dwc = gadget_to_dwc(g);
1446 	struct dwc3_ep		*dep;
1447 	unsigned long		flags;
1448 	int			ret = 0;
1449 	u32			reg;
1450 
1451 	spin_lock_irqsave(&dwc->lock, flags);
1452 
1453 	if (dwc->gadget_driver) {
1454 		dev_err(dwc->dev, "%s is already bound to %s\n",
1455 				dwc->gadget.name,
1456 				dwc->gadget_driver->function);
1457 		ret = -EBUSY;
1458 		goto err1;
1459 	}
1460 
1461 	dwc->gadget_driver	= driver;
1462 
1463 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1464 	reg &= ~(DWC3_DCFG_SPEED_MASK);
1465 
1466 	/**
1467 	 * WORKAROUND: DWC3 revision < 2.20a have an issue
1468 	 * which would cause metastability state on Run/Stop
1469 	 * bit if we try to force the IP to USB2-only mode.
1470 	 *
1471 	 * Because of that, we cannot configure the IP to any
1472 	 * speed other than the SuperSpeed
1473 	 *
1474 	 * Refers to:
1475 	 *
1476 	 * STAR#9000525659: Clock Domain Crossing on DCTL in
1477 	 * USB 2.0 Mode
1478 	 */
1479 	if (dwc->revision < DWC3_REVISION_220A) {
1480 		reg |= DWC3_DCFG_SUPERSPEED;
1481 	} else {
1482 		switch (dwc->maximum_speed) {
1483 		case USB_SPEED_LOW:
1484 			reg |= DWC3_DSTS_LOWSPEED;
1485 			break;
1486 		case USB_SPEED_FULL:
1487 			reg |= DWC3_DSTS_FULLSPEED1;
1488 			break;
1489 		case USB_SPEED_HIGH:
1490 			reg |= DWC3_DSTS_HIGHSPEED;
1491 			break;
1492 		case USB_SPEED_SUPER:	/* FALLTHROUGH */
1493 		case USB_SPEED_UNKNOWN:	/* FALTHROUGH */
1494 		default:
1495 			reg |= DWC3_DSTS_SUPERSPEED;
1496 		}
1497 	}
1498 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1499 
1500 	dwc->start_config_issued = false;
1501 
1502 	/* Start with SuperSpeed Default */
1503 	dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1504 
1505 	dep = dwc->eps[0];
1506 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1507 			false);
1508 	if (ret) {
1509 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1510 		goto err2;
1511 	}
1512 
1513 	dep = dwc->eps[1];
1514 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, false,
1515 			false);
1516 	if (ret) {
1517 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1518 		goto err3;
1519 	}
1520 
1521 	/* begin to receive SETUP packets */
1522 	dwc->ep0state = EP0_SETUP_PHASE;
1523 	dwc3_ep0_out_start(dwc);
1524 
1525 	dwc3_gadget_enable_irq(dwc);
1526 
1527 	spin_unlock_irqrestore(&dwc->lock, flags);
1528 
1529 	return 0;
1530 
1531 err3:
1532 	__dwc3_gadget_ep_disable(dwc->eps[0]);
1533 
1534 err2:
1535 	dwc->gadget_driver = NULL;
1536 
1537 err1:
1538 	spin_unlock_irqrestore(&dwc->lock, flags);
1539 
1540 	return ret;
1541 }
1542 
dwc3_gadget_stop(struct usb_gadget * g)1543 static int dwc3_gadget_stop(struct usb_gadget *g)
1544 {
1545 	struct dwc3		*dwc = gadget_to_dwc(g);
1546 	unsigned long		flags;
1547 
1548 	spin_lock_irqsave(&dwc->lock, flags);
1549 
1550 	dwc3_gadget_disable_irq(dwc);
1551 	__dwc3_gadget_ep_disable(dwc->eps[0]);
1552 	__dwc3_gadget_ep_disable(dwc->eps[1]);
1553 
1554 	dwc->gadget_driver	= NULL;
1555 
1556 	spin_unlock_irqrestore(&dwc->lock, flags);
1557 
1558 	return 0;
1559 }
1560 
1561 static const struct usb_gadget_ops dwc3_gadget_ops = {
1562 	.get_frame		= dwc3_gadget_get_frame,
1563 	.wakeup			= dwc3_gadget_wakeup,
1564 	.set_selfpowered	= dwc3_gadget_set_selfpowered,
1565 	.pullup			= dwc3_gadget_pullup,
1566 	.udc_start		= dwc3_gadget_start,
1567 	.udc_stop		= dwc3_gadget_stop,
1568 };
1569 
1570 /* -------------------------------------------------------------------------- */
1571 
dwc3_gadget_init_hw_endpoints(struct dwc3 * dwc,u8 num,u32 direction)1572 static int dwc3_gadget_init_hw_endpoints(struct dwc3 *dwc,
1573 		u8 num, u32 direction)
1574 {
1575 	struct dwc3_ep			*dep;
1576 	u8				i;
1577 
1578 	for (i = 0; i < num; i++) {
1579 		u8 epnum = (i << 1) | (!!direction);
1580 
1581 		dep = kzalloc(sizeof(*dep), GFP_KERNEL);
1582 		if (!dep)
1583 			return -ENOMEM;
1584 
1585 		dep->dwc = dwc;
1586 		dep->number = epnum;
1587 		dep->direction = !!direction;
1588 		dwc->eps[epnum] = dep;
1589 
1590 		snprintf(dep->name, sizeof(dep->name), "ep%d%s", epnum >> 1,
1591 				(epnum & 1) ? "in" : "out");
1592 
1593 		dep->endpoint.name = dep->name;
1594 
1595 		dev_vdbg(dwc->dev, "initializing %s\n", dep->name);
1596 
1597 		if (epnum == 0 || epnum == 1) {
1598 			usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
1599 			dep->endpoint.maxburst = 1;
1600 			dep->endpoint.ops = &dwc3_gadget_ep0_ops;
1601 			if (!epnum)
1602 				dwc->gadget.ep0 = &dep->endpoint;
1603 		} else {
1604 			int		ret;
1605 
1606 			usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
1607 			dep->endpoint.max_streams = 15;
1608 			dep->endpoint.ops = &dwc3_gadget_ep_ops;
1609 			list_add_tail(&dep->endpoint.ep_list,
1610 					&dwc->gadget.ep_list);
1611 
1612 			ret = dwc3_alloc_trb_pool(dep);
1613 			if (ret)
1614 				return ret;
1615 		}
1616 
1617 		INIT_LIST_HEAD(&dep->request_list);
1618 		INIT_LIST_HEAD(&dep->req_queued);
1619 	}
1620 
1621 	return 0;
1622 }
1623 
dwc3_gadget_init_endpoints(struct dwc3 * dwc)1624 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc)
1625 {
1626 	int				ret;
1627 
1628 	INIT_LIST_HEAD(&dwc->gadget.ep_list);
1629 
1630 	ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_out_eps, 0);
1631 	if (ret < 0) {
1632 		dev_vdbg(dwc->dev, "failed to allocate OUT endpoints\n");
1633 		return ret;
1634 	}
1635 
1636 	ret = dwc3_gadget_init_hw_endpoints(dwc, dwc->num_in_eps, 1);
1637 	if (ret < 0) {
1638 		dev_vdbg(dwc->dev, "failed to allocate IN endpoints\n");
1639 		return ret;
1640 	}
1641 
1642 	return 0;
1643 }
1644 
dwc3_gadget_free_endpoints(struct dwc3 * dwc)1645 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
1646 {
1647 	struct dwc3_ep			*dep;
1648 	u8				epnum;
1649 
1650 	for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1651 		dep = dwc->eps[epnum];
1652 		if (!dep)
1653 			continue;
1654 		/*
1655 		 * Physical endpoints 0 and 1 are special; they form the
1656 		 * bi-directional USB endpoint 0.
1657 		 *
1658 		 * For those two physical endpoints, we don't allocate a TRB
1659 		 * pool nor do we add them the endpoints list. Due to that, we
1660 		 * shouldn't do these two operations otherwise we would end up
1661 		 * with all sorts of bugs when removing dwc3.ko.
1662 		 */
1663 		if (epnum != 0 && epnum != 1) {
1664 			dwc3_free_trb_pool(dep);
1665 			list_del(&dep->endpoint.ep_list);
1666 		}
1667 
1668 		kfree(dep);
1669 	}
1670 }
1671 
1672 /* -------------------------------------------------------------------------- */
1673 
__dwc3_cleanup_done_trbs(struct dwc3 * dwc,struct dwc3_ep * dep,struct dwc3_request * req,struct dwc3_trb * trb,const struct dwc3_event_depevt * event,int status)1674 static int __dwc3_cleanup_done_trbs(struct dwc3 *dwc, struct dwc3_ep *dep,
1675 		struct dwc3_request *req, struct dwc3_trb *trb,
1676 		const struct dwc3_event_depevt *event, int status)
1677 {
1678 	unsigned int		count;
1679 	unsigned int		s_pkt = 0;
1680 	unsigned int		trb_status;
1681 
1682 	if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
1683 		/*
1684 		 * We continue despite the error. There is not much we
1685 		 * can do. If we don't clean it up we loop forever. If
1686 		 * we skip the TRB then it gets overwritten after a
1687 		 * while since we use them in a ring buffer. A BUG()
1688 		 * would help. Lets hope that if this occurs, someone
1689 		 * fixes the root cause instead of looking away :)
1690 		 */
1691 		dev_err(dwc->dev, "%s's TRB (%p) still owned by HW\n",
1692 				dep->name, trb);
1693 	count = trb->size & DWC3_TRB_SIZE_MASK;
1694 
1695 	if (dep->direction) {
1696 		if (count) {
1697 			trb_status = DWC3_TRB_SIZE_TRBSTS(trb->size);
1698 			if (trb_status == DWC3_TRBSTS_MISSED_ISOC) {
1699 				dev_dbg(dwc->dev, "incomplete IN transfer %s\n",
1700 						dep->name);
1701 				/*
1702 				 * If missed isoc occurred and there is
1703 				 * no request queued then issue END
1704 				 * TRANSFER, so that core generates
1705 				 * next xfernotready and we will issue
1706 				 * a fresh START TRANSFER.
1707 				 * If there are still queued request
1708 				 * then wait, do not issue either END
1709 				 * or UPDATE TRANSFER, just attach next
1710 				 * request in request_list during
1711 				 * giveback.If any future queued request
1712 				 * is successfully transferred then we
1713 				 * will issue UPDATE TRANSFER for all
1714 				 * request in the request_list.
1715 				 */
1716 				dep->flags |= DWC3_EP_MISSED_ISOC;
1717 			} else {
1718 				dev_err(dwc->dev, "incomplete IN transfer %s\n",
1719 						dep->name);
1720 				status = -ECONNRESET;
1721 			}
1722 		} else {
1723 			dep->flags &= ~DWC3_EP_MISSED_ISOC;
1724 		}
1725 	} else {
1726 		if (count && (event->status & DEPEVT_STATUS_SHORT))
1727 			s_pkt = 1;
1728 	}
1729 
1730 	/*
1731 	 * We assume here we will always receive the entire data block
1732 	 * which we should receive. Meaning, if we program RX to
1733 	 * receive 4K but we receive only 2K, we assume that's all we
1734 	 * should receive and we simply bounce the request back to the
1735 	 * gadget driver for further processing.
1736 	 */
1737 	req->request.actual += req->request.length - count;
1738 	if (s_pkt)
1739 		return 1;
1740 	if ((event->status & DEPEVT_STATUS_LST) &&
1741 			(trb->ctrl & (DWC3_TRB_CTRL_LST |
1742 				DWC3_TRB_CTRL_HWO)))
1743 		return 1;
1744 	if ((event->status & DEPEVT_STATUS_IOC) &&
1745 			(trb->ctrl & DWC3_TRB_CTRL_IOC))
1746 		return 1;
1747 	return 0;
1748 }
1749 
dwc3_cleanup_done_reqs(struct dwc3 * dwc,struct dwc3_ep * dep,const struct dwc3_event_depevt * event,int status)1750 static int dwc3_cleanup_done_reqs(struct dwc3 *dwc, struct dwc3_ep *dep,
1751 		const struct dwc3_event_depevt *event, int status)
1752 {
1753 	struct dwc3_request	*req;
1754 	struct dwc3_trb		*trb;
1755 	unsigned int		slot;
1756 
1757 	req = next_request(&dep->req_queued);
1758 	if (!req) {
1759 		WARN_ON_ONCE(1);
1760 		return 1;
1761 	}
1762 
1763 	slot = req->start_slot;
1764 	if ((slot == DWC3_TRB_NUM - 1) &&
1765 	    usb_endpoint_xfer_isoc(dep->endpoint.desc))
1766 		slot++;
1767 	slot %= DWC3_TRB_NUM;
1768 	trb = &dep->trb_pool[slot];
1769 
1770 	dwc3_flush_cache((uintptr_t)trb, sizeof(*trb));
1771 	__dwc3_cleanup_done_trbs(dwc, dep, req, trb, event, status);
1772 	dwc3_gadget_giveback(dep, req, status);
1773 
1774 	if (usb_endpoint_xfer_isoc(dep->endpoint.desc) &&
1775 			list_empty(&dep->req_queued)) {
1776 		if (list_empty(&dep->request_list)) {
1777 			/*
1778 			 * If there is no entry in request list then do
1779 			 * not issue END TRANSFER now. Just set PENDING
1780 			 * flag, so that END TRANSFER is issued when an
1781 			 * entry is added into request list.
1782 			 */
1783 			dep->flags = DWC3_EP_PENDING_REQUEST;
1784 		} else {
1785 			dwc3_stop_active_transfer(dwc, dep->number, true);
1786 			dep->flags = DWC3_EP_ENABLED;
1787 		}
1788 		return 1;
1789 	}
1790 
1791 	return 1;
1792 }
1793 
dwc3_endpoint_transfer_complete(struct dwc3 * dwc,struct dwc3_ep * dep,const struct dwc3_event_depevt * event)1794 static void dwc3_endpoint_transfer_complete(struct dwc3 *dwc,
1795 		struct dwc3_ep *dep, const struct dwc3_event_depevt *event)
1796 {
1797 	unsigned		status = 0;
1798 	int			clean_busy;
1799 
1800 	if (event->status & DEPEVT_STATUS_BUSERR)
1801 		status = -ECONNRESET;
1802 
1803 	clean_busy = dwc3_cleanup_done_reqs(dwc, dep, event, status);
1804 	if (clean_busy)
1805 		dep->flags &= ~DWC3_EP_BUSY;
1806 
1807 	/*
1808 	 * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
1809 	 * See dwc3_gadget_linksts_change_interrupt() for 1st half.
1810 	 */
1811 	if (dwc->revision < DWC3_REVISION_183A) {
1812 		u32		reg;
1813 		int		i;
1814 
1815 		for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
1816 			dep = dwc->eps[i];
1817 
1818 			if (!(dep->flags & DWC3_EP_ENABLED))
1819 				continue;
1820 
1821 			if (!list_empty(&dep->req_queued))
1822 				return;
1823 		}
1824 
1825 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1826 		reg |= dwc->u1u2;
1827 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1828 
1829 		dwc->u1u2 = 0;
1830 	}
1831 }
1832 
dwc3_endpoint_interrupt(struct dwc3 * dwc,const struct dwc3_event_depevt * event)1833 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
1834 		const struct dwc3_event_depevt *event)
1835 {
1836 	struct dwc3_ep		*dep;
1837 	u8			epnum = event->endpoint_number;
1838 
1839 	dep = dwc->eps[epnum];
1840 
1841 	if (!(dep->flags & DWC3_EP_ENABLED))
1842 		return;
1843 
1844 	if (epnum == 0 || epnum == 1) {
1845 		dwc3_ep0_interrupt(dwc, event);
1846 		return;
1847 	}
1848 
1849 	switch (event->endpoint_event) {
1850 	case DWC3_DEPEVT_XFERCOMPLETE:
1851 		dep->resource_index = 0;
1852 
1853 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1854 			dev_dbg(dwc->dev, "%s is an Isochronous endpoint\n",
1855 					dep->name);
1856 			return;
1857 		}
1858 
1859 		dwc3_endpoint_transfer_complete(dwc, dep, event);
1860 		break;
1861 	case DWC3_DEPEVT_XFERINPROGRESS:
1862 		dwc3_endpoint_transfer_complete(dwc, dep, event);
1863 		break;
1864 	case DWC3_DEPEVT_XFERNOTREADY:
1865 		if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1866 			dwc3_gadget_start_isoc(dwc, dep, event);
1867 		} else {
1868 			int ret;
1869 
1870 			dev_vdbg(dwc->dev, "%s: reason %s\n",
1871 					dep->name, event->status &
1872 					DEPEVT_STATUS_TRANSFER_ACTIVE
1873 					? "Transfer Active"
1874 					: "Transfer Not Active");
1875 
1876 			ret = __dwc3_gadget_kick_transfer(dep, 0, 1);
1877 			if (!ret || ret == -EBUSY)
1878 				return;
1879 
1880 			dev_dbg(dwc->dev, "%s: failed to kick transfers\n",
1881 					dep->name);
1882 		}
1883 
1884 		break;
1885 	case DWC3_DEPEVT_STREAMEVT:
1886 		if (!usb_endpoint_xfer_bulk(dep->endpoint.desc)) {
1887 			dev_err(dwc->dev, "Stream event for non-Bulk %s\n",
1888 					dep->name);
1889 			return;
1890 		}
1891 
1892 		switch (event->status) {
1893 		case DEPEVT_STREAMEVT_FOUND:
1894 			dev_vdbg(dwc->dev, "Stream %d found and started\n",
1895 					event->parameters);
1896 
1897 			break;
1898 		case DEPEVT_STREAMEVT_NOTFOUND:
1899 			/* FALLTHROUGH */
1900 		default:
1901 			dev_dbg(dwc->dev, "Couldn't find suitable stream\n");
1902 		}
1903 		break;
1904 	case DWC3_DEPEVT_RXTXFIFOEVT:
1905 		dev_dbg(dwc->dev, "%s FIFO Overrun\n", dep->name);
1906 		break;
1907 	case DWC3_DEPEVT_EPCMDCMPLT:
1908 		dev_vdbg(dwc->dev, "Endpoint Command Complete\n");
1909 		break;
1910 	}
1911 }
1912 
dwc3_disconnect_gadget(struct dwc3 * dwc)1913 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
1914 {
1915 	if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
1916 		spin_unlock(&dwc->lock);
1917 		dwc->gadget_driver->disconnect(&dwc->gadget);
1918 		spin_lock(&dwc->lock);
1919 	}
1920 }
1921 
dwc3_suspend_gadget(struct dwc3 * dwc)1922 static void dwc3_suspend_gadget(struct dwc3 *dwc)
1923 {
1924 	if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
1925 		spin_unlock(&dwc->lock);
1926 		dwc->gadget_driver->suspend(&dwc->gadget);
1927 		spin_lock(&dwc->lock);
1928 	}
1929 }
1930 
dwc3_resume_gadget(struct dwc3 * dwc)1931 static void dwc3_resume_gadget(struct dwc3 *dwc)
1932 {
1933 	if (dwc->gadget_driver && dwc->gadget_driver->resume) {
1934 		spin_unlock(&dwc->lock);
1935 		dwc->gadget_driver->resume(&dwc->gadget);
1936 	}
1937 }
1938 
dwc3_reset_gadget(struct dwc3 * dwc)1939 static void dwc3_reset_gadget(struct dwc3 *dwc)
1940 {
1941 	if (!dwc->gadget_driver)
1942 		return;
1943 
1944 	if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
1945 		spin_unlock(&dwc->lock);
1946 		usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
1947 		spin_lock(&dwc->lock);
1948 	}
1949 }
1950 
dwc3_stop_active_transfer(struct dwc3 * dwc,u32 epnum,bool force)1951 static void dwc3_stop_active_transfer(struct dwc3 *dwc, u32 epnum, bool force)
1952 {
1953 	struct dwc3_ep *dep;
1954 	struct dwc3_gadget_ep_cmd_params params;
1955 	u32 cmd;
1956 	int ret;
1957 
1958 	dep = dwc->eps[epnum];
1959 
1960 	if (!dep->resource_index)
1961 		return;
1962 
1963 	/*
1964 	 * NOTICE: We are violating what the Databook says about the
1965 	 * EndTransfer command. Ideally we would _always_ wait for the
1966 	 * EndTransfer Command Completion IRQ, but that's causing too
1967 	 * much trouble synchronizing between us and gadget driver.
1968 	 *
1969 	 * We have discussed this with the IP Provider and it was
1970 	 * suggested to giveback all requests here, but give HW some
1971 	 * extra time to synchronize with the interconnect. We're using
1972 	 * an arbitraty 100us delay for that.
1973 	 *
1974 	 * Note also that a similar handling was tested by Synopsys
1975 	 * (thanks a lot Paul) and nothing bad has come out of it.
1976 	 * In short, what we're doing is:
1977 	 *
1978 	 * - Issue EndTransfer WITH CMDIOC bit set
1979 	 * - Wait 100us
1980 	 */
1981 
1982 	cmd = DWC3_DEPCMD_ENDTRANSFER;
1983 	cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
1984 	cmd |= DWC3_DEPCMD_CMDIOC;
1985 	cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
1986 	memset(&params, 0, sizeof(params));
1987 	ret = dwc3_send_gadget_ep_cmd(dwc, dep->number, cmd, &params);
1988 	WARN_ON_ONCE(ret);
1989 	dep->resource_index = 0;
1990 	dep->flags &= ~DWC3_EP_BUSY;
1991 	udelay(100);
1992 }
1993 
dwc3_stop_active_transfers(struct dwc3 * dwc)1994 static void dwc3_stop_active_transfers(struct dwc3 *dwc)
1995 {
1996 	u32 epnum;
1997 
1998 	for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1999 		struct dwc3_ep *dep;
2000 
2001 		dep = dwc->eps[epnum];
2002 		if (!dep)
2003 			continue;
2004 
2005 		if (!(dep->flags & DWC3_EP_ENABLED))
2006 			continue;
2007 
2008 		dwc3_remove_requests(dwc, dep);
2009 	}
2010 }
2011 
dwc3_clear_stall_all_ep(struct dwc3 * dwc)2012 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2013 {
2014 	u32 epnum;
2015 
2016 	for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2017 		struct dwc3_ep *dep;
2018 		struct dwc3_gadget_ep_cmd_params params;
2019 		int ret;
2020 
2021 		dep = dwc->eps[epnum];
2022 		if (!dep)
2023 			continue;
2024 
2025 		if (!(dep->flags & DWC3_EP_STALL))
2026 			continue;
2027 
2028 		dep->flags &= ~DWC3_EP_STALL;
2029 
2030 		memset(&params, 0, sizeof(params));
2031 		ret = dwc3_send_gadget_ep_cmd(dwc, dep->number,
2032 				DWC3_DEPCMD_CLEARSTALL, &params);
2033 		WARN_ON_ONCE(ret);
2034 	}
2035 }
2036 
dwc3_gadget_disconnect_interrupt(struct dwc3 * dwc)2037 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2038 {
2039 	int			reg;
2040 
2041 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2042 	reg &= ~DWC3_DCTL_INITU1ENA;
2043 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2044 
2045 	reg &= ~DWC3_DCTL_INITU2ENA;
2046 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2047 
2048 	dwc3_disconnect_gadget(dwc);
2049 	dwc->start_config_issued = false;
2050 
2051 	dwc->gadget.speed = USB_SPEED_UNKNOWN;
2052 	dwc->setup_packet_pending = false;
2053 	usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2054 }
2055 
dwc3_gadget_reset_interrupt(struct dwc3 * dwc)2056 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2057 {
2058 	u32			reg;
2059 
2060 	/*
2061 	 * WORKAROUND: DWC3 revisions <1.88a have an issue which
2062 	 * would cause a missing Disconnect Event if there's a
2063 	 * pending Setup Packet in the FIFO.
2064 	 *
2065 	 * There's no suggested workaround on the official Bug
2066 	 * report, which states that "unless the driver/application
2067 	 * is doing any special handling of a disconnect event,
2068 	 * there is no functional issue".
2069 	 *
2070 	 * Unfortunately, it turns out that we _do_ some special
2071 	 * handling of a disconnect event, namely complete all
2072 	 * pending transfers, notify gadget driver of the
2073 	 * disconnection, and so on.
2074 	 *
2075 	 * Our suggested workaround is to follow the Disconnect
2076 	 * Event steps here, instead, based on a setup_packet_pending
2077 	 * flag. Such flag gets set whenever we have a XferNotReady
2078 	 * event on EP0 and gets cleared on XferComplete for the
2079 	 * same endpoint.
2080 	 *
2081 	 * Refers to:
2082 	 *
2083 	 * STAR#9000466709: RTL: Device : Disconnect event not
2084 	 * generated if setup packet pending in FIFO
2085 	 */
2086 	if (dwc->revision < DWC3_REVISION_188A) {
2087 		if (dwc->setup_packet_pending)
2088 			dwc3_gadget_disconnect_interrupt(dwc);
2089 	}
2090 
2091 	dwc3_reset_gadget(dwc);
2092 
2093 	reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2094 	reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2095 	dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2096 	dwc->test_mode = false;
2097 
2098 	dwc3_stop_active_transfers(dwc);
2099 	dwc3_clear_stall_all_ep(dwc);
2100 	dwc->start_config_issued = false;
2101 
2102 	/* Reset device address to zero */
2103 	reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2104 	reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2105 	dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2106 }
2107 
dwc3_update_ram_clk_sel(struct dwc3 * dwc,u32 speed)2108 static void dwc3_update_ram_clk_sel(struct dwc3 *dwc, u32 speed)
2109 {
2110 	u32 reg;
2111 	u32 usb30_clock = DWC3_GCTL_CLK_BUS;
2112 
2113 	/*
2114 	 * We change the clock only at SS but I dunno why I would want to do
2115 	 * this. Maybe it becomes part of the power saving plan.
2116 	 */
2117 
2118 	if (speed != DWC3_DSTS_SUPERSPEED)
2119 		return;
2120 
2121 	/*
2122 	 * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2123 	 * each time on Connect Done.
2124 	 */
2125 	if (!usb30_clock)
2126 		return;
2127 
2128 	reg = dwc3_readl(dwc->regs, DWC3_GCTL);
2129 	reg |= DWC3_GCTL_RAMCLKSEL(usb30_clock);
2130 	dwc3_writel(dwc->regs, DWC3_GCTL, reg);
2131 }
2132 
dwc3_gadget_conndone_interrupt(struct dwc3 * dwc)2133 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2134 {
2135 	struct dwc3_ep		*dep;
2136 	int			ret;
2137 	u32			reg;
2138 	u8			speed;
2139 
2140 	reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2141 	speed = reg & DWC3_DSTS_CONNECTSPD;
2142 	dwc->speed = speed;
2143 
2144 	dwc3_update_ram_clk_sel(dwc, speed);
2145 
2146 	switch (speed) {
2147 	case DWC3_DCFG_SUPERSPEED:
2148 		/*
2149 		 * WORKAROUND: DWC3 revisions <1.90a have an issue which
2150 		 * would cause a missing USB3 Reset event.
2151 		 *
2152 		 * In such situations, we should force a USB3 Reset
2153 		 * event by calling our dwc3_gadget_reset_interrupt()
2154 		 * routine.
2155 		 *
2156 		 * Refers to:
2157 		 *
2158 		 * STAR#9000483510: RTL: SS : USB3 reset event may
2159 		 * not be generated always when the link enters poll
2160 		 */
2161 		if (dwc->revision < DWC3_REVISION_190A)
2162 			dwc3_gadget_reset_interrupt(dwc);
2163 
2164 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2165 		dwc->gadget.ep0->maxpacket = 512;
2166 		dwc->gadget.speed = USB_SPEED_SUPER;
2167 		break;
2168 	case DWC3_DCFG_HIGHSPEED:
2169 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2170 		dwc->gadget.ep0->maxpacket = 64;
2171 		dwc->gadget.speed = USB_SPEED_HIGH;
2172 		break;
2173 	case DWC3_DCFG_FULLSPEED2:
2174 	case DWC3_DCFG_FULLSPEED1:
2175 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2176 		dwc->gadget.ep0->maxpacket = 64;
2177 		dwc->gadget.speed = USB_SPEED_FULL;
2178 		break;
2179 	case DWC3_DCFG_LOWSPEED:
2180 		dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2181 		dwc->gadget.ep0->maxpacket = 8;
2182 		dwc->gadget.speed = USB_SPEED_LOW;
2183 		break;
2184 	}
2185 
2186 	/* Enable USB2 LPM Capability */
2187 
2188 	if ((dwc->revision > DWC3_REVISION_194A)
2189 			&& (speed != DWC3_DCFG_SUPERSPEED)) {
2190 		reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2191 		reg |= DWC3_DCFG_LPM_CAP;
2192 		dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2193 
2194 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2195 		reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2196 
2197 		reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2198 
2199 		/*
2200 		 * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2201 		 * DCFG.LPMCap is set, core responses with an ACK and the
2202 		 * BESL value in the LPM token is less than or equal to LPM
2203 		 * NYET threshold.
2204 		 */
2205 		if (dwc->revision < DWC3_REVISION_240A 	&& dwc->has_lpm_erratum)
2206 			WARN(true, "LPM Erratum not available on dwc3 revisisions < 2.40a\n");
2207 
2208 		if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2209 			reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2210 
2211 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2212 	} else {
2213 		reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2214 		reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2215 		dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2216 	}
2217 
2218 	dep = dwc->eps[0];
2219 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2220 			false);
2221 	if (ret) {
2222 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2223 		return;
2224 	}
2225 
2226 	dep = dwc->eps[1];
2227 	ret = __dwc3_gadget_ep_enable(dep, &dwc3_gadget_ep0_desc, NULL, true,
2228 			false);
2229 	if (ret) {
2230 		dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2231 		return;
2232 	}
2233 
2234 	/*
2235 	 * Configure PHY via GUSB3PIPECTLn if required.
2236 	 *
2237 	 * Update GTXFIFOSIZn
2238 	 *
2239 	 * In both cases reset values should be sufficient.
2240 	 */
2241 }
2242 
dwc3_gadget_wakeup_interrupt(struct dwc3 * dwc)2243 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2244 {
2245 	/*
2246 	 * TODO take core out of low power mode when that's
2247 	 * implemented.
2248 	 */
2249 
2250 	dwc->gadget_driver->resume(&dwc->gadget);
2251 }
2252 
dwc3_gadget_linksts_change_interrupt(struct dwc3 * dwc,unsigned int evtinfo)2253 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2254 		unsigned int evtinfo)
2255 {
2256 	enum dwc3_link_state	next = evtinfo & DWC3_LINK_STATE_MASK;
2257 	unsigned int		pwropt;
2258 
2259 	/*
2260 	 * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2261 	 * Hibernation mode enabled which would show up when device detects
2262 	 * host-initiated U3 exit.
2263 	 *
2264 	 * In that case, device will generate a Link State Change Interrupt
2265 	 * from U3 to RESUME which is only necessary if Hibernation is
2266 	 * configured in.
2267 	 *
2268 	 * There are no functional changes due to such spurious event and we
2269 	 * just need to ignore it.
2270 	 *
2271 	 * Refers to:
2272 	 *
2273 	 * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2274 	 * operational mode
2275 	 */
2276 	pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2277 	if ((dwc->revision < DWC3_REVISION_250A) &&
2278 			(pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2279 		if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2280 				(next == DWC3_LINK_STATE_RESUME)) {
2281 			dev_vdbg(dwc->dev, "ignoring transition U3 -> Resume\n");
2282 			return;
2283 		}
2284 	}
2285 
2286 	/*
2287 	 * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2288 	 * on the link partner, the USB session might do multiple entry/exit
2289 	 * of low power states before a transfer takes place.
2290 	 *
2291 	 * Due to this problem, we might experience lower throughput. The
2292 	 * suggested workaround is to disable DCTL[12:9] bits if we're
2293 	 * transitioning from U1/U2 to U0 and enable those bits again
2294 	 * after a transfer completes and there are no pending transfers
2295 	 * on any of the enabled endpoints.
2296 	 *
2297 	 * This is the first half of that workaround.
2298 	 *
2299 	 * Refers to:
2300 	 *
2301 	 * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2302 	 * core send LGO_Ux entering U0
2303 	 */
2304 	if (dwc->revision < DWC3_REVISION_183A) {
2305 		if (next == DWC3_LINK_STATE_U0) {
2306 			u32	u1u2;
2307 			u32	reg;
2308 
2309 			switch (dwc->link_state) {
2310 			case DWC3_LINK_STATE_U1:
2311 			case DWC3_LINK_STATE_U2:
2312 				reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2313 				u1u2 = reg & (DWC3_DCTL_INITU2ENA
2314 						| DWC3_DCTL_ACCEPTU2ENA
2315 						| DWC3_DCTL_INITU1ENA
2316 						| DWC3_DCTL_ACCEPTU1ENA);
2317 
2318 				if (!dwc->u1u2)
2319 					dwc->u1u2 = reg & u1u2;
2320 
2321 				reg &= ~u1u2;
2322 
2323 				dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2324 				break;
2325 			default:
2326 				/* do nothing */
2327 				break;
2328 			}
2329 		}
2330 	}
2331 
2332 	switch (next) {
2333 	case DWC3_LINK_STATE_U1:
2334 		if (dwc->speed == USB_SPEED_SUPER)
2335 			dwc3_suspend_gadget(dwc);
2336 		break;
2337 	case DWC3_LINK_STATE_U2:
2338 	case DWC3_LINK_STATE_U3:
2339 		dwc3_suspend_gadget(dwc);
2340 		break;
2341 	case DWC3_LINK_STATE_RESUME:
2342 		dwc3_resume_gadget(dwc);
2343 		break;
2344 	default:
2345 		/* do nothing */
2346 		break;
2347 	}
2348 
2349 	dwc->link_state = next;
2350 }
2351 
dwc3_gadget_hibernation_interrupt(struct dwc3 * dwc,unsigned int evtinfo)2352 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2353 		unsigned int evtinfo)
2354 {
2355 	unsigned int is_ss = evtinfo & (1UL << 4);
2356 
2357 	/**
2358 	 * WORKAROUND: DWC3 revison 2.20a with hibernation support
2359 	 * have a known issue which can cause USB CV TD.9.23 to fail
2360 	 * randomly.
2361 	 *
2362 	 * Because of this issue, core could generate bogus hibernation
2363 	 * events which SW needs to ignore.
2364 	 *
2365 	 * Refers to:
2366 	 *
2367 	 * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2368 	 * Device Fallback from SuperSpeed
2369 	 */
2370 	if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2371 		return;
2372 
2373 	/* enter hibernation here */
2374 }
2375 
dwc3_gadget_interrupt(struct dwc3 * dwc,const struct dwc3_event_devt * event)2376 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2377 		const struct dwc3_event_devt *event)
2378 {
2379 	switch (event->type) {
2380 	case DWC3_DEVICE_EVENT_DISCONNECT:
2381 		dwc3_gadget_disconnect_interrupt(dwc);
2382 		break;
2383 	case DWC3_DEVICE_EVENT_RESET:
2384 		dwc3_gadget_reset_interrupt(dwc);
2385 		break;
2386 	case DWC3_DEVICE_EVENT_CONNECT_DONE:
2387 		dwc3_gadget_conndone_interrupt(dwc);
2388 		break;
2389 	case DWC3_DEVICE_EVENT_WAKEUP:
2390 		dwc3_gadget_wakeup_interrupt(dwc);
2391 		break;
2392 	case DWC3_DEVICE_EVENT_HIBER_REQ:
2393 		if (!dwc->has_hibernation) {
2394 			WARN(1 ,"unexpected hibernation event\n");
2395 			break;
2396 		}
2397 		dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2398 		break;
2399 	case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2400 		dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2401 		break;
2402 	case DWC3_DEVICE_EVENT_EOPF:
2403 		dev_vdbg(dwc->dev, "End of Periodic Frame\n");
2404 		break;
2405 	case DWC3_DEVICE_EVENT_SOF:
2406 		dev_vdbg(dwc->dev, "Start of Periodic Frame\n");
2407 		break;
2408 	case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2409 		dev_vdbg(dwc->dev, "Erratic Error\n");
2410 		break;
2411 	case DWC3_DEVICE_EVENT_CMD_CMPL:
2412 		dev_vdbg(dwc->dev, "Command Complete\n");
2413 		break;
2414 	case DWC3_DEVICE_EVENT_OVERFLOW:
2415 		dev_vdbg(dwc->dev, "Overflow\n");
2416 		break;
2417 	default:
2418 		dev_dbg(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2419 	}
2420 }
2421 
dwc3_process_event_entry(struct dwc3 * dwc,const union dwc3_event * event)2422 static void dwc3_process_event_entry(struct dwc3 *dwc,
2423 		const union dwc3_event *event)
2424 {
2425 	/* Endpoint IRQ, handle it and return early */
2426 	if (event->type.is_devspec == 0) {
2427 		/* depevt */
2428 		return dwc3_endpoint_interrupt(dwc, &event->depevt);
2429 	}
2430 
2431 	switch (event->type.type) {
2432 	case DWC3_EVENT_TYPE_DEV:
2433 		dwc3_gadget_interrupt(dwc, &event->devt);
2434 		break;
2435 	/* REVISIT what to do with Carkit and I2C events ? */
2436 	default:
2437 		dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
2438 	}
2439 }
2440 
dwc3_process_event_buf(struct dwc3 * dwc,u32 buf)2441 static irqreturn_t dwc3_process_event_buf(struct dwc3 *dwc, u32 buf)
2442 {
2443 	struct dwc3_event_buffer *evt;
2444 	irqreturn_t ret = IRQ_NONE;
2445 	int left;
2446 	u32 reg;
2447 
2448 	evt = dwc->ev_buffs[buf];
2449 	left = evt->count;
2450 
2451 	if (!(evt->flags & DWC3_EVENT_PENDING))
2452 		return IRQ_NONE;
2453 
2454 	while (left > 0) {
2455 		union dwc3_event event;
2456 
2457 		event.raw = *(u32 *) (evt->buf + evt->lpos);
2458 
2459 		dwc3_process_event_entry(dwc, &event);
2460 
2461 		/*
2462 		 * FIXME we wrap around correctly to the next entry as
2463 		 * almost all entries are 4 bytes in size. There is one
2464 		 * entry which has 12 bytes which is a regular entry
2465 		 * followed by 8 bytes data. ATM I don't know how
2466 		 * things are organized if we get next to the a
2467 		 * boundary so I worry about that once we try to handle
2468 		 * that.
2469 		 */
2470 		evt->lpos = (evt->lpos + 4) % DWC3_EVENT_BUFFERS_SIZE;
2471 		left -= 4;
2472 
2473 		dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(buf), 4);
2474 	}
2475 
2476 	evt->count = 0;
2477 	evt->flags &= ~DWC3_EVENT_PENDING;
2478 	ret = IRQ_HANDLED;
2479 
2480 	/* Unmask interrupt */
2481 	reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(buf));
2482 	reg &= ~DWC3_GEVNTSIZ_INTMASK;
2483 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(buf), reg);
2484 
2485 	return ret;
2486 }
2487 
dwc3_thread_interrupt(int irq,void * _dwc)2488 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc)
2489 {
2490 	struct dwc3 *dwc = _dwc;
2491 	unsigned long flags;
2492 	irqreturn_t ret = IRQ_NONE;
2493 	int i;
2494 
2495 	spin_lock_irqsave(&dwc->lock, flags);
2496 
2497 	for (i = 0; i < dwc->num_event_buffers; i++)
2498 		ret |= dwc3_process_event_buf(dwc, i);
2499 
2500 	spin_unlock_irqrestore(&dwc->lock, flags);
2501 
2502 	return ret;
2503 }
2504 
dwc3_check_event_buf(struct dwc3 * dwc,u32 buf)2505 static irqreturn_t dwc3_check_event_buf(struct dwc3 *dwc, u32 buf)
2506 {
2507 	struct dwc3_event_buffer *evt;
2508 	u32 count;
2509 	u32 reg;
2510 
2511 	evt = dwc->ev_buffs[buf];
2512 
2513 	count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(buf));
2514 	count &= DWC3_GEVNTCOUNT_MASK;
2515 	if (!count)
2516 		return IRQ_NONE;
2517 
2518 	evt->count = count;
2519 	evt->flags |= DWC3_EVENT_PENDING;
2520 
2521 	/* Mask interrupt */
2522 	reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(buf));
2523 	reg |= DWC3_GEVNTSIZ_INTMASK;
2524 	dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(buf), reg);
2525 
2526 	return IRQ_WAKE_THREAD;
2527 }
2528 
dwc3_interrupt(int irq,void * _dwc)2529 static irqreturn_t dwc3_interrupt(int irq, void *_dwc)
2530 {
2531 	struct dwc3			*dwc = _dwc;
2532 	int				i;
2533 	irqreturn_t			ret = IRQ_NONE;
2534 
2535 	spin_lock(&dwc->lock);
2536 
2537 	for (i = 0; i < dwc->num_event_buffers; i++) {
2538 		irqreturn_t status;
2539 
2540 		status = dwc3_check_event_buf(dwc, i);
2541 		if (status == IRQ_WAKE_THREAD)
2542 			ret = status;
2543 	}
2544 
2545 	spin_unlock(&dwc->lock);
2546 
2547 	return ret;
2548 }
2549 
2550 /**
2551  * dwc3_gadget_init - Initializes gadget related registers
2552  * @dwc: pointer to our controller context structure
2553  *
2554  * Returns 0 on success otherwise negative errno.
2555  */
dwc3_gadget_init(struct dwc3 * dwc)2556 int dwc3_gadget_init(struct dwc3 *dwc)
2557 {
2558 	int					ret;
2559 
2560 	dwc->ctrl_req = dma_alloc_coherent(sizeof(*dwc->ctrl_req),
2561 					(unsigned long *)&dwc->ctrl_req_addr);
2562 	if (!dwc->ctrl_req) {
2563 		dev_err(dwc->dev, "failed to allocate ctrl request\n");
2564 		ret = -ENOMEM;
2565 		goto err0;
2566 	}
2567 
2568 	dwc->ep0_trb = dma_alloc_coherent(sizeof(*dwc->ep0_trb) * 2,
2569 					  (unsigned long *)&dwc->ep0_trb_addr);
2570 	if (!dwc->ep0_trb) {
2571 		dev_err(dwc->dev, "failed to allocate ep0 trb\n");
2572 		ret = -ENOMEM;
2573 		goto err1;
2574 	}
2575 
2576 	dwc->setup_buf = memalign(CONFIG_SYS_CACHELINE_SIZE,
2577 				  DWC3_EP0_BOUNCE_SIZE);
2578 	if (!dwc->setup_buf) {
2579 		ret = -ENOMEM;
2580 		goto err2;
2581 	}
2582 
2583 	dwc->ep0_bounce = dma_alloc_coherent(DWC3_EP0_BOUNCE_SIZE,
2584 					(unsigned long *)&dwc->ep0_bounce_addr);
2585 	if (!dwc->ep0_bounce) {
2586 		dev_err(dwc->dev, "failed to allocate ep0 bounce buffer\n");
2587 		ret = -ENOMEM;
2588 		goto err3;
2589 	}
2590 
2591 	dwc->gadget.ops			= &dwc3_gadget_ops;
2592 	dwc->gadget.max_speed		= USB_SPEED_SUPER;
2593 	dwc->gadget.speed		= USB_SPEED_UNKNOWN;
2594 	dwc->gadget.name		= "dwc3-gadget";
2595 
2596 	/*
2597 	 * Per databook, DWC3 needs buffer size to be aligned to MaxPacketSize
2598 	 * on ep out.
2599 	 */
2600 	dwc->gadget.quirk_ep_out_aligned_size = true;
2601 
2602 	/*
2603 	 * REVISIT: Here we should clear all pending IRQs to be
2604 	 * sure we're starting from a well known location.
2605 	 */
2606 
2607 	ret = dwc3_gadget_init_endpoints(dwc);
2608 	if (ret)
2609 		goto err4;
2610 
2611 	ret = usb_add_gadget_udc((struct device *)dwc->dev, &dwc->gadget);
2612 	if (ret) {
2613 		dev_err(dwc->dev, "failed to register udc\n");
2614 		goto err4;
2615 	}
2616 
2617 	return 0;
2618 
2619 err4:
2620 	dwc3_gadget_free_endpoints(dwc);
2621 	dma_free_coherent(dwc->ep0_bounce);
2622 
2623 err3:
2624 	kfree(dwc->setup_buf);
2625 
2626 err2:
2627 	dma_free_coherent(dwc->ep0_trb);
2628 
2629 err1:
2630 	dma_free_coherent(dwc->ctrl_req);
2631 
2632 err0:
2633 	return ret;
2634 }
2635 
2636 /* -------------------------------------------------------------------------- */
2637 
dwc3_gadget_exit(struct dwc3 * dwc)2638 void dwc3_gadget_exit(struct dwc3 *dwc)
2639 {
2640 	usb_del_gadget_udc(&dwc->gadget);
2641 
2642 	dwc3_gadget_free_endpoints(dwc);
2643 
2644 	dma_free_coherent(dwc->ep0_bounce);
2645 
2646 	kfree(dwc->setup_buf);
2647 
2648 	dma_free_coherent(dwc->ep0_trb);
2649 
2650 	dma_free_coherent(dwc->ctrl_req);
2651 }
2652 
2653 /**
2654  * dwc3_gadget_uboot_handle_interrupt - handle dwc3 gadget interrupt
2655  * @dwc: struct dwce *
2656  *
2657  * Handles ep0 and gadget interrupt
2658  *
2659  * Should be called from dwc3 core.
2660  */
dwc3_gadget_uboot_handle_interrupt(struct dwc3 * dwc)2661 void dwc3_gadget_uboot_handle_interrupt(struct dwc3 *dwc)
2662 {
2663 	int ret = dwc3_interrupt(0, dwc);
2664 
2665 	if (ret == IRQ_WAKE_THREAD) {
2666 		int i;
2667 		struct dwc3_event_buffer *evt;
2668 
2669 		dwc3_thread_interrupt(0, dwc);
2670 
2671 		/* Clean + Invalidate the buffers after touching them */
2672 		for (i = 0; i < dwc->num_event_buffers; i++) {
2673 			evt = dwc->ev_buffs[i];
2674 			dwc3_flush_cache((uintptr_t)evt->buf, evt->length);
2675 		}
2676 	}
2677 }
2678