1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * Copyright (c) 2021 Aspeed Technology Inc.
4  */
5 
6 #include <linux/clk.h>
7 #include <linux/delay.h>
8 #include <linux/dma-mapping.h>
9 #include <linux/interrupt.h>
10 #include <linux/kernel.h>
11 #include <linux/module.h>
12 #include <linux/of.h>
13 #include <linux/platform_device.h>
14 #include <linux/prefetch.h>
15 #include <linux/usb/ch9.h>
16 #include <linux/usb/gadget.h>
17 #include <linux/slab.h>
18 
19 #define AST_UDC_NUM_ENDPOINTS		(1 + 4)
20 #define AST_UDC_EP0_MAX_PACKET		64	/* EP0's max packet size */
21 #define AST_UDC_EPn_MAX_PACKET		1024	/* Generic EPs max packet size */
22 #define AST_UDC_DESCS_COUNT		256	/* Use 256 stages descriptor mode (32/256) */
23 #define AST_UDC_DESC_MODE		1	/* Single/Multiple Stage(s) Descriptor Mode */
24 
25 #define AST_UDC_EP_DMA_SIZE		(AST_UDC_EPn_MAX_PACKET + 8 * AST_UDC_DESCS_COUNT)
26 
27 /*****************************
28  *                           *
29  * UDC register definitions  *
30  *                           *
31  *****************************/
32 
33 #define AST_UDC_FUNC_CTRL		0x00	/* Root Function Control & Status Register */
34 #define AST_UDC_CONFIG			0x04	/* Root Configuration Setting Register */
35 #define AST_UDC_IER			0x08	/* Interrupt Control Register */
36 #define AST_UDC_ISR			0x0C	/* Interrupt Status Register */
37 #define AST_UDC_EP_ACK_IER		0x10	/* Programmable ep Pool ACK Interrupt Enable Reg */
38 #define AST_UDC_EP_NAK_IER		0x14	/* Programmable ep Pool NAK Interrupt Enable Reg */
39 #define AST_UDC_EP_ACK_ISR		0x18	/* Programmable ep Pool ACK Interrupt Status Reg */
40 #define AST_UDC_EP_NAK_ISR		0x1C	/* Programmable ep Pool NAK Interrupt Status Reg */
41 #define AST_UDC_DEV_RESET		0x20	/* Device Controller Soft Reset Enable Register */
42 #define AST_UDC_STS			0x24	/* USB Status Register */
43 #define AST_VHUB_EP_DATA		0x28	/* Programmable ep Pool Data Toggle Value Set */
44 #define AST_VHUB_ISO_TX_FAIL		0x2C	/* Isochronous Transaction Fail Accumulator */
45 #define AST_UDC_EP0_CTRL		0x30	/* Endpoint 0 Control/Status Register */
46 #define AST_UDC_EP0_DATA_BUFF		0x34	/* Base Address of ep0 IN/OUT Data Buffer Reg */
47 #define AST_UDC_SETUP0			0x80    /* Root Device Setup Data Buffer0 */
48 #define AST_UDC_SETUP1			0x84    /* Root Device Setup Data Buffer1 */
49 
50 
51 /* Main control reg */
52 #define USB_PHY_CLK_EN			BIT(31)
53 #define USB_FIFO_DYN_PWRD_EN		BIT(19)
54 #define USB_EP_LONG_DESC		BIT(18)
55 #define USB_BIST_TEST_PASS		BIT(13)
56 #define USB_BIST_TURN_ON		BIT(12)
57 #define USB_PHY_RESET_DIS		BIT(11)
58 #define USB_TEST_MODE(x)		((x) << 8)
59 #define USB_FORCE_TIMER_HS		BIT(7)
60 #define USB_FORCE_HS			BIT(6)
61 #define USB_REMOTE_WAKEUP_12MS		BIT(5)
62 #define USB_REMOTE_WAKEUP_EN		BIT(4)
63 #define USB_AUTO_REMOTE_WAKEUP_EN	BIT(3)
64 #define USB_STOP_CLK_IN_SUPEND		BIT(2)
65 #define USB_UPSTREAM_FS			BIT(1)
66 #define USB_UPSTREAM_EN			BIT(0)
67 
68 /* Main config reg */
69 #define UDC_CFG_SET_ADDR(x)		((x) & 0x3f)
70 #define UDC_CFG_ADDR_MASK		(0x3f)
71 
72 /* Interrupt ctrl & status reg */
73 #define UDC_IRQ_EP_POOL_NAK		BIT(17)
74 #define UDC_IRQ_EP_POOL_ACK_STALL	BIT(16)
75 #define UDC_IRQ_BUS_RESUME		BIT(8)
76 #define UDC_IRQ_BUS_SUSPEND		BIT(7)
77 #define UDC_IRQ_BUS_RESET		BIT(6)
78 #define UDC_IRQ_EP0_IN_DATA_NAK		BIT(4)
79 #define UDC_IRQ_EP0_IN_ACK_STALL	BIT(3)
80 #define UDC_IRQ_EP0_OUT_NAK		BIT(2)
81 #define UDC_IRQ_EP0_OUT_ACK_STALL	BIT(1)
82 #define UDC_IRQ_EP0_SETUP		BIT(0)
83 #define UDC_IRQ_ACK_ALL			(0x1ff)
84 
85 /* EP isr reg */
86 #define USB_EP3_ISR			BIT(3)
87 #define USB_EP2_ISR			BIT(2)
88 #define USB_EP1_ISR			BIT(1)
89 #define USB_EP0_ISR			BIT(0)
90 #define UDC_IRQ_EP_ACK_ALL		(0xf)
91 
92 /*Soft reset reg */
93 #define ROOT_UDC_SOFT_RESET		BIT(0)
94 
95 /* USB status reg */
96 #define UDC_STS_HIGHSPEED		BIT(27)
97 
98 /* Programmable EP data toggle */
99 #define EP_TOGGLE_SET_EPNUM(x)		((x) & 0x3)
100 
101 /* EP0 ctrl reg */
102 #define EP0_GET_RX_LEN(x)		((x >> 16) & 0x7f)
103 #define EP0_TX_LEN(x)			((x & 0x7f) << 8)
104 #define EP0_RX_BUFF_RDY			BIT(2)
105 #define EP0_TX_BUFF_RDY			BIT(1)
106 #define EP0_STALL			BIT(0)
107 
108 /*************************************
109  *                                   *
110  * per-endpoint register definitions *
111  *                                   *
112  *************************************/
113 
114 #define AST_UDC_EP_CONFIG		0x00	/* Endpoint Configuration Register */
115 #define AST_UDC_EP_DMA_CTRL		0x04	/* DMA Descriptor List Control/Status Register */
116 #define AST_UDC_EP_DMA_BUFF		0x08	/* DMA Descriptor/Buffer Base Address */
117 #define AST_UDC_EP_DMA_STS		0x0C	/* DMA Descriptor List R/W Pointer and Status */
118 
119 #define AST_UDC_EP_BASE			0x200
120 #define AST_UDC_EP_OFFSET		0x10
121 
122 /* EP config reg */
123 #define EP_SET_MAX_PKT(x)		((x & 0x3ff) << 16)
124 #define EP_DATA_FETCH_CTRL(x)		((x & 0x3) << 14)
125 #define EP_AUTO_DATA_DISABLE		(0x1 << 13)
126 #define EP_SET_EP_STALL			(0x1 << 12)
127 #define EP_SET_EP_NUM(x)		((x & 0xf) << 8)
128 #define EP_SET_TYPE_MASK(x)		((x) << 5)
129 #define EP_TYPE_BULK			(0x1)
130 #define EP_TYPE_INT			(0x2)
131 #define EP_TYPE_ISO			(0x3)
132 #define EP_DIR_OUT			(0x1 << 4)
133 #define EP_ALLOCATED_MASK		(0x7 << 1)
134 #define EP_ENABLE			BIT(0)
135 
136 /* EP DMA ctrl reg */
137 #define EP_DMA_CTRL_GET_PROC_STS(x)	((x >> 4) & 0xf)
138 #define EP_DMA_CTRL_STS_RX_IDLE		0x0
139 #define EP_DMA_CTRL_STS_TX_IDLE		0x8
140 #define EP_DMA_CTRL_IN_LONG_MODE	(0x1 << 3)
141 #define EP_DMA_CTRL_RESET		(0x1 << 2)
142 #define EP_DMA_SINGLE_STAGE		(0x1 << 1)
143 #define EP_DMA_DESC_MODE		(0x1 << 0)
144 
145 /* EP DMA status reg */
146 #define EP_DMA_SET_TX_SIZE(x)		((x & 0x7ff) << 16)
147 #define EP_DMA_GET_TX_SIZE(x)		(((x) >> 16) & 0x7ff)
148 #define EP_DMA_GET_RPTR(x)		(((x) >> 8) & 0xff)
149 #define EP_DMA_GET_WPTR(x)		((x) & 0xff)
150 #define EP_DMA_SINGLE_KICK		(1 << 0) /* WPTR = 1 for single mode */
151 
152 /* EP desc reg */
153 #define AST_EP_DMA_DESC_INTR_ENABLE	BIT(31)
154 #define AST_EP_DMA_DESC_PID_DATA0	(0 << 14)
155 #define AST_EP_DMA_DESC_PID_DATA2	BIT(14)
156 #define AST_EP_DMA_DESC_PID_DATA1	(2 << 14)
157 #define AST_EP_DMA_DESC_PID_MDATA	(3 << 14)
158 #define EP_DESC1_IN_LEN(x)		((x) & 0x1fff)
159 #define AST_EP_DMA_DESC_MAX_LEN		(7680) /* Max packet length for trasmit in 1 desc */
160 
161 struct ast_udc_request {
162 	struct usb_request	req;
163 	struct list_head	queue;
164 	unsigned		mapped:1;
165 	unsigned int		actual_dma_length;
166 	u32			saved_dma_wptr;
167 };
168 
169 #define to_ast_req(__req) container_of(__req, struct ast_udc_request, req)
170 
171 struct ast_dma_desc {
172 	u32	des_0;
173 	u32	des_1;
174 };
175 
176 struct ast_udc_ep {
177 	struct usb_ep			ep;
178 
179 	/* Request queue */
180 	struct list_head		queue;
181 
182 	struct ast_udc_dev		*udc;
183 	void __iomem			*ep_reg;
184 	void				*epn_buf;
185 	dma_addr_t			epn_buf_dma;
186 	const struct usb_endpoint_descriptor	*desc;
187 
188 	/* DMA Descriptors */
189 	struct ast_dma_desc		*descs;
190 	dma_addr_t			descs_dma;
191 	u32				descs_wptr;
192 	u32				chunk_max;
193 
194 	bool				dir_in:1;
195 	unsigned			stopped:1;
196 	bool				desc_mode:1;
197 };
198 
199 #define to_ast_ep(__ep) container_of(__ep, struct ast_udc_ep, ep)
200 
201 struct ast_udc_dev {
202 	struct platform_device		*pdev;
203 	void __iomem			*reg;
204 	int				irq;
205 	spinlock_t			lock;
206 	struct clk			*clk;
207 	struct work_struct		wake_work;
208 
209 	/* EP0 DMA buffers allocated in one chunk */
210 	void				*ep0_buf;
211 	dma_addr_t			ep0_buf_dma;
212 	struct ast_udc_ep		ep[AST_UDC_NUM_ENDPOINTS];
213 
214 	struct usb_gadget		gadget;
215 	struct usb_gadget_driver	*driver;
216 	void __iomem			*creq;
217 	enum usb_device_state		suspended_from;
218 	int				desc_mode;
219 
220 	/* Force full speed only */
221 	bool				force_usb1:1;
222 	unsigned			is_control_tx:1;
223 	bool				wakeup_en:1;
224 };
225 
226 #define to_ast_dev(__g) container_of(__g, struct ast_udc_dev, gadget)
227 
228 static const char * const ast_ep_name[] = {
229 	"ep0", "ep1", "ep2", "ep3", "ep4"
230 };
231 
232 #ifdef AST_UDC_DEBUG_ALL
233 #define AST_UDC_DEBUG
234 #define AST_SETUP_DEBUG
235 #define AST_EP_DEBUG
236 #define AST_ISR_DEBUG
237 #endif
238 
239 #ifdef AST_SETUP_DEBUG
240 #define SETUP_DBG(u, fmt, ...)	\
241 	dev_dbg(&(u)->pdev->dev, "%s() " fmt, __func__, ##__VA_ARGS__)
242 #else
243 #define SETUP_DBG(u, fmt, ...)
244 #endif
245 
246 #ifdef AST_EP_DEBUG
247 #define EP_DBG(e, fmt, ...)	\
248 	dev_dbg(&(e)->udc->pdev->dev, "%s():%s " fmt, __func__,	\
249 		 (e)->ep.name, ##__VA_ARGS__)
250 #else
251 #define EP_DBG(ep, fmt, ...)	((void)(ep))
252 #endif
253 
254 #ifdef AST_UDC_DEBUG
255 #define UDC_DBG(u, fmt, ...)	\
256 	dev_dbg(&(u)->pdev->dev, "%s() " fmt, __func__, ##__VA_ARGS__)
257 #else
258 #define UDC_DBG(u, fmt, ...)
259 #endif
260 
261 #ifdef AST_ISR_DEBUG
262 #define ISR_DBG(u, fmt, ...)	\
263 	dev_dbg(&(u)->pdev->dev, "%s() " fmt, __func__, ##__VA_ARGS__)
264 #else
265 #define ISR_DBG(u, fmt, ...)
266 #endif
267 
268 /*-------------------------------------------------------------------------*/
269 #define ast_udc_read(udc, offset) \
270 	readl((udc)->reg + (offset))
271 #define ast_udc_write(udc, val, offset) \
272 	writel((val), (udc)->reg + (offset))
273 
274 #define ast_ep_read(ep, reg) \
275 	readl((ep)->ep_reg + (reg))
276 #define ast_ep_write(ep, val, reg) \
277 	writel((val), (ep)->ep_reg + (reg))
278 
279 /*-------------------------------------------------------------------------*/
280 
281 static void ast_udc_done(struct ast_udc_ep *ep, struct ast_udc_request *req,
282 			 int status)
283 {
284 	struct ast_udc_dev *udc = ep->udc;
285 
286 	EP_DBG(ep, "req @%p, len (%d/%d), buf:0x%x, dir:0x%x\n",
287 	       req, req->req.actual, req->req.length,
288 	       (u32)req->req.buf, ep->dir_in);
289 
290 	list_del(&req->queue);
291 
292 	if (req->req.status == -EINPROGRESS)
293 		req->req.status = status;
294 	else
295 		status = req->req.status;
296 
297 	if (status && status != -ESHUTDOWN)
298 		EP_DBG(ep, "done req:%p, status:%d\n", req, status);
299 
300 	spin_unlock(&udc->lock);
301 	usb_gadget_giveback_request(&ep->ep, &req->req);
302 	spin_lock(&udc->lock);
303 }
304 
305 static void ast_udc_nuke(struct ast_udc_ep *ep, int status)
306 {
307 	int count = 0;
308 
309 	while (!list_empty(&ep->queue)) {
310 		struct ast_udc_request *req;
311 
312 		req = list_entry(ep->queue.next, struct ast_udc_request,
313 				 queue);
314 		ast_udc_done(ep, req, status);
315 		count++;
316 	}
317 
318 	if (count)
319 		EP_DBG(ep, "Nuked %d request(s)\n", count);
320 }
321 
322 /*
323  * Stop activity on all endpoints.
324  * Device controller for which EP activity is to be stopped.
325  *
326  * All the endpoints are stopped and any pending transfer requests if any on
327  * the endpoint are terminated.
328  */
329 static void ast_udc_stop_activity(struct ast_udc_dev *udc)
330 {
331 	struct ast_udc_ep *ep;
332 	int i;
333 
334 	for (i = 0; i < AST_UDC_NUM_ENDPOINTS; i++) {
335 		ep = &udc->ep[i];
336 		ep->stopped = 1;
337 		ast_udc_nuke(ep, -ESHUTDOWN);
338 	}
339 }
340 
341 static int ast_udc_ep_enable(struct usb_ep *_ep,
342 			     const struct usb_endpoint_descriptor *desc)
343 {
344 	u16 maxpacket = usb_endpoint_maxp(desc);
345 	struct ast_udc_ep *ep = to_ast_ep(_ep);
346 	struct ast_udc_dev *udc = ep->udc;
347 	u8 epnum = usb_endpoint_num(desc);
348 	unsigned long flags;
349 	u32 ep_conf = 0;
350 	u8 dir_in;
351 	u8 type;
352 
353 	if (!_ep || !ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
354 	    maxpacket == 0 || maxpacket > ep->ep.maxpacket) {
355 		EP_DBG(ep, "Failed, invalid EP enable param\n");
356 		return -EINVAL;
357 	}
358 
359 	if (!udc->driver) {
360 		EP_DBG(ep, "bogus device state\n");
361 		return -ESHUTDOWN;
362 	}
363 
364 	EP_DBG(ep, "maxpacket:0x%x\n", maxpacket);
365 
366 	spin_lock_irqsave(&udc->lock, flags);
367 
368 	ep->desc = desc;
369 	ep->stopped = 0;
370 	ep->ep.maxpacket = maxpacket;
371 	ep->chunk_max = AST_EP_DMA_DESC_MAX_LEN;
372 
373 	if (maxpacket < AST_UDC_EPn_MAX_PACKET)
374 		ep_conf = EP_SET_MAX_PKT(maxpacket);
375 
376 	ep_conf |= EP_SET_EP_NUM(epnum);
377 
378 	type = usb_endpoint_type(desc);
379 	dir_in = usb_endpoint_dir_in(desc);
380 	ep->dir_in = dir_in;
381 	if (!ep->dir_in)
382 		ep_conf |= EP_DIR_OUT;
383 
384 	EP_DBG(ep, "type %d, dir_in %d\n", type, dir_in);
385 	switch (type) {
386 	case USB_ENDPOINT_XFER_ISOC:
387 		ep_conf |= EP_SET_TYPE_MASK(EP_TYPE_ISO);
388 		break;
389 
390 	case USB_ENDPOINT_XFER_BULK:
391 		ep_conf |= EP_SET_TYPE_MASK(EP_TYPE_BULK);
392 		break;
393 
394 	case USB_ENDPOINT_XFER_INT:
395 		ep_conf |= EP_SET_TYPE_MASK(EP_TYPE_INT);
396 		break;
397 	}
398 
399 	ep->desc_mode = udc->desc_mode && ep->descs_dma && ep->dir_in;
400 	if (ep->desc_mode) {
401 		ast_ep_write(ep, EP_DMA_CTRL_RESET, AST_UDC_EP_DMA_CTRL);
402 		ast_ep_write(ep, 0, AST_UDC_EP_DMA_STS);
403 		ast_ep_write(ep, ep->descs_dma, AST_UDC_EP_DMA_BUFF);
404 
405 		/* Enable Long Descriptor Mode */
406 		ast_ep_write(ep, EP_DMA_CTRL_IN_LONG_MODE | EP_DMA_DESC_MODE,
407 			     AST_UDC_EP_DMA_CTRL);
408 
409 		ep->descs_wptr = 0;
410 
411 	} else {
412 		ast_ep_write(ep, EP_DMA_CTRL_RESET, AST_UDC_EP_DMA_CTRL);
413 		ast_ep_write(ep, EP_DMA_SINGLE_STAGE, AST_UDC_EP_DMA_CTRL);
414 		ast_ep_write(ep, 0, AST_UDC_EP_DMA_STS);
415 	}
416 
417 	/* Cleanup data toggle just in case */
418 	ast_udc_write(udc, EP_TOGGLE_SET_EPNUM(epnum), AST_VHUB_EP_DATA);
419 
420 	/* Enable EP */
421 	ast_ep_write(ep, ep_conf | EP_ENABLE, AST_UDC_EP_CONFIG);
422 
423 	EP_DBG(ep, "ep_config: 0x%x\n", ast_ep_read(ep, AST_UDC_EP_CONFIG));
424 
425 	spin_unlock_irqrestore(&udc->lock, flags);
426 
427 	return 0;
428 }
429 
430 static int ast_udc_ep_disable(struct usb_ep *_ep)
431 {
432 	struct ast_udc_ep *ep = to_ast_ep(_ep);
433 	struct ast_udc_dev *udc = ep->udc;
434 	unsigned long flags;
435 
436 	spin_lock_irqsave(&udc->lock, flags);
437 
438 	ep->ep.desc = NULL;
439 	ep->stopped = 1;
440 
441 	ast_udc_nuke(ep, -ESHUTDOWN);
442 	ast_ep_write(ep, 0, AST_UDC_EP_CONFIG);
443 
444 	spin_unlock_irqrestore(&udc->lock, flags);
445 
446 	return 0;
447 }
448 
449 static struct usb_request *ast_udc_ep_alloc_request(struct usb_ep *_ep,
450 						    gfp_t gfp_flags)
451 {
452 	struct ast_udc_ep *ep = to_ast_ep(_ep);
453 	struct ast_udc_request *req;
454 
455 	req = kzalloc(sizeof(struct ast_udc_request), gfp_flags);
456 	if (!req) {
457 		EP_DBG(ep, "request allocation failed\n");
458 		return NULL;
459 	}
460 
461 	INIT_LIST_HEAD(&req->queue);
462 
463 	return &req->req;
464 }
465 
466 static void ast_udc_ep_free_request(struct usb_ep *_ep,
467 				    struct usb_request *_req)
468 {
469 	struct ast_udc_request *req = to_ast_req(_req);
470 
471 	kfree(req);
472 }
473 
474 static int ast_dma_descriptor_setup(struct ast_udc_ep *ep, u32 dma_buf,
475 				    u16 tx_len, struct ast_udc_request *req)
476 {
477 	struct ast_udc_dev *udc = ep->udc;
478 	struct device *dev = &udc->pdev->dev;
479 	bool last = false;
480 	int chunk, count;
481 	u32 offset;
482 
483 	if (!ep->descs) {
484 		dev_warn(dev, "%s: Empty DMA descs list failure\n",
485 			 ep->ep.name);
486 		return -EINVAL;
487 	}
488 
489 	chunk = tx_len;
490 	offset = count = 0;
491 
492 	EP_DBG(ep, "req @%p, %s:%d, %s:0x%x, %s:0x%x\n", req,
493 	       "wptr", ep->descs_wptr, "dma_buf", dma_buf,
494 	       "tx_len", tx_len);
495 
496 	/* Create Descriptor Lists */
497 	while (chunk >= 0 && !last && count < AST_UDC_DESCS_COUNT) {
498 
499 		ep->descs[ep->descs_wptr].des_0 = dma_buf + offset;
500 
501 		if (chunk > ep->chunk_max) {
502 			ep->descs[ep->descs_wptr].des_1 = ep->chunk_max;
503 		} else {
504 			ep->descs[ep->descs_wptr].des_1 = chunk;
505 			last = true;
506 		}
507 
508 		chunk -= ep->chunk_max;
509 
510 		EP_DBG(ep, "descs[%d]: 0x%x 0x%x\n",
511 		       ep->descs_wptr,
512 		       ep->descs[ep->descs_wptr].des_0,
513 		       ep->descs[ep->descs_wptr].des_1);
514 
515 		if (count == 0)
516 			req->saved_dma_wptr = ep->descs_wptr;
517 
518 		ep->descs_wptr++;
519 		count++;
520 
521 		if (ep->descs_wptr >= AST_UDC_DESCS_COUNT)
522 			ep->descs_wptr = 0;
523 
524 		offset = ep->chunk_max * count;
525 	}
526 
527 	return 0;
528 }
529 
530 static void ast_udc_epn_kick(struct ast_udc_ep *ep, struct ast_udc_request *req)
531 {
532 	u32 tx_len;
533 	u32 last;
534 
535 	last = req->req.length - req->req.actual;
536 	tx_len = last > ep->ep.maxpacket ? ep->ep.maxpacket : last;
537 
538 	EP_DBG(ep, "kick req @%p, len:%d, dir:%d\n",
539 	       req, tx_len, ep->dir_in);
540 
541 	ast_ep_write(ep, req->req.dma + req->req.actual, AST_UDC_EP_DMA_BUFF);
542 
543 	/* Start DMA */
544 	ast_ep_write(ep, EP_DMA_SET_TX_SIZE(tx_len), AST_UDC_EP_DMA_STS);
545 	ast_ep_write(ep, EP_DMA_SET_TX_SIZE(tx_len) | EP_DMA_SINGLE_KICK,
546 		     AST_UDC_EP_DMA_STS);
547 }
548 
549 static void ast_udc_epn_kick_desc(struct ast_udc_ep *ep,
550 				  struct ast_udc_request *req)
551 {
552 	u32 descs_max_size;
553 	u32 tx_len;
554 	u32 last;
555 
556 	descs_max_size = AST_EP_DMA_DESC_MAX_LEN * AST_UDC_DESCS_COUNT;
557 
558 	last = req->req.length - req->req.actual;
559 	tx_len = last > descs_max_size ? descs_max_size : last;
560 
561 	EP_DBG(ep, "kick req @%p, %s:%d, %s:0x%x, %s:0x%x (%d/%d), %s:0x%x\n",
562 	       req, "tx_len", tx_len, "dir_in", ep->dir_in,
563 	       "dma", req->req.dma + req->req.actual,
564 	       req->req.actual, req->req.length,
565 	       "descs_max_size", descs_max_size);
566 
567 	if (!ast_dma_descriptor_setup(ep, req->req.dma + req->req.actual,
568 				      tx_len, req))
569 		req->actual_dma_length += tx_len;
570 
571 	/* make sure CPU done everything before triggering DMA */
572 	mb();
573 
574 	ast_ep_write(ep, ep->descs_wptr, AST_UDC_EP_DMA_STS);
575 
576 	EP_DBG(ep, "descs_wptr:%d, dstat:0x%x, dctrl:0x%x\n",
577 	       ep->descs_wptr,
578 	       ast_ep_read(ep, AST_UDC_EP_DMA_STS),
579 	       ast_ep_read(ep, AST_UDC_EP_DMA_CTRL));
580 }
581 
582 static void ast_udc_ep0_queue(struct ast_udc_ep *ep,
583 			      struct ast_udc_request *req)
584 {
585 	struct ast_udc_dev *udc = ep->udc;
586 	u32 tx_len;
587 	u32 last;
588 
589 	last = req->req.length - req->req.actual;
590 	tx_len = last > ep->ep.maxpacket ? ep->ep.maxpacket : last;
591 
592 	ast_udc_write(udc, req->req.dma + req->req.actual,
593 		      AST_UDC_EP0_DATA_BUFF);
594 
595 	if (ep->dir_in) {
596 		/* IN requests, send data */
597 		SETUP_DBG(udc, "IN: %s:0x%x, %s:0x%x, %s:%d (%d/%d), %s:%d\n",
598 			  "buf", (u32)req->req.buf,
599 			  "dma", req->req.dma + req->req.actual,
600 			  "tx_len", tx_len,
601 			  req->req.actual, req->req.length,
602 			  "dir_in", ep->dir_in);
603 
604 		req->req.actual += tx_len;
605 		ast_udc_write(udc, EP0_TX_LEN(tx_len), AST_UDC_EP0_CTRL);
606 		ast_udc_write(udc, EP0_TX_LEN(tx_len) | EP0_TX_BUFF_RDY,
607 			      AST_UDC_EP0_CTRL);
608 
609 	} else {
610 		/* OUT requests, receive data */
611 		SETUP_DBG(udc, "OUT: %s:%x, %s:%x, %s:(%d/%d), %s:%d\n",
612 			  "buf", (u32)req->req.buf,
613 			  "dma", req->req.dma + req->req.actual,
614 			  "len", req->req.actual, req->req.length,
615 			  "dir_in", ep->dir_in);
616 
617 		if (!req->req.length) {
618 			/* 0 len request, send tx as completion */
619 			ast_udc_write(udc, EP0_TX_BUFF_RDY, AST_UDC_EP0_CTRL);
620 			ep->dir_in = 0x1;
621 		} else
622 			ast_udc_write(udc, EP0_RX_BUFF_RDY, AST_UDC_EP0_CTRL);
623 	}
624 }
625 
626 static int ast_udc_ep_queue(struct usb_ep *_ep, struct usb_request *_req,
627 			    gfp_t gfp_flags)
628 {
629 	struct ast_udc_request *req = to_ast_req(_req);
630 	struct ast_udc_ep *ep = to_ast_ep(_ep);
631 	struct ast_udc_dev *udc = ep->udc;
632 	struct device *dev = &udc->pdev->dev;
633 	unsigned long flags;
634 	int rc;
635 
636 	if (unlikely(!_req || !_req->complete || !_req->buf || !_ep)) {
637 		dev_warn(dev, "Invalid EP request !\n");
638 		return -EINVAL;
639 	}
640 
641 	if (ep->stopped) {
642 		dev_warn(dev, "%s is already stopped !\n", _ep->name);
643 		return -ESHUTDOWN;
644 	}
645 
646 	spin_lock_irqsave(&udc->lock, flags);
647 
648 	list_add_tail(&req->queue, &ep->queue);
649 
650 	req->req.actual = 0;
651 	req->req.status = -EINPROGRESS;
652 	req->actual_dma_length = 0;
653 
654 	rc = usb_gadget_map_request(&udc->gadget, &req->req, ep->dir_in);
655 	if (rc) {
656 		EP_DBG(ep, "Request mapping failure %d\n", rc);
657 		dev_warn(dev, "Request mapping failure %d\n", rc);
658 		goto end;
659 	}
660 
661 	EP_DBG(ep, "enqueue req @%p\n", req);
662 	EP_DBG(ep, "l=%d, dma:0x%x, zero:%d, is_in:%d\n",
663 		_req->length, _req->dma, _req->zero, ep->dir_in);
664 
665 	/* EP0 request enqueue */
666 	if (ep->ep.desc == NULL) {
667 		if ((req->req.dma % 4) != 0) {
668 			dev_warn(dev, "EP0 req dma alignment error\n");
669 			rc = -ESHUTDOWN;
670 			goto end;
671 		}
672 
673 		ast_udc_ep0_queue(ep, req);
674 		goto end;
675 	}
676 
677 	/* EPn request enqueue */
678 	if (list_is_singular(&ep->queue)) {
679 		if (ep->desc_mode)
680 			ast_udc_epn_kick_desc(ep, req);
681 		else
682 			ast_udc_epn_kick(ep, req);
683 	}
684 
685 end:
686 	spin_unlock_irqrestore(&udc->lock, flags);
687 
688 	return rc;
689 }
690 
691 static int ast_udc_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
692 {
693 	struct ast_udc_ep *ep = to_ast_ep(_ep);
694 	struct ast_udc_dev *udc = ep->udc;
695 	struct ast_udc_request *req;
696 	unsigned long flags;
697 	int rc = 0;
698 
699 	spin_lock_irqsave(&udc->lock, flags);
700 
701 	/* make sure it's actually queued on this endpoint */
702 	list_for_each_entry(req, &ep->queue, queue) {
703 		if (&req->req == _req) {
704 			list_del_init(&req->queue);
705 			ast_udc_done(ep, req, -ESHUTDOWN);
706 			_req->status = -ECONNRESET;
707 			break;
708 		}
709 	}
710 
711 	/* dequeue request not found */
712 	if (&req->req != _req)
713 		rc = -EINVAL;
714 
715 	spin_unlock_irqrestore(&udc->lock, flags);
716 
717 	return rc;
718 }
719 
720 static int ast_udc_ep_set_halt(struct usb_ep *_ep, int value)
721 {
722 	struct ast_udc_ep *ep = to_ast_ep(_ep);
723 	struct ast_udc_dev *udc = ep->udc;
724 	unsigned long flags;
725 	int epnum;
726 	u32 ctrl;
727 
728 	EP_DBG(ep, "val:%d\n", value);
729 
730 	spin_lock_irqsave(&udc->lock, flags);
731 
732 	epnum = usb_endpoint_num(ep->desc);
733 
734 	/* EP0 */
735 	if (epnum == 0) {
736 		ctrl = ast_udc_read(udc, AST_UDC_EP0_CTRL);
737 		if (value)
738 			ctrl |= EP0_STALL;
739 		else
740 			ctrl &= ~EP0_STALL;
741 
742 		ast_udc_write(udc, ctrl, AST_UDC_EP0_CTRL);
743 
744 	} else {
745 	/* EPn */
746 		ctrl = ast_udc_read(udc, AST_UDC_EP_CONFIG);
747 		if (value)
748 			ctrl |= EP_SET_EP_STALL;
749 		else
750 			ctrl &= ~EP_SET_EP_STALL;
751 
752 		ast_ep_write(ep, ctrl, AST_UDC_EP_CONFIG);
753 
754 		/* only epn is stopped and waits for clear */
755 		ep->stopped = value ? 1 : 0;
756 	}
757 
758 	spin_unlock_irqrestore(&udc->lock, flags);
759 
760 	return 0;
761 }
762 
763 static const struct usb_ep_ops ast_udc_ep_ops = {
764 	.enable		= ast_udc_ep_enable,
765 	.disable	= ast_udc_ep_disable,
766 	.alloc_request	= ast_udc_ep_alloc_request,
767 	.free_request	= ast_udc_ep_free_request,
768 	.queue		= ast_udc_ep_queue,
769 	.dequeue	= ast_udc_ep_dequeue,
770 	.set_halt	= ast_udc_ep_set_halt,
771 	/* there's only imprecise fifo status reporting */
772 };
773 
774 static void ast_udc_ep0_rx(struct ast_udc_dev *udc)
775 {
776 	ast_udc_write(udc, udc->ep0_buf_dma, AST_UDC_EP0_DATA_BUFF);
777 	ast_udc_write(udc, EP0_RX_BUFF_RDY, AST_UDC_EP0_CTRL);
778 }
779 
780 static void ast_udc_ep0_tx(struct ast_udc_dev *udc)
781 {
782 	ast_udc_write(udc, udc->ep0_buf_dma, AST_UDC_EP0_DATA_BUFF);
783 	ast_udc_write(udc, EP0_TX_BUFF_RDY, AST_UDC_EP0_CTRL);
784 }
785 
786 static void ast_udc_ep0_out(struct ast_udc_dev *udc)
787 {
788 	struct device *dev = &udc->pdev->dev;
789 	struct ast_udc_ep *ep = &udc->ep[0];
790 	struct ast_udc_request *req;
791 	u16 rx_len;
792 
793 	if (list_empty(&ep->queue))
794 		return;
795 
796 	req = list_entry(ep->queue.next, struct ast_udc_request, queue);
797 
798 	rx_len = EP0_GET_RX_LEN(ast_udc_read(udc, AST_UDC_EP0_CTRL));
799 	req->req.actual += rx_len;
800 
801 	SETUP_DBG(udc, "req %p (%d/%d)\n", req,
802 		  req->req.actual, req->req.length);
803 
804 	if ((rx_len < ep->ep.maxpacket) ||
805 	    (req->req.actual == req->req.length)) {
806 		ast_udc_ep0_tx(udc);
807 		if (!ep->dir_in)
808 			ast_udc_done(ep, req, 0);
809 
810 	} else {
811 		if (rx_len > req->req.length) {
812 			// Issue Fix
813 			dev_warn(dev, "Something wrong (%d/%d)\n",
814 				 req->req.actual, req->req.length);
815 			ast_udc_ep0_tx(udc);
816 			ast_udc_done(ep, req, 0);
817 			return;
818 		}
819 
820 		ep->dir_in = 0;
821 
822 		/* More works */
823 		ast_udc_ep0_queue(ep, req);
824 	}
825 }
826 
827 static void ast_udc_ep0_in(struct ast_udc_dev *udc)
828 {
829 	struct ast_udc_ep *ep = &udc->ep[0];
830 	struct ast_udc_request *req;
831 
832 	if (list_empty(&ep->queue)) {
833 		if (udc->is_control_tx) {
834 			ast_udc_ep0_rx(udc);
835 			udc->is_control_tx = 0;
836 		}
837 
838 		return;
839 	}
840 
841 	req = list_entry(ep->queue.next, struct ast_udc_request, queue);
842 
843 	SETUP_DBG(udc, "req %p (%d/%d)\n", req,
844 		  req->req.actual, req->req.length);
845 
846 	if (req->req.length == req->req.actual) {
847 		if (req->req.length)
848 			ast_udc_ep0_rx(udc);
849 
850 		if (ep->dir_in)
851 			ast_udc_done(ep, req, 0);
852 
853 	} else {
854 		/* More works */
855 		ast_udc_ep0_queue(ep, req);
856 	}
857 }
858 
859 static void ast_udc_epn_handle(struct ast_udc_dev *udc, u16 ep_num)
860 {
861 	struct ast_udc_ep *ep = &udc->ep[ep_num];
862 	struct ast_udc_request *req;
863 	u16 len = 0;
864 
865 	if (list_empty(&ep->queue))
866 		return;
867 
868 	req = list_first_entry(&ep->queue, struct ast_udc_request, queue);
869 
870 	len = EP_DMA_GET_TX_SIZE(ast_ep_read(ep, AST_UDC_EP_DMA_STS));
871 	req->req.actual += len;
872 
873 	EP_DBG(ep, "req @%p, length:(%d/%d), %s:0x%x\n", req,
874 		req->req.actual, req->req.length, "len", len);
875 
876 	/* Done this request */
877 	if (req->req.length == req->req.actual) {
878 		ast_udc_done(ep, req, 0);
879 		req = list_first_entry_or_null(&ep->queue,
880 					       struct ast_udc_request,
881 					       queue);
882 
883 	} else {
884 		/* Check for short packet */
885 		if (len < ep->ep.maxpacket) {
886 			ast_udc_done(ep, req, 0);
887 			req = list_first_entry_or_null(&ep->queue,
888 						       struct ast_udc_request,
889 						       queue);
890 		}
891 	}
892 
893 	/* More requests */
894 	if (req)
895 		ast_udc_epn_kick(ep, req);
896 }
897 
898 static void ast_udc_epn_handle_desc(struct ast_udc_dev *udc, u16 ep_num)
899 {
900 	struct ast_udc_ep *ep = &udc->ep[ep_num];
901 	struct device *dev = &udc->pdev->dev;
902 	struct ast_udc_request *req;
903 	u32 proc_sts, wr_ptr, rd_ptr;
904 	u32 len_in_desc, ctrl;
905 	u16 total_len = 0;
906 	int i;
907 
908 	if (list_empty(&ep->queue)) {
909 		dev_warn(dev, "%s request queue empty!\n", ep->ep.name);
910 		return;
911 	}
912 
913 	req = list_first_entry(&ep->queue, struct ast_udc_request, queue);
914 
915 	ctrl = ast_ep_read(ep, AST_UDC_EP_DMA_CTRL);
916 	proc_sts = EP_DMA_CTRL_GET_PROC_STS(ctrl);
917 
918 	/* Check processing status is idle */
919 	if (proc_sts != EP_DMA_CTRL_STS_RX_IDLE &&
920 	    proc_sts != EP_DMA_CTRL_STS_TX_IDLE) {
921 		dev_warn(dev, "EP DMA CTRL: 0x%x, PS:0x%x\n",
922 			 ast_ep_read(ep, AST_UDC_EP_DMA_CTRL),
923 			 proc_sts);
924 		return;
925 	}
926 
927 	ctrl = ast_ep_read(ep, AST_UDC_EP_DMA_STS);
928 	rd_ptr = EP_DMA_GET_RPTR(ctrl);
929 	wr_ptr = EP_DMA_GET_WPTR(ctrl);
930 
931 	if (rd_ptr != wr_ptr) {
932 		dev_warn(dev, "desc list is not empty ! %s:%d, %s:%d\n",
933 		"rptr", rd_ptr, "wptr", wr_ptr);
934 		return;
935 	}
936 
937 	EP_DBG(ep, "rd_ptr:%d, wr_ptr:%d\n", rd_ptr, wr_ptr);
938 	i = req->saved_dma_wptr;
939 
940 	do {
941 		len_in_desc = EP_DESC1_IN_LEN(ep->descs[i].des_1);
942 		EP_DBG(ep, "desc[%d] len: %d\n", i, len_in_desc);
943 		total_len += len_in_desc;
944 		i++;
945 		if (i >= AST_UDC_DESCS_COUNT)
946 			i = 0;
947 
948 	} while (i != wr_ptr);
949 
950 	req->req.actual += total_len;
951 
952 	EP_DBG(ep, "req @%p, length:(%d/%d), %s:0x%x\n", req,
953 		req->req.actual, req->req.length, "len", total_len);
954 
955 	/* Done this request */
956 	if (req->req.length == req->req.actual) {
957 		ast_udc_done(ep, req, 0);
958 		req = list_first_entry_or_null(&ep->queue,
959 					       struct ast_udc_request,
960 					       queue);
961 
962 	} else {
963 		/* Check for short packet */
964 		if (total_len < ep->ep.maxpacket) {
965 			ast_udc_done(ep, req, 0);
966 			req = list_first_entry_or_null(&ep->queue,
967 						       struct ast_udc_request,
968 						       queue);
969 		}
970 	}
971 
972 	/* More requests & dma descs not setup yet */
973 	if (req && (req->actual_dma_length == req->req.actual)) {
974 		EP_DBG(ep, "More requests\n");
975 		ast_udc_epn_kick_desc(ep, req);
976 	}
977 }
978 
979 static void ast_udc_ep0_data_tx(struct ast_udc_dev *udc, u8 *tx_data, u32 len)
980 {
981 	if (len) {
982 		memcpy(udc->ep0_buf, tx_data, len);
983 
984 		ast_udc_write(udc, udc->ep0_buf_dma, AST_UDC_EP0_DATA_BUFF);
985 		ast_udc_write(udc, EP0_TX_LEN(len), AST_UDC_EP0_CTRL);
986 		ast_udc_write(udc, EP0_TX_LEN(len) | EP0_TX_BUFF_RDY,
987 			      AST_UDC_EP0_CTRL);
988 		udc->is_control_tx = 1;
989 
990 	} else
991 		ast_udc_write(udc, EP0_TX_BUFF_RDY, AST_UDC_EP0_CTRL);
992 }
993 
994 static void ast_udc_getstatus(struct ast_udc_dev *udc)
995 {
996 	struct usb_ctrlrequest crq;
997 	struct ast_udc_ep *ep;
998 	u16 status = 0;
999 	u16 epnum = 0;
1000 
1001 	memcpy_fromio(&crq, udc->creq, sizeof(crq));
1002 
1003 	switch (crq.bRequestType & USB_RECIP_MASK) {
1004 	case USB_RECIP_DEVICE:
1005 		/* Get device status */
1006 		status = 1 << USB_DEVICE_SELF_POWERED;
1007 		break;
1008 	case USB_RECIP_INTERFACE:
1009 		break;
1010 	case USB_RECIP_ENDPOINT:
1011 		epnum = crq.wIndex & USB_ENDPOINT_NUMBER_MASK;
1012 		status = udc->ep[epnum].stopped;
1013 		break;
1014 	default:
1015 		goto stall;
1016 	}
1017 
1018 	ep = &udc->ep[epnum];
1019 	EP_DBG(ep, "status: 0x%x\n", status);
1020 	ast_udc_ep0_data_tx(udc, (u8 *)&status, sizeof(status));
1021 
1022 	return;
1023 
1024 stall:
1025 	EP_DBG(ep, "Can't respond request\n");
1026 	ast_udc_write(udc, ast_udc_read(udc, AST_UDC_EP0_CTRL) | EP0_STALL,
1027 		      AST_UDC_EP0_CTRL);
1028 }
1029 
1030 static void ast_udc_ep0_handle_setup(struct ast_udc_dev *udc)
1031 {
1032 	struct ast_udc_ep *ep = &udc->ep[0];
1033 	struct ast_udc_request *req;
1034 	struct usb_ctrlrequest crq;
1035 	int req_num = 0;
1036 	int rc = 0;
1037 	u32 reg;
1038 
1039 	memcpy_fromio(&crq, udc->creq, sizeof(crq));
1040 
1041 	SETUP_DBG(udc, "SETUP packet: %02x/%02x/%04x/%04x/%04x\n",
1042 		  crq.bRequestType, crq.bRequest, le16_to_cpu(crq.wValue),
1043 		  le16_to_cpu(crq.wIndex), le16_to_cpu(crq.wLength));
1044 
1045 	/*
1046 	 * Cleanup ep0 request(s) in queue because
1047 	 * there is a new control setup comes.
1048 	 */
1049 	list_for_each_entry(req, &udc->ep[0].queue, queue) {
1050 		req_num++;
1051 		EP_DBG(ep, "there is req %p in ep0 queue !\n", req);
1052 	}
1053 
1054 	if (req_num)
1055 		ast_udc_nuke(&udc->ep[0], -ETIMEDOUT);
1056 
1057 	udc->ep[0].dir_in = crq.bRequestType & USB_DIR_IN;
1058 
1059 	if ((crq.bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1060 		switch (crq.bRequest) {
1061 		case USB_REQ_SET_ADDRESS:
1062 			if (ast_udc_read(udc, AST_UDC_STS) & UDC_STS_HIGHSPEED)
1063 				udc->gadget.speed = USB_SPEED_HIGH;
1064 			else
1065 				udc->gadget.speed = USB_SPEED_FULL;
1066 
1067 			SETUP_DBG(udc, "set addr: 0x%x\n", crq.wValue);
1068 			reg = ast_udc_read(udc, AST_UDC_CONFIG);
1069 			reg &= ~UDC_CFG_ADDR_MASK;
1070 			reg |= UDC_CFG_SET_ADDR(crq.wValue);
1071 			ast_udc_write(udc, reg, AST_UDC_CONFIG);
1072 			goto req_complete;
1073 
1074 		case USB_REQ_CLEAR_FEATURE:
1075 			SETUP_DBG(udc, "ep0: CLEAR FEATURE\n");
1076 			goto req_driver;
1077 
1078 		case USB_REQ_SET_FEATURE:
1079 			SETUP_DBG(udc, "ep0: SET FEATURE\n");
1080 			goto req_driver;
1081 
1082 		case USB_REQ_GET_STATUS:
1083 			ast_udc_getstatus(udc);
1084 			return;
1085 
1086 		default:
1087 			goto req_driver;
1088 		}
1089 
1090 	}
1091 
1092 req_driver:
1093 	if (udc->driver) {
1094 		SETUP_DBG(udc, "Forwarding %s to gadget...\n",
1095 			  udc->gadget.name);
1096 
1097 		spin_unlock(&udc->lock);
1098 		rc = udc->driver->setup(&udc->gadget, &crq);
1099 		spin_lock(&udc->lock);
1100 
1101 	} else {
1102 		SETUP_DBG(udc, "No gadget for request !\n");
1103 	}
1104 
1105 	if (rc >= 0)
1106 		return;
1107 
1108 	/* Stall if gadget failed */
1109 	SETUP_DBG(udc, "Stalling, rc:0x%x\n", rc);
1110 	ast_udc_write(udc, ast_udc_read(udc, AST_UDC_EP0_CTRL) | EP0_STALL,
1111 		      AST_UDC_EP0_CTRL);
1112 	return;
1113 
1114 req_complete:
1115 	SETUP_DBG(udc, "ep0: Sending IN status without data\n");
1116 	ast_udc_write(udc, EP0_TX_BUFF_RDY, AST_UDC_EP0_CTRL);
1117 }
1118 
1119 static irqreturn_t ast_udc_isr(int irq, void *data)
1120 {
1121 	struct ast_udc_dev *udc = (struct ast_udc_dev *)data;
1122 	struct ast_udc_ep *ep;
1123 	u32 isr, ep_isr;
1124 	int i;
1125 
1126 	spin_lock(&udc->lock);
1127 
1128 	isr = ast_udc_read(udc, AST_UDC_ISR);
1129 	if (!isr)
1130 		goto done;
1131 
1132 	/* Ack interrupts */
1133 	ast_udc_write(udc, isr, AST_UDC_ISR);
1134 
1135 	if (isr & UDC_IRQ_BUS_RESET) {
1136 		ISR_DBG(udc, "UDC_IRQ_BUS_RESET\n");
1137 		udc->gadget.speed = USB_SPEED_UNKNOWN;
1138 
1139 		ep = &udc->ep[1];
1140 		EP_DBG(ep, "dctrl:0x%x\n",
1141 		       ast_ep_read(ep, AST_UDC_EP_DMA_CTRL));
1142 
1143 		if (udc->driver && udc->driver->reset) {
1144 			spin_unlock(&udc->lock);
1145 			udc->driver->reset(&udc->gadget);
1146 			spin_lock(&udc->lock);
1147 		}
1148 	}
1149 
1150 	if (isr & UDC_IRQ_BUS_SUSPEND) {
1151 		ISR_DBG(udc, "UDC_IRQ_BUS_SUSPEND\n");
1152 		udc->suspended_from = udc->gadget.state;
1153 		usb_gadget_set_state(&udc->gadget, USB_STATE_SUSPENDED);
1154 
1155 		if (udc->driver && udc->driver->suspend) {
1156 			spin_unlock(&udc->lock);
1157 			udc->driver->suspend(&udc->gadget);
1158 			spin_lock(&udc->lock);
1159 		}
1160 	}
1161 
1162 	if (isr & UDC_IRQ_BUS_RESUME) {
1163 		ISR_DBG(udc, "UDC_IRQ_BUS_RESUME\n");
1164 		usb_gadget_set_state(&udc->gadget, udc->suspended_from);
1165 
1166 		if (udc->driver && udc->driver->resume) {
1167 			spin_unlock(&udc->lock);
1168 			udc->driver->resume(&udc->gadget);
1169 			spin_lock(&udc->lock);
1170 		}
1171 	}
1172 
1173 	if (isr & UDC_IRQ_EP0_IN_ACK_STALL) {
1174 		ISR_DBG(udc, "UDC_IRQ_EP0_IN_ACK_STALL\n");
1175 		ast_udc_ep0_in(udc);
1176 	}
1177 
1178 	if (isr & UDC_IRQ_EP0_OUT_ACK_STALL) {
1179 		ISR_DBG(udc, "UDC_IRQ_EP0_OUT_ACK_STALL\n");
1180 		ast_udc_ep0_out(udc);
1181 	}
1182 
1183 	if (isr & UDC_IRQ_EP0_SETUP) {
1184 		ISR_DBG(udc, "UDC_IRQ_EP0_SETUP\n");
1185 		ast_udc_ep0_handle_setup(udc);
1186 	}
1187 
1188 	if (isr & UDC_IRQ_EP_POOL_ACK_STALL) {
1189 		ISR_DBG(udc, "UDC_IRQ_EP_POOL_ACK_STALL\n");
1190 		ep_isr = ast_udc_read(udc, AST_UDC_EP_ACK_ISR);
1191 
1192 		/* Ack EP interrupts */
1193 		ast_udc_write(udc, ep_isr, AST_UDC_EP_ACK_ISR);
1194 
1195 		/* Handle each EP */
1196 		for (i = 0; i < AST_UDC_NUM_ENDPOINTS - 1; i++) {
1197 			if (ep_isr & (0x1 << i)) {
1198 				ep = &udc->ep[i + 1];
1199 				if (ep->desc_mode)
1200 					ast_udc_epn_handle_desc(udc, i + 1);
1201 				else
1202 					ast_udc_epn_handle(udc, i + 1);
1203 			}
1204 		}
1205 	}
1206 
1207 done:
1208 	spin_unlock(&udc->lock);
1209 	return IRQ_HANDLED;
1210 }
1211 
1212 static int ast_udc_gadget_getframe(struct usb_gadget *gadget)
1213 {
1214 	struct ast_udc_dev *udc = to_ast_dev(gadget);
1215 
1216 	return (ast_udc_read(udc, AST_UDC_STS) >> 16) & 0x7ff;
1217 }
1218 
1219 static void ast_udc_wake_work(struct work_struct *work)
1220 {
1221 	struct ast_udc_dev *udc = container_of(work, struct ast_udc_dev,
1222 					       wake_work);
1223 	unsigned long flags;
1224 	u32 ctrl;
1225 
1226 	spin_lock_irqsave(&udc->lock, flags);
1227 
1228 	UDC_DBG(udc, "Wakeup Host !\n");
1229 	ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL);
1230 	ast_udc_write(udc, ctrl | USB_REMOTE_WAKEUP_EN, AST_UDC_FUNC_CTRL);
1231 
1232 	spin_unlock_irqrestore(&udc->lock, flags);
1233 }
1234 
1235 static void ast_udc_wakeup_all(struct ast_udc_dev *udc)
1236 {
1237 	/*
1238 	 * A device is trying to wake the world, because this
1239 	 * can recurse into the device, we break the call chain
1240 	 * using a work queue
1241 	 */
1242 	schedule_work(&udc->wake_work);
1243 }
1244 
1245 static int ast_udc_wakeup(struct usb_gadget *gadget)
1246 {
1247 	struct ast_udc_dev *udc = to_ast_dev(gadget);
1248 	unsigned long flags;
1249 	int rc = 0;
1250 
1251 	spin_lock_irqsave(&udc->lock, flags);
1252 
1253 	if (!udc->wakeup_en) {
1254 		UDC_DBG(udc, "Remote Wakeup is disabled\n");
1255 		rc = -EINVAL;
1256 		goto err;
1257 	}
1258 
1259 	UDC_DBG(udc, "Device initiated wakeup\n");
1260 	ast_udc_wakeup_all(udc);
1261 
1262 err:
1263 	spin_unlock_irqrestore(&udc->lock, flags);
1264 	return rc;
1265 }
1266 
1267 /*
1268  * Activate/Deactivate link with host
1269  */
1270 static int ast_udc_pullup(struct usb_gadget *gadget, int is_on)
1271 {
1272 	struct ast_udc_dev *udc = to_ast_dev(gadget);
1273 	unsigned long flags;
1274 	u32 ctrl;
1275 
1276 	spin_lock_irqsave(&udc->lock, flags);
1277 
1278 	UDC_DBG(udc, "is_on: %d\n", is_on);
1279 	if (is_on)
1280 		ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL) | USB_UPSTREAM_EN;
1281 	else
1282 		ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL) & ~USB_UPSTREAM_EN;
1283 
1284 	ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1285 
1286 	spin_unlock_irqrestore(&udc->lock, flags);
1287 
1288 	return 0;
1289 }
1290 
1291 static int ast_udc_start(struct usb_gadget *gadget,
1292 			 struct usb_gadget_driver *driver)
1293 {
1294 	struct ast_udc_dev *udc = to_ast_dev(gadget);
1295 	struct ast_udc_ep *ep;
1296 	unsigned long flags;
1297 	int i;
1298 
1299 	spin_lock_irqsave(&udc->lock, flags);
1300 
1301 	UDC_DBG(udc, "\n");
1302 	udc->driver = driver;
1303 	udc->gadget.dev.of_node = udc->pdev->dev.of_node;
1304 
1305 	for (i = 0; i < AST_UDC_NUM_ENDPOINTS; i++) {
1306 		ep = &udc->ep[i];
1307 		ep->stopped = 0;
1308 	}
1309 
1310 	spin_unlock_irqrestore(&udc->lock, flags);
1311 
1312 	return 0;
1313 }
1314 
1315 static int ast_udc_stop(struct usb_gadget *gadget)
1316 {
1317 	struct ast_udc_dev *udc = to_ast_dev(gadget);
1318 	unsigned long flags;
1319 	u32 ctrl;
1320 
1321 	spin_lock_irqsave(&udc->lock, flags);
1322 
1323 	UDC_DBG(udc, "\n");
1324 	ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL) & ~USB_UPSTREAM_EN;
1325 	ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1326 
1327 	udc->gadget.speed = USB_SPEED_UNKNOWN;
1328 	udc->driver = NULL;
1329 
1330 	ast_udc_stop_activity(udc);
1331 	usb_gadget_set_state(&udc->gadget, USB_STATE_NOTATTACHED);
1332 
1333 	spin_unlock_irqrestore(&udc->lock, flags);
1334 
1335 	return 0;
1336 }
1337 
1338 static const struct usb_gadget_ops ast_udc_ops = {
1339 	.get_frame		= ast_udc_gadget_getframe,
1340 	.wakeup			= ast_udc_wakeup,
1341 	.pullup			= ast_udc_pullup,
1342 	.udc_start		= ast_udc_start,
1343 	.udc_stop		= ast_udc_stop,
1344 };
1345 
1346 /*
1347  * Support 1 Control Endpoint.
1348  * Support multiple programmable endpoints that can be configured to
1349  * Bulk IN/OUT, Interrupt IN/OUT, and Isochronous IN/OUT type endpoint.
1350  */
1351 static void ast_udc_init_ep(struct ast_udc_dev *udc)
1352 {
1353 	struct ast_udc_ep *ep;
1354 	int i;
1355 
1356 	for (i = 0; i < AST_UDC_NUM_ENDPOINTS; i++) {
1357 		ep = &udc->ep[i];
1358 		ep->ep.name = ast_ep_name[i];
1359 		if (i == 0) {
1360 			ep->ep.caps.type_control = true;
1361 		} else {
1362 			ep->ep.caps.type_iso = true;
1363 			ep->ep.caps.type_bulk = true;
1364 			ep->ep.caps.type_int = true;
1365 		}
1366 		ep->ep.caps.dir_in = true;
1367 		ep->ep.caps.dir_out = true;
1368 
1369 		ep->ep.ops = &ast_udc_ep_ops;
1370 		ep->udc = udc;
1371 
1372 		INIT_LIST_HEAD(&ep->queue);
1373 
1374 		if (i == 0) {
1375 			usb_ep_set_maxpacket_limit(&ep->ep,
1376 						   AST_UDC_EP0_MAX_PACKET);
1377 			continue;
1378 		}
1379 
1380 		ep->ep_reg = udc->reg + AST_UDC_EP_BASE +
1381 				(AST_UDC_EP_OFFSET * (i - 1));
1382 
1383 		ep->epn_buf = udc->ep0_buf + (i * AST_UDC_EP_DMA_SIZE);
1384 		ep->epn_buf_dma = udc->ep0_buf_dma + (i * AST_UDC_EP_DMA_SIZE);
1385 		usb_ep_set_maxpacket_limit(&ep->ep, AST_UDC_EPn_MAX_PACKET);
1386 
1387 		ep->descs = ep->epn_buf + AST_UDC_EPn_MAX_PACKET;
1388 		ep->descs_dma = ep->epn_buf_dma + AST_UDC_EPn_MAX_PACKET;
1389 		ep->descs_wptr = 0;
1390 
1391 		list_add_tail(&ep->ep.ep_list, &udc->gadget.ep_list);
1392 	}
1393 }
1394 
1395 static void ast_udc_init_dev(struct ast_udc_dev *udc)
1396 {
1397 	INIT_WORK(&udc->wake_work, ast_udc_wake_work);
1398 }
1399 
1400 static void ast_udc_init_hw(struct ast_udc_dev *udc)
1401 {
1402 	u32 ctrl;
1403 
1404 	/* Enable PHY */
1405 	ctrl = USB_PHY_CLK_EN | USB_PHY_RESET_DIS;
1406 	ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1407 
1408 	udelay(1);
1409 	ast_udc_write(udc, 0, AST_UDC_DEV_RESET);
1410 
1411 	/* Set descriptor ring size */
1412 	if (AST_UDC_DESCS_COUNT == 256) {
1413 		ctrl |= USB_EP_LONG_DESC;
1414 		ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1415 	}
1416 
1417 	/* Mask & ack all interrupts before installing the handler */
1418 	ast_udc_write(udc, 0, AST_UDC_IER);
1419 	ast_udc_write(udc, UDC_IRQ_ACK_ALL, AST_UDC_ISR);
1420 
1421 	/* Enable some interrupts */
1422 	ctrl = UDC_IRQ_EP_POOL_ACK_STALL | UDC_IRQ_BUS_RESUME |
1423 	       UDC_IRQ_BUS_SUSPEND | UDC_IRQ_BUS_RESET |
1424 	       UDC_IRQ_EP0_IN_ACK_STALL | UDC_IRQ_EP0_OUT_ACK_STALL |
1425 	       UDC_IRQ_EP0_SETUP;
1426 	ast_udc_write(udc, ctrl, AST_UDC_IER);
1427 
1428 	/* Cleanup and enable ep ACK interrupts */
1429 	ast_udc_write(udc, UDC_IRQ_EP_ACK_ALL, AST_UDC_EP_ACK_IER);
1430 	ast_udc_write(udc, UDC_IRQ_EP_ACK_ALL, AST_UDC_EP_ACK_ISR);
1431 
1432 	ast_udc_write(udc, 0, AST_UDC_EP0_CTRL);
1433 }
1434 
1435 static int ast_udc_remove(struct platform_device *pdev)
1436 {
1437 	struct ast_udc_dev *udc = platform_get_drvdata(pdev);
1438 	unsigned long flags;
1439 	u32 ctrl;
1440 
1441 	usb_del_gadget_udc(&udc->gadget);
1442 	if (udc->driver)
1443 		return -EBUSY;
1444 
1445 	spin_lock_irqsave(&udc->lock, flags);
1446 
1447 	/* Disable upstream port connection */
1448 	ctrl = ast_udc_read(udc, AST_UDC_FUNC_CTRL) & ~USB_UPSTREAM_EN;
1449 	ast_udc_write(udc, ctrl, AST_UDC_FUNC_CTRL);
1450 
1451 	clk_disable_unprepare(udc->clk);
1452 
1453 	spin_unlock_irqrestore(&udc->lock, flags);
1454 
1455 	if (udc->ep0_buf)
1456 		dma_free_coherent(&pdev->dev,
1457 				  AST_UDC_EP_DMA_SIZE * AST_UDC_NUM_ENDPOINTS,
1458 				  udc->ep0_buf,
1459 				  udc->ep0_buf_dma);
1460 
1461 	udc->ep0_buf = NULL;
1462 
1463 	return 0;
1464 }
1465 
1466 static int ast_udc_probe(struct platform_device *pdev)
1467 {
1468 	enum usb_device_speed max_speed;
1469 	struct device *dev = &pdev->dev;
1470 	struct ast_udc_dev *udc;
1471 	struct resource *res;
1472 	int rc;
1473 
1474 	udc = devm_kzalloc(&pdev->dev, sizeof(struct ast_udc_dev), GFP_KERNEL);
1475 	if (!udc)
1476 		return -ENOMEM;
1477 
1478 	udc->gadget.dev.parent = dev;
1479 	udc->pdev = pdev;
1480 	spin_lock_init(&udc->lock);
1481 
1482 	udc->gadget.ops = &ast_udc_ops;
1483 	udc->gadget.ep0 = &udc->ep[0].ep;
1484 	udc->gadget.name = "aspeed-udc";
1485 	udc->gadget.dev.init_name = "gadget";
1486 
1487 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1488 	udc->reg = devm_ioremap_resource(&pdev->dev, res);
1489 	if (IS_ERR(udc->reg)) {
1490 		dev_err(&pdev->dev, "Failed to map resources\n");
1491 		return PTR_ERR(udc->reg);
1492 	}
1493 
1494 	platform_set_drvdata(pdev, udc);
1495 
1496 	udc->clk = devm_clk_get(&pdev->dev, NULL);
1497 	if (IS_ERR(udc->clk)) {
1498 		rc = PTR_ERR(udc->clk);
1499 		goto err;
1500 	}
1501 	rc = clk_prepare_enable(udc->clk);
1502 	if (rc) {
1503 		dev_err(&pdev->dev, "Failed to enable clock (0x%x)\n", rc);
1504 		goto err;
1505 	}
1506 
1507 	/* Check if we need to limit the HW to USB1 */
1508 	max_speed = usb_get_maximum_speed(&pdev->dev);
1509 	if (max_speed != USB_SPEED_UNKNOWN && max_speed < USB_SPEED_HIGH)
1510 		udc->force_usb1 = true;
1511 
1512 	/*
1513 	 * Allocate DMA buffers for all EPs in one chunk
1514 	 */
1515 	udc->ep0_buf = dma_alloc_coherent(&pdev->dev,
1516 					  AST_UDC_EP_DMA_SIZE *
1517 					  AST_UDC_NUM_ENDPOINTS,
1518 					  &udc->ep0_buf_dma, GFP_KERNEL);
1519 
1520 	udc->gadget.speed = USB_SPEED_UNKNOWN;
1521 	udc->gadget.max_speed = USB_SPEED_HIGH;
1522 	udc->creq = udc->reg + AST_UDC_SETUP0;
1523 
1524 	/*
1525 	 * Support single stage mode or 32/256 stages descriptor mode.
1526 	 * Set default as Descriptor Mode.
1527 	 */
1528 	udc->desc_mode = AST_UDC_DESC_MODE;
1529 
1530 	dev_info(&pdev->dev, "DMA %s\n", udc->desc_mode ?
1531 		 "descriptor mode" : "single mode");
1532 
1533 	INIT_LIST_HEAD(&udc->gadget.ep_list);
1534 	INIT_LIST_HEAD(&udc->gadget.ep0->ep_list);
1535 
1536 	/* Initialized udc ep */
1537 	ast_udc_init_ep(udc);
1538 
1539 	/* Initialized udc device */
1540 	ast_udc_init_dev(udc);
1541 
1542 	/* Initialized udc hardware */
1543 	ast_udc_init_hw(udc);
1544 
1545 	/* Find interrupt and install handler */
1546 	udc->irq = platform_get_irq(pdev, 0);
1547 	if (udc->irq < 0) {
1548 		rc = udc->irq;
1549 		goto err;
1550 	}
1551 
1552 	rc = devm_request_irq(&pdev->dev, udc->irq, ast_udc_isr, 0,
1553 			      KBUILD_MODNAME, udc);
1554 	if (rc) {
1555 		dev_err(&pdev->dev, "Failed to request interrupt\n");
1556 		goto err;
1557 	}
1558 
1559 	rc = usb_add_gadget_udc(&pdev->dev, &udc->gadget);
1560 	if (rc) {
1561 		dev_err(&pdev->dev, "Failed to add gadget udc\n");
1562 		goto err;
1563 	}
1564 
1565 	dev_info(&pdev->dev, "Initialized udc in USB%s mode\n",
1566 		 udc->force_usb1 ? "1" : "2");
1567 
1568 	return 0;
1569 
1570 err:
1571 	dev_err(&pdev->dev, "Failed to udc probe, rc:0x%x\n", rc);
1572 	ast_udc_remove(pdev);
1573 
1574 	return rc;
1575 }
1576 
1577 static const struct of_device_id ast_udc_of_dt_ids[] = {
1578 	{ .compatible = "aspeed,ast2600-udc", },
1579 	{}
1580 };
1581 
1582 MODULE_DEVICE_TABLE(of, ast_udc_of_dt_ids);
1583 
1584 static struct platform_driver ast_udc_driver = {
1585 	.probe			= ast_udc_probe,
1586 	.remove			= ast_udc_remove,
1587 	.driver			= {
1588 		.name			= KBUILD_MODNAME,
1589 		.of_match_table		= ast_udc_of_dt_ids,
1590 	},
1591 };
1592 
1593 module_platform_driver(ast_udc_driver);
1594 
1595 MODULE_DESCRIPTION("ASPEED UDC driver");
1596 MODULE_AUTHOR("Neal Liu <neal_liu@aspeedtech.com>");
1597 MODULE_LICENSE("GPL");
1598