1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Cadence USBHS-DEV Driver - gadget side.
4  *
5  * Copyright (C) 2023 Cadence Design Systems.
6  *
7  * Authors: Pawel Laszczak <pawell@cadence.com>
8  */
9 
10 /*
11  * Work around 1:
12  * At some situations, the controller may get stale data address in TRB
13  * at below sequences:
14  * 1. Controller read TRB includes data address
15  * 2. Software updates TRBs includes data address and Cycle bit
16  * 3. Controller read TRB which includes Cycle bit
17  * 4. DMA run with stale data address
18  *
19  * To fix this problem, driver needs to make the first TRB in TD as invalid.
20  * After preparing all TRBs driver needs to check the position of DMA and
21  * if the DMA point to the first just added TRB and doorbell is 1,
22  * then driver must defer making this TRB as valid. This TRB will be make
23  * as valid during adding next TRB only if DMA is stopped or at TRBERR
24  * interrupt.
25  *
26  */
27 
28 #include <linux/dma-mapping.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/interrupt.h>
31 #include <linux/property.h>
32 #include <linux/dmapool.h>
33 #include <linux/iopoll.h>
34 
35 #include "cdns2-gadget.h"
36 #include "cdns2-trace.h"
37 
38 /**
39  * set_reg_bit_32 - set bit in given 32 bits register.
40  * @ptr: register address.
41  * @mask: bits to set.
42  */
43 static void set_reg_bit_32(void __iomem *ptr, u32 mask)
44 {
45 	mask = readl(ptr) | mask;
46 	writel(mask, ptr);
47 }
48 
49 /*
50  * clear_reg_bit_32 - clear bit in given 32 bits register.
51  * @ptr: register address.
52  * @mask: bits to clear.
53  */
54 static void clear_reg_bit_32(void __iomem *ptr, u32 mask)
55 {
56 	mask = readl(ptr) & ~mask;
57 	writel(mask, ptr);
58 }
59 
60 /* Clear bit in given 8 bits register. */
61 static void clear_reg_bit_8(void __iomem *ptr, u8 mask)
62 {
63 	mask = readb(ptr) & ~mask;
64 	writeb(mask, ptr);
65 }
66 
67 /* Set bit in given 16 bits register. */
68 void set_reg_bit_8(void __iomem *ptr, u8 mask)
69 {
70 	mask = readb(ptr) | mask;
71 	writeb(mask, ptr);
72 }
73 
74 static int cdns2_get_dma_pos(struct cdns2_device *pdev,
75 			     struct cdns2_endpoint *pep)
76 {
77 	int dma_index;
78 
79 	dma_index = readl(&pdev->adma_regs->ep_traddr) - pep->ring.dma;
80 
81 	return dma_index / TRB_SIZE;
82 }
83 
84 /* Get next private request from list. */
85 struct cdns2_request *cdns2_next_preq(struct list_head *list)
86 {
87 	return list_first_entry_or_null(list, struct cdns2_request, list);
88 }
89 
90 void cdns2_select_ep(struct cdns2_device *pdev, u32 ep)
91 {
92 	if (pdev->selected_ep == ep)
93 		return;
94 
95 	pdev->selected_ep = ep;
96 	writel(ep, &pdev->adma_regs->ep_sel);
97 }
98 
99 dma_addr_t cdns2_trb_virt_to_dma(struct cdns2_endpoint *pep,
100 				 struct cdns2_trb *trb)
101 {
102 	u32 offset = (char *)trb - (char *)pep->ring.trbs;
103 
104 	return pep->ring.dma + offset;
105 }
106 
107 static void cdns2_free_tr_segment(struct cdns2_endpoint *pep)
108 {
109 	struct cdns2_device *pdev = pep->pdev;
110 	struct cdns2_ring *ring = &pep->ring;
111 
112 	if (pep->ring.trbs) {
113 		dma_pool_free(pdev->eps_dma_pool, ring->trbs, ring->dma);
114 		memset(ring, 0, sizeof(*ring));
115 	}
116 }
117 
118 /* Allocates Transfer Ring segment. */
119 static int cdns2_alloc_tr_segment(struct cdns2_endpoint *pep)
120 {
121 	struct cdns2_device *pdev = pep->pdev;
122 	struct cdns2_trb *link_trb;
123 	struct cdns2_ring *ring;
124 
125 	ring = &pep->ring;
126 
127 	if (!ring->trbs) {
128 		ring->trbs = dma_pool_alloc(pdev->eps_dma_pool,
129 					    GFP_DMA32 | GFP_ATOMIC,
130 					    &ring->dma);
131 		if (!ring->trbs)
132 			return -ENOMEM;
133 	}
134 
135 	memset(ring->trbs, 0, TR_SEG_SIZE);
136 
137 	if (!pep->num)
138 		return 0;
139 
140 	/* Initialize the last TRB as Link TRB */
141 	link_trb = (ring->trbs + (TRBS_PER_SEGMENT - 1));
142 	link_trb->buffer = cpu_to_le32(TRB_BUFFER(ring->dma));
143 	link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) |
144 					TRB_TOGGLE);
145 
146 	return 0;
147 }
148 
149 /*
150  * Stalls and flushes selected endpoint.
151  * Endpoint must be selected before invoking this function.
152  */
153 static void cdns2_ep_stall_flush(struct cdns2_endpoint *pep)
154 {
155 	struct cdns2_device *pdev = pep->pdev;
156 	int val;
157 
158 	trace_cdns2_ep_halt(pep, 1, 1);
159 
160 	writel(DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
161 
162 	/* Wait for DFLUSH cleared. */
163 	readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
164 				  !(val & DMA_EP_CMD_DFLUSH), 1, 1000);
165 	pep->ep_state |= EP_STALLED;
166 	pep->ep_state &= ~EP_STALL_PENDING;
167 }
168 
169 /*
170  * Increment a trb index.
171  *
172  * The index should never point to the last link TRB in TR. After incrementing,
173  * if it point to the link TRB, wrap around to the beginning and revert
174  * cycle state bit. The link TRB is always at the last TRB entry.
175  */
176 static void cdns2_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
177 {
178 	(*index)++;
179 	if (*index == (trb_in_seg - 1)) {
180 		*index = 0;
181 		*cs ^=  1;
182 	}
183 }
184 
185 static void cdns2_ep_inc_enq(struct cdns2_ring *ring)
186 {
187 	ring->free_trbs--;
188 	cdns2_ep_inc_trb(&ring->enqueue, &ring->pcs, TRBS_PER_SEGMENT);
189 }
190 
191 static void cdns2_ep_inc_deq(struct cdns2_ring *ring)
192 {
193 	ring->free_trbs++;
194 	cdns2_ep_inc_trb(&ring->dequeue, &ring->ccs, TRBS_PER_SEGMENT);
195 }
196 
197 /*
198  * Enable/disable LPM.
199  *
200  * If bit USBCS_LPMNYET is not set and device receive Extended Token packet,
201  * then controller answer with ACK handshake.
202  * If bit USBCS_LPMNYET is set and device receive Extended Token packet,
203  * then controller answer with NYET handshake.
204  */
205 static void cdns2_enable_l1(struct cdns2_device *pdev, int enable)
206 {
207 	if (enable) {
208 		clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
209 		writeb(LPMCLOCK_SLEEP_ENTRY, &pdev->usb_regs->lpmclock);
210 	} else {
211 		set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_LPMNYET);
212 	}
213 }
214 
215 static enum usb_device_speed cdns2_get_speed(struct cdns2_device *pdev)
216 {
217 	u8 speed = readb(&pdev->usb_regs->speedctrl);
218 
219 	if (speed & SPEEDCTRL_HS)
220 		return USB_SPEED_HIGH;
221 	else if (speed & SPEEDCTRL_FS)
222 		return USB_SPEED_FULL;
223 
224 	return USB_SPEED_UNKNOWN;
225 }
226 
227 static struct cdns2_trb *cdns2_next_trb(struct cdns2_endpoint *pep,
228 					struct cdns2_trb *trb)
229 {
230 	if (trb == (pep->ring.trbs + (TRBS_PER_SEGMENT - 1)))
231 		return pep->ring.trbs;
232 	else
233 		return ++trb;
234 }
235 
236 void cdns2_gadget_giveback(struct cdns2_endpoint *pep,
237 			   struct cdns2_request *preq,
238 			   int status)
239 {
240 	struct usb_request *request = &preq->request;
241 	struct cdns2_device *pdev = pep->pdev;
242 
243 	list_del_init(&preq->list);
244 
245 	if (request->status == -EINPROGRESS)
246 		request->status = status;
247 
248 	usb_gadget_unmap_request_by_dev(pdev->dev, request, pep->dir);
249 
250 	/* All TRBs have finished, clear the counter. */
251 	preq->finished_trb = 0;
252 
253 	trace_cdns2_request_giveback(preq);
254 
255 	if (request->complete) {
256 		spin_unlock(&pdev->lock);
257 		usb_gadget_giveback_request(&pep->endpoint, request);
258 		spin_lock(&pdev->lock);
259 	}
260 
261 	if (request->buf == pdev->zlp_buf)
262 		cdns2_gadget_ep_free_request(&pep->endpoint, request);
263 }
264 
265 static void cdns2_wa1_restore_cycle_bit(struct cdns2_endpoint *pep)
266 {
267 	/* Work around for stale data address in TRB. */
268 	if (pep->wa1_set) {
269 		trace_cdns2_wa1(pep, "restore cycle bit");
270 
271 		pep->wa1_set = 0;
272 		pep->wa1_trb_index = 0xFFFF;
273 		if (pep->wa1_cycle_bit)
274 			pep->wa1_trb->control |= cpu_to_le32(0x1);
275 		else
276 			pep->wa1_trb->control &= cpu_to_le32(~0x1);
277 	}
278 }
279 
280 static int cdns2_wa1_update_guard(struct cdns2_endpoint *pep,
281 				  struct cdns2_trb *trb)
282 {
283 	struct cdns2_device *pdev = pep->pdev;
284 
285 	if (!pep->wa1_set) {
286 		u32 doorbell;
287 
288 		doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
289 
290 		if (doorbell) {
291 			pep->wa1_cycle_bit = pep->ring.pcs ? TRB_CYCLE : 0;
292 			pep->wa1_set = 1;
293 			pep->wa1_trb = trb;
294 			pep->wa1_trb_index = pep->ring.enqueue;
295 			trace_cdns2_wa1(pep, "set guard");
296 			return 0;
297 		}
298 	}
299 	return 1;
300 }
301 
302 static void cdns2_wa1_tray_restore_cycle_bit(struct cdns2_device *pdev,
303 					     struct cdns2_endpoint *pep)
304 {
305 	int dma_index;
306 	u32 doorbell;
307 
308 	doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
309 	dma_index = cdns2_get_dma_pos(pdev, pep);
310 
311 	if (!doorbell || dma_index != pep->wa1_trb_index)
312 		cdns2_wa1_restore_cycle_bit(pep);
313 }
314 
315 static int cdns2_prepare_ring(struct cdns2_device *pdev,
316 			      struct cdns2_endpoint *pep,
317 			      int num_trbs)
318 {
319 	struct cdns2_trb *link_trb = NULL;
320 	int doorbell, dma_index;
321 	struct cdns2_ring *ring;
322 	u32 ch_bit = 0;
323 
324 	ring = &pep->ring;
325 
326 	if (num_trbs > ring->free_trbs) {
327 		pep->ep_state |= EP_RING_FULL;
328 		trace_cdns2_no_room_on_ring("Ring full\n");
329 		return -ENOBUFS;
330 	}
331 
332 	if ((ring->enqueue + num_trbs)  >= (TRBS_PER_SEGMENT - 1)) {
333 		doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
334 		dma_index = cdns2_get_dma_pos(pdev, pep);
335 
336 		/* Driver can't update LINK TRB if it is current processed. */
337 		if (doorbell && dma_index == TRBS_PER_SEGMENT - 1) {
338 			pep->ep_state |= EP_DEFERRED_DRDY;
339 			return -ENOBUFS;
340 		}
341 
342 		/* Update C bt in Link TRB before starting DMA. */
343 		link_trb = ring->trbs + (TRBS_PER_SEGMENT - 1);
344 
345 		/*
346 		 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
347 		 * that DMA stuck at the LINK TRB.
348 		 * On the other hand, removing TRB_CHAIN for longer TRs for
349 		 * epXout cause that DMA stuck after handling LINK TRB.
350 		 * To eliminate this strange behavioral driver set TRB_CHAIN
351 		 * bit only for TR size > 2.
352 		 */
353 		if (pep->type == USB_ENDPOINT_XFER_ISOC || TRBS_PER_SEGMENT > 2)
354 			ch_bit = TRB_CHAIN;
355 
356 		link_trb->control = cpu_to_le32(((ring->pcs) ? TRB_CYCLE : 0) |
357 				    TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
358 	}
359 
360 	return 0;
361 }
362 
363 static void cdns2_dbg_request_trbs(struct cdns2_endpoint *pep,
364 				   struct cdns2_request *preq)
365 {
366 	struct cdns2_trb *link_trb = pep->ring.trbs + (TRBS_PER_SEGMENT - 1);
367 	struct cdns2_trb *trb = preq->trb;
368 	int num_trbs = preq->num_of_trb;
369 	int i = 0;
370 
371 	while (i < num_trbs) {
372 		trace_cdns2_queue_trb(pep, trb + i);
373 		if (trb + i == link_trb) {
374 			trb = pep->ring.trbs;
375 			num_trbs = num_trbs - i;
376 			i = 0;
377 		} else {
378 			i++;
379 		}
380 	}
381 }
382 
383 static unsigned int cdns2_count_trbs(struct cdns2_endpoint *pep,
384 				     u64 addr, u64 len)
385 {
386 	unsigned int num_trbs = 1;
387 
388 	if (pep->type == USB_ENDPOINT_XFER_ISOC) {
389 		/*
390 		 * To speed up DMA performance address should not exceed 4KB.
391 		 * for high bandwidth transfer and driver will split
392 		 * such buffer into two TRBs.
393 		 */
394 		num_trbs = DIV_ROUND_UP(len +
395 					(addr & (TRB_MAX_ISO_BUFF_SIZE - 1)),
396 					TRB_MAX_ISO_BUFF_SIZE);
397 
398 		if (pep->interval > 1)
399 			num_trbs = pep->dir ? num_trbs * pep->interval : 1;
400 	} else if (pep->dir) {
401 		/*
402 		 * One extra link trb for IN direction.
403 		 * Sometimes DMA doesn't want advance to next TD and transfer
404 		 * hangs. This extra Link TRB force DMA to advance to next TD.
405 		 */
406 		num_trbs++;
407 	}
408 
409 	return num_trbs;
410 }
411 
412 static unsigned int cdns2_count_sg_trbs(struct cdns2_endpoint *pep,
413 					struct usb_request *req)
414 {
415 	unsigned int i, len, full_len, num_trbs = 0;
416 	struct scatterlist *sg;
417 	int trb_len = 0;
418 
419 	full_len = req->length;
420 
421 	for_each_sg(req->sg, sg, req->num_sgs, i) {
422 		len = sg_dma_len(sg);
423 		num_trbs += cdns2_count_trbs(pep, sg_dma_address(sg), len);
424 		len = min(len, full_len);
425 
426 		/*
427 		 * For HS ISO transfer TRBs should not exceed max packet size.
428 		 * When DMA is working, and data exceed max packet size then
429 		 * some data will be read in single mode instead burst mode.
430 		 * This behavior will drastically reduce the copying speed.
431 		 * To avoid this we need one or two extra TRBs.
432 		 * This issue occurs for UVC class with sg_supported = 1
433 		 * because buffers addresses are not aligned to 1024.
434 		 */
435 		if (pep->type == USB_ENDPOINT_XFER_ISOC) {
436 			u8 temp;
437 
438 			trb_len += len;
439 			temp = trb_len >> 10;
440 
441 			if (temp) {
442 				if (trb_len % 1024)
443 					num_trbs = num_trbs + temp;
444 				else
445 					num_trbs = num_trbs + temp - 1;
446 
447 				trb_len = trb_len - (temp << 10);
448 			}
449 		}
450 
451 		full_len -= len;
452 		if (full_len == 0)
453 			break;
454 	}
455 
456 	return num_trbs;
457 }
458 
459 /*
460  * Function prepares the array with optimized AXI burst value for different
461  * transfer lengths. Controller handles the final data which are less
462  * then AXI burst size as single byte transactions.
463  * e.g.:
464  * Let's assume that driver prepares trb with trb->length 700 and burst size
465  * will be set to 128. In this case the controller will handle a first 512 as
466  * single AXI transaction but the next 188 bytes will be handled
467  * as 47 separate AXI transaction.
468  * The better solution is to use the burst size equal 16 and then we will
469  * have only 25 AXI transaction (10 * 64 + 15 *4).
470  */
471 static void cdsn2_isoc_burst_opt(struct cdns2_device *pdev)
472 {
473 	int axi_burst_option[]  =  {1, 2, 4, 8, 16, 32, 64, 128};
474 	int best_burst;
475 	int array_size;
476 	int opt_burst;
477 	int trb_size;
478 	int i, j;
479 
480 	array_size = ARRAY_SIZE(axi_burst_option);
481 
482 	for (i = 0; i <= MAX_ISO_SIZE; i++) {
483 		trb_size = i / 4;
484 		best_burst = trb_size ? trb_size : 1;
485 
486 		for (j = 0; j < array_size; j++) {
487 			opt_burst = trb_size / axi_burst_option[j];
488 			opt_burst += trb_size % axi_burst_option[j];
489 
490 			if (opt_burst < best_burst) {
491 				best_burst = opt_burst;
492 				pdev->burst_opt[i] = axi_burst_option[j];
493 			}
494 		}
495 	}
496 }
497 
498 static void cdns2_ep_tx_isoc(struct cdns2_endpoint *pep,
499 			     struct cdns2_request *preq,
500 			     int num_trbs)
501 {
502 	struct scatterlist *sg = NULL;
503 	u32 remaining_packet_size = 0;
504 	struct cdns2_trb *trb;
505 	bool first_trb = true;
506 	dma_addr_t trb_dma;
507 	u32 trb_buff_len;
508 	u32 block_length;
509 	int td_idx = 0;
510 	int split_size;
511 	u32 full_len;
512 	int enqd_len;
513 	int sent_len;
514 	int sg_iter;
515 	u32 control;
516 	int num_tds;
517 	u32 length;
518 
519 	/*
520 	 * For OUT direction 1 TD per interval is enough
521 	 * because TRBs are not dumped by controller.
522 	 */
523 	num_tds = pep->dir ? pep->interval : 1;
524 	split_size = preq->request.num_sgs ? 1024 : 3072;
525 
526 	for (td_idx = 0; td_idx < num_tds; td_idx++) {
527 		if (preq->request.num_sgs) {
528 			sg = preq->request.sg;
529 			trb_dma = sg_dma_address(sg);
530 			block_length = sg_dma_len(sg);
531 		} else {
532 			trb_dma = preq->request.dma;
533 			block_length = preq->request.length;
534 		}
535 
536 		full_len = preq->request.length;
537 		sg_iter = preq->request.num_sgs ? preq->request.num_sgs : 1;
538 		remaining_packet_size = split_size;
539 
540 		for (enqd_len = 0;  enqd_len < full_len;
541 		     enqd_len += trb_buff_len) {
542 			if (remaining_packet_size == 0)
543 				remaining_packet_size = split_size;
544 
545 			/*
546 			 * Calculate TRB length.- buffer can't across 4KB
547 			 * and max packet size.
548 			 */
549 			trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(trb_dma);
550 			trb_buff_len = min(trb_buff_len, remaining_packet_size);
551 			trb_buff_len = min(trb_buff_len, block_length);
552 
553 			if (trb_buff_len > full_len - enqd_len)
554 				trb_buff_len = full_len - enqd_len;
555 
556 			control = TRB_TYPE(TRB_NORMAL);
557 
558 			/*
559 			 * For IN direction driver has to set the IOC for
560 			 * last TRB in last TD.
561 			 * For OUT direction driver must set IOC and ISP
562 			 * only for last TRB in each TDs.
563 			 */
564 			if (enqd_len + trb_buff_len >= full_len || !pep->dir)
565 				control |= TRB_IOC | TRB_ISP;
566 
567 			/*
568 			 * Don't give the first TRB to the hardware (by toggling
569 			 * the cycle bit) until we've finished creating all the
570 			 * other TRBs.
571 			 */
572 			if (first_trb) {
573 				first_trb = false;
574 				if (pep->ring.pcs == 0)
575 					control |= TRB_CYCLE;
576 			} else {
577 				control |= pep->ring.pcs;
578 			}
579 
580 			if (enqd_len + trb_buff_len < full_len)
581 				control |= TRB_CHAIN;
582 
583 			length = TRB_LEN(trb_buff_len) |
584 				 TRB_BURST(pep->pdev->burst_opt[trb_buff_len]);
585 
586 			trb = pep->ring.trbs + pep->ring.enqueue;
587 			trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
588 			trb->length = cpu_to_le32(length);
589 			trb->control = cpu_to_le32(control);
590 
591 			trb_dma += trb_buff_len;
592 			sent_len = trb_buff_len;
593 
594 			if (sg && sent_len >= block_length) {
595 				/* New sg entry */
596 				--sg_iter;
597 				sent_len -= block_length;
598 				if (sg_iter != 0) {
599 					sg = sg_next(sg);
600 					trb_dma = sg_dma_address(sg);
601 					block_length = sg_dma_len(sg);
602 				}
603 			}
604 
605 			remaining_packet_size -= trb_buff_len;
606 			block_length -= sent_len;
607 			preq->end_trb = pep->ring.enqueue;
608 
609 			cdns2_ep_inc_enq(&pep->ring);
610 		}
611 	}
612 }
613 
614 static void cdns2_ep_tx_bulk(struct cdns2_endpoint *pep,
615 			     struct cdns2_request *preq,
616 			     int trbs_per_td)
617 {
618 	struct scatterlist *sg = NULL;
619 	struct cdns2_ring *ring;
620 	struct cdns2_trb *trb;
621 	dma_addr_t trb_dma;
622 	int sg_iter = 0;
623 	u32 control;
624 	u32 length;
625 
626 	if (preq->request.num_sgs) {
627 		sg = preq->request.sg;
628 		trb_dma = sg_dma_address(sg);
629 		length = sg_dma_len(sg);
630 	} else {
631 		trb_dma = preq->request.dma;
632 		length = preq->request.length;
633 	}
634 
635 	ring = &pep->ring;
636 
637 	for (sg_iter = 0; sg_iter < trbs_per_td; sg_iter++) {
638 		control = TRB_TYPE(TRB_NORMAL) | ring->pcs | TRB_ISP;
639 		trb = pep->ring.trbs + ring->enqueue;
640 
641 		if (pep->dir && sg_iter == trbs_per_td - 1) {
642 			preq->end_trb = ring->enqueue;
643 			control = ring->pcs | TRB_TYPE(TRB_LINK) | TRB_CHAIN
644 				  | TRB_IOC;
645 			cdns2_ep_inc_enq(&pep->ring);
646 
647 			if (ring->enqueue == 0)
648 				control |= TRB_TOGGLE;
649 
650 			/* Point to next bad TRB. */
651 			trb->buffer = cpu_to_le32(pep->ring.dma +
652 						  (ring->enqueue * TRB_SIZE));
653 			trb->length = 0;
654 			trb->control = cpu_to_le32(control);
655 			break;
656 		}
657 
658 		/*
659 		 * Don't give the first TRB to the hardware (by toggling
660 		 * the cycle bit) until we've finished creating all the
661 		 * other TRBs.
662 		 */
663 		if (sg_iter == 0)
664 			control = control ^ TRB_CYCLE;
665 
666 		/* For last TRB in TD. */
667 		if (sg_iter == (trbs_per_td - (pep->dir ? 2 : 1)))
668 			control |= TRB_IOC;
669 		else
670 			control |= TRB_CHAIN;
671 
672 		trb->buffer = cpu_to_le32(trb_dma);
673 		trb->length = cpu_to_le32(TRB_BURST(pep->trb_burst_size) |
674 					   TRB_LEN(length));
675 		trb->control = cpu_to_le32(control);
676 
677 		if (sg && sg_iter < (trbs_per_td - 1)) {
678 			sg = sg_next(sg);
679 			trb_dma = sg_dma_address(sg);
680 			length = sg_dma_len(sg);
681 		}
682 
683 		preq->end_trb = ring->enqueue;
684 		cdns2_ep_inc_enq(&pep->ring);
685 	}
686 }
687 
688 static void cdns2_set_drdy(struct cdns2_device *pdev,
689 			   struct cdns2_endpoint *pep)
690 {
691 	trace_cdns2_ring(pep);
692 
693 	/*
694 	 * Memory barrier - Cycle Bit must be set before doorbell.
695 	 */
696 	dma_wmb();
697 
698 	/* Clearing TRBERR and DESCMIS before setting DRDY. */
699 	writel(DMA_EP_STS_TRBERR | DMA_EP_STS_DESCMIS,
700 	       &pdev->adma_regs->ep_sts);
701 	writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
702 
703 	if (readl(&pdev->adma_regs->ep_sts) & DMA_EP_STS_TRBERR) {
704 		writel(DMA_EP_STS_TRBERR, &pdev->adma_regs->ep_sts);
705 		writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
706 	}
707 
708 	trace_cdns2_doorbell_epx(pep, readl(&pdev->adma_regs->ep_traddr));
709 }
710 
711 static int cdns2_prepare_first_isoc_transfer(struct cdns2_device *pdev,
712 					     struct cdns2_endpoint *pep)
713 {
714 	struct cdns2_trb *trb;
715 	u32 buffer;
716 	u8 hw_ccs;
717 
718 	if ((readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY))
719 		return -EBUSY;
720 
721 	if (!pep->dir) {
722 		set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
723 		writel(pep->ring.dma + pep->ring.dequeue,
724 		       &pdev->adma_regs->ep_traddr);
725 		return 0;
726 	}
727 
728 	/*
729 	 * The first packet after doorbell can be corrupted so,
730 	 * driver prepares 0 length packet as first packet.
731 	 */
732 	buffer = pep->ring.dma + pep->ring.dequeue * TRB_SIZE;
733 	hw_ccs = !!DMA_EP_STS_CCS(readl(&pdev->adma_regs->ep_sts));
734 
735 	trb = &pep->ring.trbs[TRBS_PER_SEGMENT];
736 	trb->length = 0;
737 	trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
738 	trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) | TRB_TYPE(TRB_NORMAL));
739 
740 	/*
741 	 * LINK TRB is used to force updating cycle bit in controller and
742 	 * move to correct place in transfer ring.
743 	 */
744 	trb++;
745 	trb->length = 0;
746 	trb->buffer = cpu_to_le32(TRB_BUFFER(buffer));
747 	trb->control = cpu_to_le32((hw_ccs ? TRB_CYCLE : 0) |
748 				    TRB_TYPE(TRB_LINK) | TRB_CHAIN);
749 
750 	if (hw_ccs !=  pep->ring.ccs)
751 		trb->control |= cpu_to_le32(TRB_TOGGLE);
752 
753 	set_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
754 	writel(pep->ring.dma + (TRBS_PER_SEGMENT * TRB_SIZE),
755 	       &pdev->adma_regs->ep_traddr);
756 
757 	return 0;
758 }
759 
760 /* Prepare and start transfer on no-default endpoint. */
761 static int cdns2_ep_run_transfer(struct cdns2_endpoint *pep,
762 				 struct cdns2_request *preq)
763 {
764 	struct cdns2_device *pdev = pep->pdev;
765 	struct cdns2_ring *ring;
766 	u32 togle_pcs = 1;
767 	int num_trbs;
768 	int ret;
769 
770 	cdns2_select_ep(pdev, pep->endpoint.address);
771 
772 	if (preq->request.sg)
773 		num_trbs = cdns2_count_sg_trbs(pep, &preq->request);
774 	else
775 		num_trbs = cdns2_count_trbs(pep, preq->request.dma,
776 					    preq->request.length);
777 
778 	ret = cdns2_prepare_ring(pdev, pep, num_trbs);
779 	if (ret)
780 		return ret;
781 
782 	ring = &pep->ring;
783 	preq->start_trb = ring->enqueue;
784 	preq->trb = ring->trbs + ring->enqueue;
785 
786 	if (usb_endpoint_xfer_isoc(pep->endpoint.desc)) {
787 		cdns2_ep_tx_isoc(pep, preq, num_trbs);
788 	} else {
789 		togle_pcs = cdns2_wa1_update_guard(pep, ring->trbs + ring->enqueue);
790 		cdns2_ep_tx_bulk(pep, preq, num_trbs);
791 	}
792 
793 	preq->num_of_trb = num_trbs;
794 
795 	/*
796 	 * Memory barrier - cycle bit must be set as the last operation.
797 	 */
798 	dma_wmb();
799 
800 	/* Give the TD to the consumer. */
801 	if (togle_pcs)
802 		preq->trb->control = preq->trb->control ^ cpu_to_le32(1);
803 
804 	cdns2_wa1_tray_restore_cycle_bit(pdev, pep);
805 	cdns2_dbg_request_trbs(pep, preq);
806 
807 	if (!pep->wa1_set && !(pep->ep_state & EP_STALLED) && !pep->skip) {
808 		if (pep->type == USB_ENDPOINT_XFER_ISOC) {
809 			ret = cdns2_prepare_first_isoc_transfer(pdev, pep);
810 			if (ret)
811 				return 0;
812 		}
813 
814 		cdns2_set_drdy(pdev, pep);
815 	}
816 
817 	return 0;
818 }
819 
820 /* Prepare and start transfer for all not started requests. */
821 static int cdns2_start_all_request(struct cdns2_device *pdev,
822 				   struct cdns2_endpoint *pep)
823 {
824 	struct cdns2_request *preq;
825 	int ret;
826 
827 	while (!list_empty(&pep->deferred_list)) {
828 		preq = cdns2_next_preq(&pep->deferred_list);
829 
830 		ret = cdns2_ep_run_transfer(pep, preq);
831 		if (ret)
832 			return ret;
833 
834 		list_move_tail(&preq->list, &pep->pending_list);
835 	}
836 
837 	pep->ep_state &= ~EP_RING_FULL;
838 
839 	return 0;
840 }
841 
842 /*
843  * Check whether trb has been handled by DMA.
844  *
845  * Endpoint must be selected before invoking this function.
846  *
847  * Returns false if request has not been handled by DMA, else returns true.
848  *
849  * SR - start ring
850  * ER - end ring
851  * DQ = ring->dequeue - dequeue position
852  * EQ = ring->enqueue - enqueue position
853  * ST = preq->start_trb - index of first TRB in transfer ring
854  * ET = preq->end_trb - index of last TRB in transfer ring
855  * CI = current_index - index of processed TRB by DMA.
856  *
857  * As first step, we check if the TRB between the ST and ET.
858  * Then, we check if cycle bit for index pep->dequeue
859  * is correct.
860  *
861  * some rules:
862  * 1. ring->dequeue never equals to current_index.
863  * 2  ring->enqueue never exceed ring->dequeue
864  * 3. exception: ring->enqueue == ring->dequeue
865  *    and ring->free_trbs is zero.
866  *    This case indicate that TR is full.
867  *
868  * At below two cases, the request have been handled.
869  * Case 1 - ring->dequeue < current_index
870  *      SR ... EQ ... DQ ... CI ... ER
871  *      SR ... DQ ... CI ... EQ ... ER
872  *
873  * Case 2 - ring->dequeue > current_index
874  * This situation takes place when CI go through the LINK TRB at the end of
875  * transfer ring.
876  *      SR ... CI ... EQ ... DQ ... ER
877  */
878 static bool cdns2_trb_handled(struct cdns2_endpoint *pep,
879 			      struct cdns2_request *preq)
880 {
881 	struct cdns2_device *pdev = pep->pdev;
882 	struct cdns2_ring *ring;
883 	struct cdns2_trb *trb;
884 	int current_index = 0;
885 	int handled = 0;
886 	int doorbell;
887 
888 	ring = &pep->ring;
889 	current_index = cdns2_get_dma_pos(pdev, pep);
890 	doorbell = !!(readl(&pdev->adma_regs->ep_cmd) & DMA_EP_CMD_DRDY);
891 
892 	/*
893 	 * Only ISO transfer can use 2 entries outside the standard
894 	 * Transfer Ring. First of them is used as zero length packet and the
895 	 * second as LINK TRB.
896 	 */
897 	if (current_index >= TRBS_PER_SEGMENT)
898 		goto finish;
899 
900 	/* Current trb doesn't belong to this request. */
901 	if (preq->start_trb < preq->end_trb) {
902 		if (ring->dequeue > preq->end_trb)
903 			goto finish;
904 
905 		if (ring->dequeue < preq->start_trb)
906 			goto finish;
907 	}
908 
909 	if (preq->start_trb > preq->end_trb && ring->dequeue > preq->end_trb &&
910 	    ring->dequeue < preq->start_trb)
911 		goto finish;
912 
913 	if (preq->start_trb == preq->end_trb && ring->dequeue != preq->end_trb)
914 		goto finish;
915 
916 	trb = &ring->trbs[ring->dequeue];
917 
918 	if ((le32_to_cpu(trb->control) & TRB_CYCLE) != ring->ccs)
919 		goto finish;
920 
921 	if (doorbell == 1 && current_index == ring->dequeue)
922 		goto finish;
923 
924 	/* The corner case for TRBS_PER_SEGMENT equal 2). */
925 	if (TRBS_PER_SEGMENT == 2 && pep->type != USB_ENDPOINT_XFER_ISOC) {
926 		handled = 1;
927 		goto finish;
928 	}
929 
930 	if (ring->enqueue == ring->dequeue &&
931 	    ring->free_trbs == 0) {
932 		handled = 1;
933 	} else if (ring->dequeue < current_index) {
934 		if ((current_index == (TRBS_PER_SEGMENT - 1)) &&
935 		    !ring->dequeue)
936 			goto finish;
937 
938 		handled = 1;
939 	} else if (ring->dequeue  > current_index) {
940 		handled = 1;
941 	}
942 
943 finish:
944 	trace_cdns2_request_handled(preq, current_index, handled);
945 
946 	return handled;
947 }
948 
949 static void cdns2_skip_isoc_td(struct cdns2_device *pdev,
950 			       struct cdns2_endpoint *pep,
951 			       struct cdns2_request *preq)
952 {
953 	struct cdns2_trb *trb;
954 	int i;
955 
956 	trb = pep->ring.trbs + pep->ring.dequeue;
957 
958 	for (i = preq->finished_trb ; i < preq->num_of_trb; i++) {
959 		preq->finished_trb++;
960 		trace_cdns2_complete_trb(pep, trb);
961 		cdns2_ep_inc_deq(&pep->ring);
962 		trb = cdns2_next_trb(pep, trb);
963 	}
964 
965 	cdns2_gadget_giveback(pep, preq, 0);
966 	cdns2_prepare_first_isoc_transfer(pdev, pep);
967 	pep->skip = false;
968 	cdns2_set_drdy(pdev, pep);
969 }
970 
971 static void cdns2_transfer_completed(struct cdns2_device *pdev,
972 				     struct cdns2_endpoint *pep)
973 {
974 	struct cdns2_request *preq = NULL;
975 	bool request_handled = false;
976 	struct cdns2_trb *trb;
977 
978 	while (!list_empty(&pep->pending_list)) {
979 		preq = cdns2_next_preq(&pep->pending_list);
980 		trb = pep->ring.trbs + pep->ring.dequeue;
981 
982 		/*
983 		 * The TRB was changed as link TRB, and the request
984 		 * was handled at ep_dequeue.
985 		 */
986 		while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK &&
987 		       le32_to_cpu(trb->length)) {
988 			trace_cdns2_complete_trb(pep, trb);
989 			cdns2_ep_inc_deq(&pep->ring);
990 			trb = pep->ring.trbs + pep->ring.dequeue;
991 		}
992 
993 		/*
994 		 * Re-select endpoint. It could be changed by other CPU
995 		 * during handling usb_gadget_giveback_request.
996 		 */
997 		cdns2_select_ep(pdev, pep->endpoint.address);
998 
999 		while (cdns2_trb_handled(pep, preq)) {
1000 			preq->finished_trb++;
1001 
1002 			if (preq->finished_trb >= preq->num_of_trb)
1003 				request_handled = true;
1004 
1005 			trb = pep->ring.trbs + pep->ring.dequeue;
1006 			trace_cdns2_complete_trb(pep, trb);
1007 
1008 			if (pep->dir && pep->type == USB_ENDPOINT_XFER_ISOC)
1009 				/*
1010 				 * For ISOC IN controller doens't update the
1011 				 * trb->length.
1012 				 */
1013 				preq->request.actual = preq->request.length;
1014 			else
1015 				preq->request.actual +=
1016 					TRB_LEN(le32_to_cpu(trb->length));
1017 
1018 			cdns2_ep_inc_deq(&pep->ring);
1019 		}
1020 
1021 		if (request_handled) {
1022 			cdns2_gadget_giveback(pep, preq, 0);
1023 			request_handled = false;
1024 		} else {
1025 			goto prepare_next_td;
1026 		}
1027 
1028 		if (pep->type != USB_ENDPOINT_XFER_ISOC &&
1029 		    TRBS_PER_SEGMENT == 2)
1030 			break;
1031 	}
1032 
1033 prepare_next_td:
1034 	if (pep->skip && preq)
1035 		cdns2_skip_isoc_td(pdev, pep, preq);
1036 
1037 	if (!(pep->ep_state & EP_STALLED) &&
1038 	    !(pep->ep_state & EP_STALL_PENDING))
1039 		cdns2_start_all_request(pdev, pep);
1040 }
1041 
1042 static void cdns2_wakeup(struct cdns2_device *pdev)
1043 {
1044 	if (!pdev->may_wakeup)
1045 		return;
1046 
1047 	/* Start driving resume signaling to indicate remote wakeup. */
1048 	set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_SIGRSUME);
1049 }
1050 
1051 static void cdns2_rearm_transfer(struct cdns2_endpoint *pep, u8 rearm)
1052 {
1053 	struct cdns2_device *pdev = pep->pdev;
1054 
1055 	cdns2_wa1_restore_cycle_bit(pep);
1056 
1057 	if (rearm) {
1058 		trace_cdns2_ring(pep);
1059 
1060 		/* Cycle Bit must be updated before arming DMA. */
1061 		dma_wmb();
1062 
1063 		writel(DMA_EP_CMD_DRDY, &pdev->adma_regs->ep_cmd);
1064 
1065 		cdns2_wakeup(pdev);
1066 
1067 		trace_cdns2_doorbell_epx(pep,
1068 					 readl(&pdev->adma_regs->ep_traddr));
1069 	}
1070 }
1071 
1072 static void cdns2_handle_epx_interrupt(struct cdns2_endpoint *pep)
1073 {
1074 	struct cdns2_device *pdev = pep->pdev;
1075 	u8 isoerror = 0;
1076 	u32 ep_sts_reg;
1077 	u32 val;
1078 
1079 	cdns2_select_ep(pdev, pep->endpoint.address);
1080 
1081 	trace_cdns2_epx_irq(pdev, pep);
1082 
1083 	ep_sts_reg = readl(&pdev->adma_regs->ep_sts);
1084 	writel(ep_sts_reg, &pdev->adma_regs->ep_sts);
1085 
1086 	if (pep->type == USB_ENDPOINT_XFER_ISOC) {
1087 		u8 mult;
1088 		u8 cs;
1089 
1090 		mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
1091 		cs = pep->dir ? readb(&pdev->epx_regs->ep[pep->num - 1].txcs) :
1092 				readb(&pdev->epx_regs->ep[pep->num - 1].rxcs);
1093 		if (mult > 0)
1094 			isoerror = EPX_CS_ERR(cs);
1095 	}
1096 
1097 	/*
1098 	 * Sometimes ISO Error for mult=1 or mult=2 is not propagated on time
1099 	 * from USB module to DMA module. To protect against this driver
1100 	 * checks also the txcs/rxcs registers.
1101 	 */
1102 	if ((ep_sts_reg & DMA_EP_STS_ISOERR) || isoerror) {
1103 		clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
1104 
1105 		/* Wait for DBUSY cleared. */
1106 		readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
1107 					  !(val & DMA_EP_STS_DBUSY), 1, 125);
1108 
1109 		writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
1110 
1111 		/* Wait for DFLUSH cleared. */
1112 		readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
1113 					  !(val & DMA_EP_CMD_DFLUSH), 1, 10);
1114 
1115 		pep->skip = true;
1116 	}
1117 
1118 	if (ep_sts_reg & DMA_EP_STS_TRBERR || pep->skip) {
1119 		if (pep->ep_state & EP_STALL_PENDING &&
1120 		    !(ep_sts_reg & DMA_EP_STS_DESCMIS))
1121 			cdns2_ep_stall_flush(pep);
1122 
1123 		/*
1124 		 * For isochronous transfer driver completes request on
1125 		 * IOC or on TRBERR. IOC appears only when device receive
1126 		 * OUT data packet. If host disable stream or lost some packet
1127 		 * then the only way to finish all queued transfer is to do it
1128 		 * on TRBERR event.
1129 		 */
1130 		if (pep->type == USB_ENDPOINT_XFER_ISOC && !pep->wa1_set) {
1131 			if (!pep->dir)
1132 				clear_reg_bit_32(&pdev->adma_regs->ep_cfg,
1133 						 DMA_EP_CFG_ENABLE);
1134 
1135 			cdns2_transfer_completed(pdev, pep);
1136 			if (pep->ep_state & EP_DEFERRED_DRDY) {
1137 				pep->ep_state &= ~EP_DEFERRED_DRDY;
1138 				cdns2_set_drdy(pdev, pep);
1139 			}
1140 
1141 			return;
1142 		}
1143 
1144 		cdns2_transfer_completed(pdev, pep);
1145 
1146 		if (!(pep->ep_state & EP_STALLED) &&
1147 		    !(pep->ep_state & EP_STALL_PENDING)) {
1148 			if (pep->ep_state & EP_DEFERRED_DRDY) {
1149 				pep->ep_state &= ~EP_DEFERRED_DRDY;
1150 				cdns2_start_all_request(pdev, pep);
1151 			} else {
1152 				cdns2_rearm_transfer(pep, pep->wa1_set);
1153 			}
1154 		}
1155 
1156 		return;
1157 	}
1158 
1159 	if ((ep_sts_reg & DMA_EP_STS_IOC) || (ep_sts_reg & DMA_EP_STS_ISP))
1160 		cdns2_transfer_completed(pdev, pep);
1161 }
1162 
1163 static void cdns2_disconnect_gadget(struct cdns2_device *pdev)
1164 {
1165 	if (pdev->gadget_driver && pdev->gadget_driver->disconnect)
1166 		pdev->gadget_driver->disconnect(&pdev->gadget);
1167 }
1168 
1169 static irqreturn_t cdns2_usb_irq_handler(int irq, void *data)
1170 {
1171 	struct cdns2_device *pdev = data;
1172 	unsigned long reg_ep_ists;
1173 	u8 reg_usb_irq_m;
1174 	u8 reg_ext_irq_m;
1175 	u8 reg_usb_irq;
1176 	u8 reg_ext_irq;
1177 
1178 	if (pdev->in_lpm)
1179 		return IRQ_NONE;
1180 
1181 	reg_usb_irq_m = readb(&pdev->interrupt_regs->usbien);
1182 	reg_ext_irq_m = readb(&pdev->interrupt_regs->extien);
1183 
1184 	/* Mask all sources of interrupt. */
1185 	writeb(0, &pdev->interrupt_regs->usbien);
1186 	writeb(0, &pdev->interrupt_regs->extien);
1187 	writel(0, &pdev->adma_regs->ep_ien);
1188 
1189 	/* Clear interrupt sources. */
1190 	writel(0, &pdev->adma_regs->ep_sts);
1191 	writeb(0, &pdev->interrupt_regs->usbirq);
1192 	writeb(0, &pdev->interrupt_regs->extirq);
1193 
1194 	reg_ep_ists = readl(&pdev->adma_regs->ep_ists);
1195 	reg_usb_irq = readb(&pdev->interrupt_regs->usbirq);
1196 	reg_ext_irq = readb(&pdev->interrupt_regs->extirq);
1197 
1198 	if (reg_ep_ists || (reg_usb_irq & reg_usb_irq_m) ||
1199 	    (reg_ext_irq & reg_ext_irq_m))
1200 		return IRQ_WAKE_THREAD;
1201 
1202 	writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
1203 	writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
1204 	writel(~0, &pdev->adma_regs->ep_ien);
1205 
1206 	return IRQ_NONE;
1207 }
1208 
1209 static irqreturn_t cdns2_thread_usb_irq_handler(struct cdns2_device *pdev)
1210 {
1211 	u8 usb_irq, ext_irq;
1212 	int speed;
1213 	int i;
1214 
1215 	ext_irq = readb(&pdev->interrupt_regs->extirq) & EXTIRQ_WAKEUP;
1216 	writeb(ext_irq, &pdev->interrupt_regs->extirq);
1217 
1218 	usb_irq = readb(&pdev->interrupt_regs->usbirq) & USB_IEN_INIT;
1219 	writeb(usb_irq, &pdev->interrupt_regs->usbirq);
1220 
1221 	if (!ext_irq && !usb_irq)
1222 		return IRQ_NONE;
1223 
1224 	trace_cdns2_usb_irq(usb_irq, ext_irq);
1225 
1226 	if (ext_irq & EXTIRQ_WAKEUP) {
1227 		if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1228 			spin_unlock(&pdev->lock);
1229 			pdev->gadget_driver->resume(&pdev->gadget);
1230 			spin_lock(&pdev->lock);
1231 		}
1232 	}
1233 
1234 	if (usb_irq & USBIRQ_LPM) {
1235 		u8 reg = readb(&pdev->usb_regs->lpmctrl);
1236 
1237 		/* LPM1 enter */
1238 		if (!(reg & LPMCTRLLH_LPMNYET))
1239 			writeb(0, &pdev->usb_regs->sleep_clkgate);
1240 	}
1241 
1242 	if (usb_irq & USBIRQ_SUSPEND) {
1243 		if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1244 			spin_unlock(&pdev->lock);
1245 			pdev->gadget_driver->suspend(&pdev->gadget);
1246 			spin_lock(&pdev->lock);
1247 		}
1248 	}
1249 
1250 	if (usb_irq & USBIRQ_URESET) {
1251 		if (pdev->gadget_driver) {
1252 			pdev->dev_address = 0;
1253 
1254 			spin_unlock(&pdev->lock);
1255 			usb_gadget_udc_reset(&pdev->gadget,
1256 					     pdev->gadget_driver);
1257 			spin_lock(&pdev->lock);
1258 
1259 			/*
1260 			 * The USBIRQ_URESET is reported at the beginning of
1261 			 * reset signal. 100ms is enough time to finish reset
1262 			 * process. For high-speed reset procedure is completed
1263 			 * when controller detect HS mode.
1264 			 */
1265 			for (i = 0; i < 100; i++) {
1266 				mdelay(1);
1267 				speed = cdns2_get_speed(pdev);
1268 				if (speed == USB_SPEED_HIGH)
1269 					break;
1270 			}
1271 
1272 			pdev->gadget.speed = speed;
1273 			cdns2_enable_l1(pdev, 0);
1274 			cdns2_ep0_config(pdev);
1275 			pdev->may_wakeup = 0;
1276 		}
1277 	}
1278 
1279 	if (usb_irq & USBIRQ_SUDAV) {
1280 		pdev->ep0_stage = CDNS2_SETUP_STAGE;
1281 		cdns2_handle_setup_packet(pdev);
1282 	}
1283 
1284 	return IRQ_HANDLED;
1285 }
1286 
1287 /* Deferred USB interrupt handler. */
1288 static irqreturn_t cdns2_thread_irq_handler(int irq, void *data)
1289 {
1290 	struct cdns2_device *pdev = data;
1291 	unsigned long  dma_ep_ists;
1292 	unsigned long flags;
1293 	unsigned int bit;
1294 
1295 	local_bh_disable();
1296 	spin_lock_irqsave(&pdev->lock, flags);
1297 
1298 	cdns2_thread_usb_irq_handler(pdev);
1299 
1300 	dma_ep_ists = readl(&pdev->adma_regs->ep_ists);
1301 	if (!dma_ep_ists)
1302 		goto unlock;
1303 
1304 	trace_cdns2_dma_ep_ists(dma_ep_ists);
1305 
1306 	/* Handle default endpoint OUT. */
1307 	if (dma_ep_ists & DMA_EP_ISTS_EP_OUT0)
1308 		cdns2_handle_ep0_interrupt(pdev, USB_DIR_OUT);
1309 
1310 	/* Handle default endpoint IN. */
1311 	if (dma_ep_ists & DMA_EP_ISTS_EP_IN0)
1312 		cdns2_handle_ep0_interrupt(pdev, USB_DIR_IN);
1313 
1314 	dma_ep_ists &= ~(DMA_EP_ISTS_EP_OUT0 | DMA_EP_ISTS_EP_IN0);
1315 
1316 	for_each_set_bit(bit, &dma_ep_ists, sizeof(u32) * BITS_PER_BYTE) {
1317 		u8 ep_idx = bit > 16 ? (bit - 16) * 2 : (bit * 2) - 1;
1318 
1319 		/*
1320 		 * Endpoints in pdev->eps[] are held in order:
1321 		 * ep0, ep1out, ep1in, ep2out, ep2in... ep15out, ep15in.
1322 		 * but in dma_ep_ists in order:
1323 		 * ep0 ep1out ep2out ... ep15out ep0in ep1in .. ep15in
1324 		 */
1325 		cdns2_handle_epx_interrupt(&pdev->eps[ep_idx]);
1326 	}
1327 
1328 unlock:
1329 	writel(~0, &pdev->adma_regs->ep_ien);
1330 	writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
1331 	writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
1332 
1333 	spin_unlock_irqrestore(&pdev->lock, flags);
1334 	local_bh_enable();
1335 
1336 	return IRQ_HANDLED;
1337 }
1338 
1339 /* Calculates and assigns onchip memory for endpoints. */
1340 static void cdns2_eps_onchip_buffer_init(struct cdns2_device *pdev)
1341 {
1342 	struct cdns2_endpoint *pep;
1343 	int min_buf_tx = 0;
1344 	int min_buf_rx = 0;
1345 	u16 tx_offset = 0;
1346 	u16 rx_offset = 0;
1347 	int free;
1348 	int i;
1349 
1350 	for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
1351 		pep = &pdev->eps[i];
1352 
1353 		if (!(pep->ep_state & EP_CLAIMED))
1354 			continue;
1355 
1356 		if (pep->dir)
1357 			min_buf_tx += pep->buffering;
1358 		else
1359 			min_buf_rx += pep->buffering;
1360 	}
1361 
1362 	for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
1363 		pep = &pdev->eps[i];
1364 
1365 		if (!(pep->ep_state & EP_CLAIMED))
1366 			continue;
1367 
1368 		if (pep->dir) {
1369 			free = pdev->onchip_tx_buf - min_buf_tx;
1370 
1371 			if (free + pep->buffering >= 4)
1372 				free = 4;
1373 			else
1374 				free = free + pep->buffering;
1375 
1376 			min_buf_tx = min_buf_tx - pep->buffering + free;
1377 
1378 			pep->buffering = free;
1379 
1380 			writel(tx_offset,
1381 			       &pdev->epx_regs->txstaddr[pep->num - 1]);
1382 			pdev->epx_regs->txstaddr[pep->num - 1] = tx_offset;
1383 
1384 			dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
1385 				pep->name, tx_offset, pep->buffering);
1386 
1387 			tx_offset += pep->buffering * 1024;
1388 		} else {
1389 			free = pdev->onchip_rx_buf - min_buf_rx;
1390 
1391 			if (free + pep->buffering >= 4)
1392 				free = 4;
1393 			else
1394 				free = free + pep->buffering;
1395 
1396 			min_buf_rx = min_buf_rx - pep->buffering + free;
1397 
1398 			pep->buffering = free;
1399 			writel(rx_offset,
1400 			       &pdev->epx_regs->rxstaddr[pep->num - 1]);
1401 
1402 			dev_dbg(pdev->dev, "%s onchip address %04x, buffering: %d\n",
1403 				pep->name, rx_offset, pep->buffering);
1404 
1405 			rx_offset += pep->buffering * 1024;
1406 		}
1407 	}
1408 }
1409 
1410 /* Configure hardware endpoint. */
1411 static int cdns2_ep_config(struct cdns2_endpoint *pep, bool enable)
1412 {
1413 	bool is_iso_ep = (pep->type == USB_ENDPOINT_XFER_ISOC);
1414 	struct cdns2_device *pdev = pep->pdev;
1415 	u32 max_packet_size;
1416 	u8 dir = 0;
1417 	u8 ep_cfg;
1418 	u8 mult;
1419 	u32 val;
1420 	int ret;
1421 
1422 	switch (pep->type) {
1423 	case USB_ENDPOINT_XFER_INT:
1424 		ep_cfg = EPX_CON_TYPE_INT;
1425 		break;
1426 	case USB_ENDPOINT_XFER_BULK:
1427 		ep_cfg = EPX_CON_TYPE_BULK;
1428 		break;
1429 	default:
1430 		mult = USB_EP_MAXP_MULT(pep->endpoint.desc->wMaxPacketSize);
1431 		ep_cfg = mult << EPX_CON_ISOD_SHIFT;
1432 		ep_cfg |= EPX_CON_TYPE_ISOC;
1433 
1434 		if (pep->dir) {
1435 			set_reg_bit_8(&pdev->epx_regs->isoautoarm, BIT(pep->num));
1436 			set_reg_bit_8(&pdev->epx_regs->isoautodump, BIT(pep->num));
1437 			set_reg_bit_8(&pdev->epx_regs->isodctrl, BIT(pep->num));
1438 		}
1439 	}
1440 
1441 	switch (pdev->gadget.speed) {
1442 	case USB_SPEED_FULL:
1443 		max_packet_size = is_iso_ep ? 1023 : 64;
1444 		break;
1445 	case USB_SPEED_HIGH:
1446 		max_packet_size = is_iso_ep ? 1024 : 512;
1447 		break;
1448 	default:
1449 		/* All other speed are not supported. */
1450 		return -EINVAL;
1451 	}
1452 
1453 	ep_cfg |= (EPX_CON_VAL | (pep->buffering - 1));
1454 
1455 	if (pep->dir) {
1456 		dir = FIFOCTRL_IO_TX;
1457 		writew(max_packet_size, &pdev->epx_regs->txmaxpack[pep->num - 1]);
1458 		writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].txcon);
1459 	} else {
1460 		writew(max_packet_size, &pdev->epx_regs->rxmaxpack[pep->num - 1]);
1461 		writeb(ep_cfg, &pdev->epx_regs->ep[pep->num - 1].rxcon);
1462 	}
1463 
1464 	writeb(pep->num | dir | FIFOCTRL_FIFOAUTO,
1465 	       &pdev->usb_regs->fifoctrl);
1466 	writeb(pep->num | dir, &pdev->epx_regs->endprst);
1467 	writeb(pep->num | ENDPRST_FIFORST | ENDPRST_TOGRST | dir,
1468 	       &pdev->epx_regs->endprst);
1469 
1470 	if (max_packet_size == 1024)
1471 		pep->trb_burst_size = 128;
1472 	else if (max_packet_size >= 512)
1473 		pep->trb_burst_size = 64;
1474 	else
1475 		pep->trb_burst_size = 16;
1476 
1477 	cdns2_select_ep(pdev, pep->num | pep->dir);
1478 	writel(DMA_EP_CMD_EPRST | DMA_EP_CMD_DFLUSH, &pdev->adma_regs->ep_cmd);
1479 
1480 	ret = readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
1481 					!(val & (DMA_EP_CMD_DFLUSH |
1482 					DMA_EP_CMD_EPRST)),
1483 					1, 1000);
1484 
1485 	if (ret)
1486 		return ret;
1487 
1488 	writel(DMA_EP_STS_TRBERR | DMA_EP_STS_ISOERR, &pdev->adma_regs->ep_sts_en);
1489 
1490 	if (enable)
1491 		writel(DMA_EP_CFG_ENABLE, &pdev->adma_regs->ep_cfg);
1492 
1493 	trace_cdns2_epx_hw_cfg(pdev, pep);
1494 
1495 	dev_dbg(pdev->dev, "Configure %s: with MPS: %08x, ep con: %02x\n",
1496 		pep->name, max_packet_size, ep_cfg);
1497 
1498 	return 0;
1499 }
1500 
1501 struct usb_request *cdns2_gadget_ep_alloc_request(struct usb_ep *ep,
1502 						  gfp_t gfp_flags)
1503 {
1504 	struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
1505 	struct cdns2_request *preq;
1506 
1507 	preq = kzalloc(sizeof(*preq), gfp_flags);
1508 	if (!preq)
1509 		return NULL;
1510 
1511 	preq->pep = pep;
1512 
1513 	trace_cdns2_alloc_request(preq);
1514 
1515 	return &preq->request;
1516 }
1517 
1518 void cdns2_gadget_ep_free_request(struct usb_ep *ep,
1519 				  struct usb_request *request)
1520 {
1521 	struct cdns2_request *preq = to_cdns2_request(request);
1522 
1523 	trace_cdns2_free_request(preq);
1524 	kfree(preq);
1525 }
1526 
1527 static int cdns2_gadget_ep_enable(struct usb_ep *ep,
1528 				  const struct usb_endpoint_descriptor *desc)
1529 {
1530 	u32 reg = DMA_EP_STS_EN_TRBERREN;
1531 	struct cdns2_endpoint *pep;
1532 	struct cdns2_device *pdev;
1533 	unsigned long flags;
1534 	int enable = 1;
1535 	int ret = 0;
1536 
1537 	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
1538 	    !desc->wMaxPacketSize) {
1539 		return -EINVAL;
1540 	}
1541 
1542 	pep = ep_to_cdns2_ep(ep);
1543 	pdev = pep->pdev;
1544 
1545 	if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
1546 			  "%s is already enabled\n", pep->name))
1547 		return 0;
1548 
1549 	spin_lock_irqsave(&pdev->lock, flags);
1550 
1551 	pep->type = usb_endpoint_type(desc);
1552 	pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
1553 
1554 	if (pdev->gadget.speed == USB_SPEED_FULL)
1555 		if (pep->type == USB_ENDPOINT_XFER_INT)
1556 			pep->interval = desc->bInterval;
1557 
1558 	if (pep->interval > ISO_MAX_INTERVAL &&
1559 	    pep->type == USB_ENDPOINT_XFER_ISOC) {
1560 		dev_err(pdev->dev, "ISO period is limited to %d (current: %d)\n",
1561 			ISO_MAX_INTERVAL, pep->interval);
1562 
1563 		ret =  -EINVAL;
1564 		goto exit;
1565 	}
1566 
1567 	/*
1568 	 * During ISO OUT traffic DMA reads Transfer Ring for the EP which has
1569 	 * never got doorbell.
1570 	 * This issue was detected only on simulation, but to avoid this issue
1571 	 * driver add protection against it. To fix it driver enable ISO OUT
1572 	 * endpoint before setting DRBL. This special treatment of ISO OUT
1573 	 * endpoints are recommended by controller specification.
1574 	 */
1575 	if (pep->type == USB_ENDPOINT_XFER_ISOC  && !pep->dir)
1576 		enable = 0;
1577 
1578 	ret = cdns2_alloc_tr_segment(pep);
1579 	if (ret)
1580 		goto exit;
1581 
1582 	ret = cdns2_ep_config(pep, enable);
1583 	if (ret) {
1584 		cdns2_free_tr_segment(pep);
1585 		ret =  -EINVAL;
1586 		goto exit;
1587 	}
1588 
1589 	trace_cdns2_gadget_ep_enable(pep);
1590 
1591 	pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
1592 	pep->ep_state |= EP_ENABLED;
1593 	pep->wa1_set = 0;
1594 	pep->ring.enqueue = 0;
1595 	pep->ring.dequeue = 0;
1596 	reg = readl(&pdev->adma_regs->ep_sts);
1597 	pep->ring.pcs = !!DMA_EP_STS_CCS(reg);
1598 	pep->ring.ccs = !!DMA_EP_STS_CCS(reg);
1599 
1600 	writel(pep->ring.dma, &pdev->adma_regs->ep_traddr);
1601 
1602 	/* one TRB is reserved for link TRB used in DMULT mode*/
1603 	pep->ring.free_trbs = TRBS_PER_SEGMENT - 1;
1604 
1605 exit:
1606 	spin_unlock_irqrestore(&pdev->lock, flags);
1607 
1608 	return ret;
1609 }
1610 
1611 static int cdns2_gadget_ep_disable(struct usb_ep *ep)
1612 {
1613 	struct cdns2_endpoint *pep;
1614 	struct cdns2_request *preq;
1615 	struct cdns2_device *pdev;
1616 	unsigned long flags;
1617 	int val;
1618 
1619 	if (!ep)
1620 		return -EINVAL;
1621 
1622 	pep = ep_to_cdns2_ep(ep);
1623 	pdev = pep->pdev;
1624 
1625 	if (dev_WARN_ONCE(pdev->dev, !(pep->ep_state & EP_ENABLED),
1626 			  "%s is already disabled\n", pep->name))
1627 		return 0;
1628 
1629 	spin_lock_irqsave(&pdev->lock, flags);
1630 
1631 	trace_cdns2_gadget_ep_disable(pep);
1632 
1633 	cdns2_select_ep(pdev, ep->desc->bEndpointAddress);
1634 
1635 	clear_reg_bit_32(&pdev->adma_regs->ep_cfg, DMA_EP_CFG_ENABLE);
1636 
1637 	/*
1638 	 * Driver needs some time before resetting endpoint.
1639 	 * It need waits for clearing DBUSY bit or for timeout expired.
1640 	 * 10us is enough time for controller to stop transfer.
1641 	 */
1642 	readl_poll_timeout_atomic(&pdev->adma_regs->ep_sts, val,
1643 				  !(val & DMA_EP_STS_DBUSY), 1, 10);
1644 	writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
1645 
1646 	readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
1647 				  !(val & (DMA_EP_CMD_DFLUSH | DMA_EP_CMD_EPRST)),
1648 				  1, 1000);
1649 
1650 	while (!list_empty(&pep->pending_list)) {
1651 		preq = cdns2_next_preq(&pep->pending_list);
1652 		cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
1653 	}
1654 
1655 	while (!list_empty(&pep->deferred_list)) {
1656 		preq = cdns2_next_preq(&pep->deferred_list);
1657 		cdns2_gadget_giveback(pep, preq, -ESHUTDOWN);
1658 	}
1659 
1660 	ep->desc = NULL;
1661 	pep->ep_state &= ~EP_ENABLED;
1662 
1663 	spin_unlock_irqrestore(&pdev->lock, flags);
1664 
1665 	return 0;
1666 }
1667 
1668 static int cdns2_ep_enqueue(struct cdns2_endpoint *pep,
1669 			    struct cdns2_request *preq,
1670 			    gfp_t gfp_flags)
1671 {
1672 	struct cdns2_device *pdev = pep->pdev;
1673 	struct usb_request *request;
1674 	int ret;
1675 
1676 	request = &preq->request;
1677 	request->actual = 0;
1678 	request->status = -EINPROGRESS;
1679 
1680 	ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->dir);
1681 	if (ret) {
1682 		trace_cdns2_request_enqueue_error(preq);
1683 		return ret;
1684 	}
1685 
1686 	list_add_tail(&preq->list, &pep->deferred_list);
1687 	trace_cdns2_request_enqueue(preq);
1688 
1689 	if (!(pep->ep_state & EP_STALLED) && !(pep->ep_state & EP_STALL_PENDING))
1690 		cdns2_start_all_request(pdev, pep);
1691 
1692 	return 0;
1693 }
1694 
1695 static int cdns2_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1696 				 gfp_t gfp_flags)
1697 {
1698 	struct usb_request *zlp_request;
1699 	struct cdns2_request *preq;
1700 	struct cdns2_endpoint *pep;
1701 	struct cdns2_device *pdev;
1702 	unsigned long flags;
1703 	int ret;
1704 
1705 	if (!request || !ep)
1706 		return -EINVAL;
1707 
1708 	pep = ep_to_cdns2_ep(ep);
1709 	pdev = pep->pdev;
1710 
1711 	if (!(pep->ep_state & EP_ENABLED)) {
1712 		dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1713 			pep->name);
1714 		return -EINVAL;
1715 	}
1716 
1717 	spin_lock_irqsave(&pdev->lock, flags);
1718 
1719 	preq =  to_cdns2_request(request);
1720 	ret = cdns2_ep_enqueue(pep, preq, gfp_flags);
1721 
1722 	if (ret == 0 && request->zero && request->length &&
1723 	    (request->length % ep->maxpacket == 0)) {
1724 		struct cdns2_request *preq;
1725 
1726 		zlp_request = cdns2_gadget_ep_alloc_request(ep, GFP_ATOMIC);
1727 		zlp_request->buf = pdev->zlp_buf;
1728 		zlp_request->length = 0;
1729 
1730 		preq = to_cdns2_request(zlp_request);
1731 		ret = cdns2_ep_enqueue(pep, preq, gfp_flags);
1732 	}
1733 
1734 	spin_unlock_irqrestore(&pdev->lock, flags);
1735 	return ret;
1736 }
1737 
1738 int cdns2_gadget_ep_dequeue(struct usb_ep *ep,
1739 			    struct usb_request *request)
1740 {
1741 	struct cdns2_request *preq, *preq_temp, *cur_preq;
1742 	struct cdns2_endpoint *pep;
1743 	struct cdns2_trb *link_trb;
1744 	u8 req_on_hw_ring = 0;
1745 	unsigned long flags;
1746 	u32 buffer;
1747 	int val, i;
1748 
1749 	if (!ep || !request || !ep->desc)
1750 		return -EINVAL;
1751 
1752 	pep = ep_to_cdns2_ep(ep);
1753 	if (!pep->endpoint.desc) {
1754 		dev_err(pep->pdev->dev, "%s: can't dequeue to disabled endpoint\n",
1755 			pep->name);
1756 		return -ESHUTDOWN;
1757 	}
1758 
1759 	/* Requests has been dequeued during disabling endpoint. */
1760 	if (!(pep->ep_state & EP_ENABLED))
1761 		return 0;
1762 
1763 	spin_lock_irqsave(&pep->pdev->lock, flags);
1764 
1765 	cur_preq = to_cdns2_request(request);
1766 	trace_cdns2_request_dequeue(cur_preq);
1767 
1768 	list_for_each_entry_safe(preq, preq_temp, &pep->pending_list, list) {
1769 		if (cur_preq == preq) {
1770 			req_on_hw_ring = 1;
1771 			goto found;
1772 		}
1773 	}
1774 
1775 	list_for_each_entry_safe(preq, preq_temp, &pep->deferred_list, list) {
1776 		if (cur_preq == preq)
1777 			goto found;
1778 	}
1779 
1780 	goto not_found;
1781 
1782 found:
1783 	link_trb = preq->trb;
1784 
1785 	/* Update ring only if removed request is on pending_req_list list. */
1786 	if (req_on_hw_ring && link_trb) {
1787 		/* Stop DMA */
1788 		writel(DMA_EP_CMD_DFLUSH, &pep->pdev->adma_regs->ep_cmd);
1789 
1790 		/* Wait for DFLUSH cleared. */
1791 		readl_poll_timeout_atomic(&pep->pdev->adma_regs->ep_cmd, val,
1792 					  !(val & DMA_EP_CMD_DFLUSH), 1, 1000);
1793 
1794 		buffer = cpu_to_le32(TRB_BUFFER(pep->ring.dma +
1795 				    ((preq->end_trb + 1) * TRB_SIZE)));
1796 
1797 		for (i = 0; i < preq->num_of_trb; i++) {
1798 			link_trb->buffer = buffer;
1799 			link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control)
1800 					    & TRB_CYCLE) | TRB_CHAIN |
1801 					    TRB_TYPE(TRB_LINK));
1802 
1803 			trace_cdns2_queue_trb(pep, link_trb);
1804 			link_trb = cdns2_next_trb(pep, link_trb);
1805 		}
1806 
1807 		if (pep->wa1_trb == preq->trb)
1808 			cdns2_wa1_restore_cycle_bit(pep);
1809 	}
1810 
1811 	cdns2_gadget_giveback(pep, cur_preq, -ECONNRESET);
1812 
1813 	preq = cdns2_next_preq(&pep->pending_list);
1814 	if (preq)
1815 		cdns2_rearm_transfer(pep, 1);
1816 
1817 not_found:
1818 	spin_unlock_irqrestore(&pep->pdev->lock, flags);
1819 	return 0;
1820 }
1821 
1822 int cdns2_halt_endpoint(struct cdns2_device *pdev,
1823 			struct cdns2_endpoint *pep,
1824 			int value)
1825 {
1826 	u8 __iomem *conf;
1827 	int dir = 0;
1828 
1829 	if (!(pep->ep_state & EP_ENABLED))
1830 		return -EPERM;
1831 
1832 	if (pep->dir) {
1833 		dir = ENDPRST_IO_TX;
1834 		conf = &pdev->epx_regs->ep[pep->num - 1].txcon;
1835 	} else {
1836 		conf = &pdev->epx_regs->ep[pep->num - 1].rxcon;
1837 	}
1838 
1839 	if (!value) {
1840 		struct cdns2_trb *trb = NULL;
1841 		struct cdns2_request *preq;
1842 		struct cdns2_trb trb_tmp;
1843 
1844 		preq = cdns2_next_preq(&pep->pending_list);
1845 		if (preq) {
1846 			trb = preq->trb;
1847 			if (trb) {
1848 				trb_tmp = *trb;
1849 				trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
1850 			}
1851 		}
1852 
1853 		trace_cdns2_ep_halt(pep, 0, 0);
1854 
1855 		/* Resets Sequence Number */
1856 		writeb(dir | pep->num, &pdev->epx_regs->endprst);
1857 		writeb(dir | ENDPRST_TOGRST | pep->num,
1858 		       &pdev->epx_regs->endprst);
1859 
1860 		clear_reg_bit_8(conf, EPX_CON_STALL);
1861 
1862 		pep->ep_state &= ~(EP_STALLED | EP_STALL_PENDING);
1863 
1864 		if (preq) {
1865 			if (trb)
1866 				*trb = trb_tmp;
1867 
1868 			cdns2_rearm_transfer(pep, 1);
1869 		}
1870 
1871 		cdns2_start_all_request(pdev, pep);
1872 	} else {
1873 		trace_cdns2_ep_halt(pep, 1, 0);
1874 		set_reg_bit_8(conf, EPX_CON_STALL);
1875 		writeb(dir | pep->num, &pdev->epx_regs->endprst);
1876 		writeb(dir | ENDPRST_FIFORST | pep->num,
1877 		       &pdev->epx_regs->endprst);
1878 		pep->ep_state |= EP_STALLED;
1879 	}
1880 
1881 	return 0;
1882 }
1883 
1884 /* Sets/clears stall on selected endpoint. */
1885 static int cdns2_gadget_ep_set_halt(struct usb_ep *ep, int value)
1886 {
1887 	struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
1888 	struct cdns2_device *pdev = pep->pdev;
1889 	struct cdns2_request *preq;
1890 	unsigned long flags = 0;
1891 	int ret;
1892 
1893 	spin_lock_irqsave(&pdev->lock, flags);
1894 
1895 	preq = cdns2_next_preq(&pep->pending_list);
1896 	if (value && preq) {
1897 		trace_cdns2_ep_busy_try_halt_again(pep);
1898 		ret = -EAGAIN;
1899 		goto done;
1900 	}
1901 
1902 	if (!value)
1903 		pep->ep_state &= ~EP_WEDGE;
1904 
1905 	ret = cdns2_halt_endpoint(pdev, pep, value);
1906 
1907 done:
1908 	spin_unlock_irqrestore(&pdev->lock, flags);
1909 	return ret;
1910 }
1911 
1912 static int cdns2_gadget_ep_set_wedge(struct usb_ep *ep)
1913 {
1914 	struct cdns2_endpoint *pep = ep_to_cdns2_ep(ep);
1915 
1916 	cdns2_gadget_ep_set_halt(ep, 1);
1917 	pep->ep_state |= EP_WEDGE;
1918 
1919 	return 0;
1920 }
1921 
1922 static struct
1923 cdns2_endpoint *cdns2_find_available_ep(struct cdns2_device *pdev,
1924 					struct usb_endpoint_descriptor *desc)
1925 {
1926 	struct cdns2_endpoint *pep;
1927 	struct usb_ep *ep;
1928 	int ep_correct;
1929 
1930 	list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
1931 		unsigned long num;
1932 		int ret;
1933 		/* ep name pattern likes epXin or epXout. */
1934 		char c[2] = {ep->name[2], '\0'};
1935 
1936 		ret = kstrtoul(c, 10, &num);
1937 		if (ret)
1938 			return ERR_PTR(ret);
1939 		pep = ep_to_cdns2_ep(ep);
1940 
1941 		if (pep->num != num)
1942 			continue;
1943 
1944 		ep_correct = (pep->endpoint.caps.dir_in &&
1945 			      usb_endpoint_dir_in(desc)) ||
1946 			     (pep->endpoint.caps.dir_out &&
1947 			      usb_endpoint_dir_out(desc));
1948 
1949 		if (ep_correct && !(pep->ep_state & EP_CLAIMED))
1950 			return pep;
1951 	}
1952 
1953 	return ERR_PTR(-ENOENT);
1954 }
1955 
1956 /*
1957  * Function used to recognize which endpoints will be used to optimize
1958  * on-chip memory usage.
1959  */
1960 static struct
1961 usb_ep *cdns2_gadget_match_ep(struct usb_gadget *gadget,
1962 			      struct usb_endpoint_descriptor *desc,
1963 			      struct usb_ss_ep_comp_descriptor *comp_desc)
1964 {
1965 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
1966 	struct cdns2_endpoint *pep;
1967 	unsigned long flags;
1968 
1969 	pep = cdns2_find_available_ep(pdev, desc);
1970 	if (IS_ERR(pep)) {
1971 		dev_err(pdev->dev, "no available ep\n");
1972 		return NULL;
1973 	}
1974 
1975 	spin_lock_irqsave(&pdev->lock, flags);
1976 
1977 	if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
1978 		pep->buffering = 4;
1979 	else
1980 		pep->buffering = 1;
1981 
1982 	pep->ep_state |= EP_CLAIMED;
1983 	spin_unlock_irqrestore(&pdev->lock, flags);
1984 
1985 	return &pep->endpoint;
1986 }
1987 
1988 static const struct usb_ep_ops cdns2_gadget_ep_ops = {
1989 	.enable = cdns2_gadget_ep_enable,
1990 	.disable = cdns2_gadget_ep_disable,
1991 	.alloc_request = cdns2_gadget_ep_alloc_request,
1992 	.free_request = cdns2_gadget_ep_free_request,
1993 	.queue = cdns2_gadget_ep_queue,
1994 	.dequeue = cdns2_gadget_ep_dequeue,
1995 	.set_halt = cdns2_gadget_ep_set_halt,
1996 	.set_wedge = cdns2_gadget_ep_set_wedge,
1997 };
1998 
1999 static int cdns2_gadget_get_frame(struct usb_gadget *gadget)
2000 {
2001 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2002 
2003 	return readw(&pdev->usb_regs->frmnr);
2004 }
2005 
2006 static int cdns2_gadget_wakeup(struct usb_gadget *gadget)
2007 {
2008 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2009 	unsigned long flags;
2010 
2011 	spin_lock_irqsave(&pdev->lock, flags);
2012 	cdns2_wakeup(pdev);
2013 	spin_unlock_irqrestore(&pdev->lock, flags);
2014 
2015 	return 0;
2016 }
2017 
2018 static int cdns2_gadget_set_selfpowered(struct usb_gadget *gadget,
2019 					int is_selfpowered)
2020 {
2021 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2022 	unsigned long flags;
2023 
2024 	spin_lock_irqsave(&pdev->lock, flags);
2025 	pdev->is_selfpowered = !!is_selfpowered;
2026 	spin_unlock_irqrestore(&pdev->lock, flags);
2027 	return 0;
2028 }
2029 
2030 /*  Disable interrupts and begin the controller halting process. */
2031 static void cdns2_quiesce(struct cdns2_device *pdev)
2032 {
2033 	set_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2034 
2035 	/* Disable interrupt. */
2036 	writeb(0, &pdev->interrupt_regs->extien),
2037 	writeb(0, &pdev->interrupt_regs->usbien),
2038 	writew(0, &pdev->adma_regs->ep_ien);
2039 
2040 	/* Clear interrupt line. */
2041 	writeb(0x0, &pdev->interrupt_regs->usbirq);
2042 }
2043 
2044 static void cdns2_gadget_config(struct cdns2_device *pdev)
2045 {
2046 	cdns2_ep0_config(pdev);
2047 
2048 	/* Enable DMA interrupts for all endpoints. */
2049 	writel(~0x0, &pdev->adma_regs->ep_ien);
2050 	cdns2_enable_l1(pdev, 0);
2051 	writeb(USB_IEN_INIT, &pdev->interrupt_regs->usbien);
2052 	writeb(EXTIRQ_WAKEUP, &pdev->interrupt_regs->extien);
2053 	writel(DMA_CONF_DMULT, &pdev->adma_regs->conf);
2054 }
2055 
2056 static int cdns2_gadget_pullup(struct usb_gadget *gadget, int is_on)
2057 {
2058 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2059 	unsigned long flags;
2060 
2061 	trace_cdns2_pullup(is_on);
2062 
2063 	/*
2064 	 * Disable events handling while controller is being
2065 	 * enabled/disabled.
2066 	 */
2067 	disable_irq(pdev->irq);
2068 	spin_lock_irqsave(&pdev->lock, flags);
2069 
2070 	if (is_on) {
2071 		cdns2_gadget_config(pdev);
2072 		clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2073 	} else {
2074 		cdns2_quiesce(pdev);
2075 	}
2076 
2077 	spin_unlock_irqrestore(&pdev->lock, flags);
2078 	enable_irq(pdev->irq);
2079 
2080 	return 0;
2081 }
2082 
2083 static int cdns2_gadget_udc_start(struct usb_gadget *gadget,
2084 				  struct usb_gadget_driver *driver)
2085 {
2086 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2087 	enum usb_device_speed max_speed = driver->max_speed;
2088 	unsigned long flags;
2089 
2090 	spin_lock_irqsave(&pdev->lock, flags);
2091 	pdev->gadget_driver = driver;
2092 
2093 	/* Limit speed if necessary. */
2094 	max_speed = min(driver->max_speed, gadget->max_speed);
2095 
2096 	switch (max_speed) {
2097 	case USB_SPEED_FULL:
2098 		writeb(SPEEDCTRL_HSDISABLE, &pdev->usb_regs->speedctrl);
2099 		break;
2100 	case USB_SPEED_HIGH:
2101 		writeb(0, &pdev->usb_regs->speedctrl);
2102 		break;
2103 	default:
2104 		dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
2105 			max_speed);
2106 		fallthrough;
2107 	case USB_SPEED_UNKNOWN:
2108 		/* Default to highspeed. */
2109 		max_speed = USB_SPEED_HIGH;
2110 		break;
2111 	}
2112 
2113 	/* Reset all USB endpoints. */
2114 	writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
2115 	writeb(ENDPRST_FIFORST | ENDPRST_TOGRST | ENDPRST_IO_TX,
2116 	       &pdev->usb_regs->endprst);
2117 	writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->usb_regs->endprst);
2118 
2119 	cdns2_eps_onchip_buffer_init(pdev);
2120 
2121 	cdns2_gadget_config(pdev);
2122 	spin_unlock_irqrestore(&pdev->lock, flags);
2123 
2124 	return 0;
2125 }
2126 
2127 static int cdns2_gadget_udc_stop(struct usb_gadget *gadget)
2128 {
2129 	struct cdns2_device *pdev = gadget_to_cdns2_device(gadget);
2130 	struct cdns2_endpoint *pep;
2131 	u32 bEndpointAddress;
2132 	struct usb_ep *ep;
2133 	int val;
2134 
2135 	pdev->gadget_driver = NULL;
2136 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
2137 
2138 	list_for_each_entry(ep, &pdev->gadget.ep_list, ep_list) {
2139 		pep = ep_to_cdns2_ep(ep);
2140 		bEndpointAddress = pep->num | pep->dir;
2141 		cdns2_select_ep(pdev, bEndpointAddress);
2142 		writel(DMA_EP_CMD_EPRST, &pdev->adma_regs->ep_cmd);
2143 		readl_poll_timeout_atomic(&pdev->adma_regs->ep_cmd, val,
2144 					  !(val & DMA_EP_CMD_EPRST), 1, 100);
2145 	}
2146 
2147 	cdns2_quiesce(pdev);
2148 
2149 	writeb(ENDPRST_IO_TX, &pdev->usb_regs->endprst);
2150 	writeb(ENDPRST_FIFORST | ENDPRST_TOGRST | ENDPRST_IO_TX,
2151 	       &pdev->epx_regs->endprst);
2152 	writeb(ENDPRST_FIFORST | ENDPRST_TOGRST, &pdev->epx_regs->endprst);
2153 
2154 	return 0;
2155 }
2156 
2157 static const struct usb_gadget_ops cdns2_gadget_ops = {
2158 	.get_frame = cdns2_gadget_get_frame,
2159 	.wakeup = cdns2_gadget_wakeup,
2160 	.set_selfpowered = cdns2_gadget_set_selfpowered,
2161 	.pullup = cdns2_gadget_pullup,
2162 	.udc_start = cdns2_gadget_udc_start,
2163 	.udc_stop = cdns2_gadget_udc_stop,
2164 	.match_ep = cdns2_gadget_match_ep,
2165 };
2166 
2167 static void cdns2_free_all_eps(struct cdns2_device *pdev)
2168 {
2169 	int i;
2170 
2171 	for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++)
2172 		cdns2_free_tr_segment(&pdev->eps[i]);
2173 }
2174 
2175 /* Initializes software endpoints of gadget. */
2176 static int cdns2_init_eps(struct cdns2_device *pdev)
2177 {
2178 	struct cdns2_endpoint *pep;
2179 	int i;
2180 
2181 	for (i = 0; i < CDNS2_ENDPOINTS_NUM; i++) {
2182 		bool direction = !(i & 1); /* Start from OUT endpoint. */
2183 		u8 epnum = ((i + 1) >> 1);
2184 
2185 		/*
2186 		 * Endpoints are being held in pdev->eps[] in form:
2187 		 * ep0, ep1out, ep1in ... ep15out, ep15in.
2188 		 */
2189 		if (!CDNS2_IF_EP_EXIST(pdev, epnum, direction))
2190 			continue;
2191 
2192 		pep = &pdev->eps[i];
2193 		pep->pdev = pdev;
2194 		pep->num = epnum;
2195 		/* 0 for OUT, 1 for IN. */
2196 		pep->dir = direction ? USB_DIR_IN : USB_DIR_OUT;
2197 		pep->idx = i;
2198 
2199 		/* Ep0in and ep0out are represented by pdev->eps[0]. */
2200 		if (!epnum) {
2201 			int ret;
2202 
2203 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
2204 				 epnum, "BiDir");
2205 
2206 			cdns2_init_ep0(pdev, pep);
2207 
2208 			ret = cdns2_alloc_tr_segment(pep);
2209 			if (ret) {
2210 				dev_err(pdev->dev, "Failed to init ep0\n");
2211 				return ret;
2212 			}
2213 		} else {
2214 			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
2215 				 epnum, !!direction ? "in" : "out");
2216 			pep->endpoint.name = pep->name;
2217 
2218 			usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
2219 			pep->endpoint.ops = &cdns2_gadget_ep_ops;
2220 			list_add_tail(&pep->endpoint.ep_list, &pdev->gadget.ep_list);
2221 
2222 			pep->endpoint.caps.dir_in = direction;
2223 			pep->endpoint.caps.dir_out = !direction;
2224 
2225 			pep->endpoint.caps.type_iso = 1;
2226 			pep->endpoint.caps.type_bulk = 1;
2227 			pep->endpoint.caps.type_int = 1;
2228 		}
2229 
2230 		pep->endpoint.name = pep->name;
2231 		pep->ep_state = 0;
2232 
2233 		dev_dbg(pdev->dev, "Init %s, SupType: CTRL: %s, INT: %s, "
2234 			"BULK: %s, ISOC %s, SupDir IN: %s, OUT: %s\n",
2235 			pep->name,
2236 			(pep->endpoint.caps.type_control) ? "yes" : "no",
2237 			(pep->endpoint.caps.type_int) ? "yes" : "no",
2238 			(pep->endpoint.caps.type_bulk) ? "yes" : "no",
2239 			(pep->endpoint.caps.type_iso) ? "yes" : "no",
2240 			(pep->endpoint.caps.dir_in) ? "yes" : "no",
2241 			(pep->endpoint.caps.dir_out) ? "yes" : "no");
2242 
2243 		INIT_LIST_HEAD(&pep->pending_list);
2244 		INIT_LIST_HEAD(&pep->deferred_list);
2245 	}
2246 
2247 	return 0;
2248 }
2249 
2250 static int cdns2_gadget_start(struct cdns2_device *pdev)
2251 {
2252 	u32 max_speed;
2253 	void *buf;
2254 	int val;
2255 	int ret;
2256 
2257 	pdev->usb_regs = pdev->regs;
2258 	pdev->ep0_regs = pdev->regs;
2259 	pdev->epx_regs = pdev->regs;
2260 	pdev->interrupt_regs = pdev->regs;
2261 	pdev->adma_regs = pdev->regs + CDNS2_ADMA_REGS_OFFSET;
2262 
2263 	/* Reset controller. */
2264 	set_reg_bit_8(&pdev->usb_regs->cpuctrl, CPUCTRL_SW_RST);
2265 
2266 	ret = readl_poll_timeout_atomic(&pdev->usb_regs->cpuctrl, val,
2267 					!(val & CPUCTRL_SW_RST), 1, 10000);
2268 	if (ret) {
2269 		dev_err(pdev->dev, "Error: reset controller timeout\n");
2270 		return -EINVAL;
2271 	}
2272 
2273 	usb_initialize_gadget(pdev->dev, &pdev->gadget, NULL);
2274 
2275 	device_property_read_u16(pdev->dev, "cdns,on-chip-tx-buff-size",
2276 				 &pdev->onchip_tx_buf);
2277 	device_property_read_u16(pdev->dev, "cdns,on-chip-rx-buff-size",
2278 				 &pdev->onchip_rx_buf);
2279 	device_property_read_u32(pdev->dev, "cdns,avail-endpoints",
2280 				 &pdev->eps_supported);
2281 
2282 	/*
2283 	 * Driver assumes that each USBHS controller has at least
2284 	 * one IN and one OUT non control endpoint.
2285 	 */
2286 	if (!pdev->onchip_tx_buf && !pdev->onchip_rx_buf) {
2287 		ret = -EINVAL;
2288 		dev_err(pdev->dev, "Invalid on-chip memory configuration\n");
2289 		goto put_gadget;
2290 	}
2291 
2292 	if (!(pdev->eps_supported & ~0x00010001)) {
2293 		ret = -EINVAL;
2294 		dev_err(pdev->dev, "No hardware endpoints available\n");
2295 		goto put_gadget;
2296 	}
2297 
2298 	max_speed = usb_get_maximum_speed(pdev->dev);
2299 
2300 	switch (max_speed) {
2301 	case USB_SPEED_FULL:
2302 	case USB_SPEED_HIGH:
2303 		break;
2304 	default:
2305 		dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
2306 			max_speed);
2307 		fallthrough;
2308 	case USB_SPEED_UNKNOWN:
2309 		max_speed = USB_SPEED_HIGH;
2310 		break;
2311 	}
2312 
2313 	pdev->gadget.max_speed = max_speed;
2314 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
2315 	pdev->gadget.ops = &cdns2_gadget_ops;
2316 	pdev->gadget.name = "usbhs-gadget";
2317 	pdev->gadget.quirk_avoids_skb_reserve = 1;
2318 	pdev->gadget.irq = pdev->irq;
2319 
2320 	spin_lock_init(&pdev->lock);
2321 	INIT_WORK(&pdev->pending_status_wq, cdns2_pending_setup_status_handler);
2322 
2323 	/* Initialize endpoint container. */
2324 	INIT_LIST_HEAD(&pdev->gadget.ep_list);
2325 	pdev->eps_dma_pool = dma_pool_create("cdns2_eps_dma_pool", pdev->dev,
2326 					     TR_SEG_SIZE, 8, 0);
2327 	if (!pdev->eps_dma_pool) {
2328 		dev_err(pdev->dev, "Failed to create TRB dma pool\n");
2329 		ret = -ENOMEM;
2330 		goto put_gadget;
2331 	}
2332 
2333 	ret = cdns2_init_eps(pdev);
2334 	if (ret) {
2335 		dev_err(pdev->dev, "Failed to create endpoints\n");
2336 		goto destroy_dma_pool;
2337 	}
2338 
2339 	pdev->gadget.sg_supported = 1;
2340 
2341 	pdev->zlp_buf = kzalloc(CDNS2_EP_ZLP_BUF_SIZE, GFP_KERNEL);
2342 	if (!pdev->zlp_buf) {
2343 		ret = -ENOMEM;
2344 		goto destroy_dma_pool;
2345 	}
2346 
2347 	/* Allocate memory for setup packet buffer. */
2348 	buf = dma_alloc_coherent(pdev->dev, 8, &pdev->ep0_preq.request.dma,
2349 				 GFP_DMA);
2350 	pdev->ep0_preq.request.buf = buf;
2351 
2352 	if (!pdev->ep0_preq.request.buf) {
2353 		ret = -ENOMEM;
2354 		goto free_zlp_buf;
2355 	}
2356 
2357 	/* Add USB gadget device. */
2358 	ret = usb_add_gadget(&pdev->gadget);
2359 	if (ret < 0) {
2360 		dev_err(pdev->dev, "Failed to add gadget\n");
2361 		goto free_ep0_buf;
2362 	}
2363 
2364 	return 0;
2365 
2366 free_ep0_buf:
2367 	dma_free_coherent(pdev->dev, 8, pdev->ep0_preq.request.buf,
2368 			  pdev->ep0_preq.request.dma);
2369 free_zlp_buf:
2370 	kfree(pdev->zlp_buf);
2371 destroy_dma_pool:
2372 	dma_pool_destroy(pdev->eps_dma_pool);
2373 put_gadget:
2374 	usb_put_gadget(&pdev->gadget);
2375 
2376 	return ret;
2377 }
2378 
2379 int cdns2_gadget_suspend(struct cdns2_device *pdev)
2380 {
2381 	unsigned long flags;
2382 
2383 	cdns2_disconnect_gadget(pdev);
2384 
2385 	spin_lock_irqsave(&pdev->lock, flags);
2386 	pdev->gadget.speed = USB_SPEED_UNKNOWN;
2387 
2388 	trace_cdns2_device_state("notattached");
2389 	usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
2390 	cdns2_enable_l1(pdev, 0);
2391 
2392 	/* Disable interrupt for device. */
2393 	writeb(0, &pdev->interrupt_regs->usbien);
2394 	writel(0, &pdev->adma_regs->ep_ien);
2395 	spin_unlock_irqrestore(&pdev->lock, flags);
2396 
2397 	return 0;
2398 }
2399 
2400 int cdns2_gadget_resume(struct cdns2_device *pdev, bool hibernated)
2401 {
2402 	unsigned long flags;
2403 
2404 	spin_lock_irqsave(&pdev->lock, flags);
2405 
2406 	if (!pdev->gadget_driver) {
2407 		spin_unlock_irqrestore(&pdev->lock, flags);
2408 		return 0;
2409 	}
2410 
2411 	cdns2_gadget_config(pdev);
2412 
2413 	if (hibernated)
2414 		clear_reg_bit_8(&pdev->usb_regs->usbcs, USBCS_DISCON);
2415 
2416 	spin_unlock_irqrestore(&pdev->lock, flags);
2417 
2418 	return 0;
2419 }
2420 
2421 void cdns2_gadget_remove(struct cdns2_device *pdev)
2422 {
2423 	pm_runtime_mark_last_busy(pdev->dev);
2424 	pm_runtime_put_autosuspend(pdev->dev);
2425 
2426 	usb_del_gadget(&pdev->gadget);
2427 	cdns2_free_all_eps(pdev);
2428 
2429 	dma_pool_destroy(pdev->eps_dma_pool);
2430 	kfree(pdev->zlp_buf);
2431 	usb_put_gadget(&pdev->gadget);
2432 }
2433 
2434 int cdns2_gadget_init(struct cdns2_device *pdev)
2435 {
2436 	int ret;
2437 
2438 	/* Ensure 32-bit DMA Mask. */
2439 	ret = dma_set_mask_and_coherent(pdev->dev, DMA_BIT_MASK(32));
2440 	if (ret) {
2441 		dev_err(pdev->dev, "Failed to set dma mask: %d\n", ret);
2442 		return ret;
2443 	}
2444 
2445 	pm_runtime_get_sync(pdev->dev);
2446 
2447 	cdsn2_isoc_burst_opt(pdev);
2448 
2449 	ret = cdns2_gadget_start(pdev);
2450 	if (ret) {
2451 		pm_runtime_put_sync(pdev->dev);
2452 		return ret;
2453 	}
2454 
2455 	/*
2456 	 * Because interrupt line can be shared with other components in
2457 	 * driver it can't use IRQF_ONESHOT flag here.
2458 	 */
2459 	ret = devm_request_threaded_irq(pdev->dev, pdev->irq,
2460 					cdns2_usb_irq_handler,
2461 					cdns2_thread_irq_handler,
2462 					IRQF_SHARED,
2463 					dev_name(pdev->dev),
2464 					pdev);
2465 	if (ret)
2466 		goto err0;
2467 
2468 	return 0;
2469 
2470 err0:
2471 	cdns2_gadget_remove(pdev);
2472 
2473 	return ret;
2474 }
2475