1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Renesas USB3.0 Peripheral driver (USB gadget)
4  *
5  * Copyright (C) 2015-2017  Renesas Electronics Corporation
6  */
7 
8 #include <linux/debugfs.h>
9 #include <linux/delay.h>
10 #include <linux/device.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/err.h>
13 #include <linux/extcon-provider.h>
14 #include <linux/interrupt.h>
15 #include <linux/io.h>
16 #include <linux/module.h>
17 #include <linux/of_device.h>
18 #include <linux/phy/phy.h>
19 #include <linux/platform_device.h>
20 #include <linux/pm_runtime.h>
21 #include <linux/reset.h>
22 #include <linux/sizes.h>
23 #include <linux/slab.h>
24 #include <linux/string.h>
25 #include <linux/uaccess.h>
26 #include <linux/usb/ch9.h>
27 #include <linux/usb/gadget.h>
28 #include <linux/usb/of.h>
29 #include <linux/usb/role.h>
30 #include <linux/usb/rzv2m_usb3drd.h>
31 
32 /* register definitions */
33 #define USB3_AXI_INT_STA	0x008
34 #define USB3_AXI_INT_ENA	0x00c
35 #define USB3_DMA_INT_STA	0x010
36 #define USB3_DMA_INT_ENA	0x014
37 #define USB3_DMA_CH0_CON(n)	(0x030 + ((n) - 1) * 0x10) /* n = 1 to 4 */
38 #define USB3_DMA_CH0_PRD_ADR(n)	(0x034 + ((n) - 1) * 0x10) /* n = 1 to 4 */
39 #define USB3_USB_COM_CON	0x200
40 #define USB3_USB20_CON		0x204
41 #define USB3_USB30_CON		0x208
42 #define USB3_USB_STA		0x210
43 #define USB3_DRD_CON(p)		((p)->is_rzv2m ? 0x400 : 0x218)
44 #define USB3_USB_INT_STA_1	0x220
45 #define USB3_USB_INT_STA_2	0x224
46 #define USB3_USB_INT_ENA_1	0x228
47 #define USB3_USB_INT_ENA_2	0x22c
48 #define USB3_STUP_DAT_0		0x230
49 #define USB3_STUP_DAT_1		0x234
50 #define USB3_USB_OTG_STA(p)	((p)->is_rzv2m ? 0x410 : 0x268)
51 #define USB3_USB_OTG_INT_STA(p)	((p)->is_rzv2m ? 0x414 : 0x26c)
52 #define USB3_USB_OTG_INT_ENA(p)	((p)->is_rzv2m ? 0x418 : 0x270)
53 #define USB3_P0_MOD		0x280
54 #define USB3_P0_CON		0x288
55 #define USB3_P0_STA		0x28c
56 #define USB3_P0_INT_STA		0x290
57 #define USB3_P0_INT_ENA		0x294
58 #define USB3_P0_LNG		0x2a0
59 #define USB3_P0_READ		0x2a4
60 #define USB3_P0_WRITE		0x2a8
61 #define USB3_PIPE_COM		0x2b0
62 #define USB3_PN_MOD		0x2c0
63 #define USB3_PN_RAMMAP		0x2c4
64 #define USB3_PN_CON		0x2c8
65 #define USB3_PN_STA		0x2cc
66 #define USB3_PN_INT_STA		0x2d0
67 #define USB3_PN_INT_ENA		0x2d4
68 #define USB3_PN_LNG		0x2e0
69 #define USB3_PN_READ		0x2e4
70 #define USB3_PN_WRITE		0x2e8
71 #define USB3_SSIFCMD		0x340
72 
73 /* AXI_INT_ENA and AXI_INT_STA */
74 #define AXI_INT_DMAINT		BIT(31)
75 #define AXI_INT_EPCINT		BIT(30)
76 /* PRD's n = from 1 to 4 */
77 #define AXI_INT_PRDEN_CLR_STA_SHIFT(n)	(16 + (n) - 1)
78 #define AXI_INT_PRDERR_STA_SHIFT(n)	(0 + (n) - 1)
79 #define AXI_INT_PRDEN_CLR_STA(n)	(1 << AXI_INT_PRDEN_CLR_STA_SHIFT(n))
80 #define AXI_INT_PRDERR_STA(n)		(1 << AXI_INT_PRDERR_STA_SHIFT(n))
81 
82 /* DMA_INT_ENA and DMA_INT_STA */
83 #define DMA_INT(n)		BIT(n)
84 
85 /* DMA_CH0_CONn */
86 #define DMA_CON_PIPE_DIR	BIT(15)		/* 1: In Transfer */
87 #define DMA_CON_PIPE_NO_SHIFT	8
88 #define DMA_CON_PIPE_NO_MASK	GENMASK(12, DMA_CON_PIPE_NO_SHIFT)
89 #define DMA_COM_PIPE_NO(n)	(((n) << DMA_CON_PIPE_NO_SHIFT) & \
90 					 DMA_CON_PIPE_NO_MASK)
91 #define DMA_CON_PRD_EN		BIT(0)
92 
93 /* LCLKSEL */
94 #define LCLKSEL_LSEL		BIT(18)
95 
96 /* USB_COM_CON */
97 #define USB_COM_CON_CONF		BIT(24)
98 #define USB_COM_CON_PN_WDATAIF_NL	BIT(23)
99 #define USB_COM_CON_PN_RDATAIF_NL	BIT(22)
100 #define USB_COM_CON_PN_LSTTR_PP		BIT(21)
101 #define USB_COM_CON_SPD_MODE		BIT(17)
102 #define USB_COM_CON_EP0_EN		BIT(16)
103 #define USB_COM_CON_DEV_ADDR_SHIFT	8
104 #define USB_COM_CON_DEV_ADDR_MASK	GENMASK(14, USB_COM_CON_DEV_ADDR_SHIFT)
105 #define USB_COM_CON_DEV_ADDR(n)		(((n) << USB_COM_CON_DEV_ADDR_SHIFT) & \
106 					 USB_COM_CON_DEV_ADDR_MASK)
107 #define USB_COM_CON_RX_DETECTION	BIT(1)
108 #define USB_COM_CON_PIPE_CLR		BIT(0)
109 
110 /* USB20_CON */
111 #define USB20_CON_B2_PUE		BIT(31)
112 #define USB20_CON_B2_SUSPEND		BIT(24)
113 #define USB20_CON_B2_CONNECT		BIT(17)
114 #define USB20_CON_B2_TSTMOD_SHIFT	8
115 #define USB20_CON_B2_TSTMOD_MASK	GENMASK(10, USB20_CON_B2_TSTMOD_SHIFT)
116 #define USB20_CON_B2_TSTMOD(n)		(((n) << USB20_CON_B2_TSTMOD_SHIFT) & \
117 					 USB20_CON_B2_TSTMOD_MASK)
118 #define USB20_CON_B2_TSTMOD_EN		BIT(0)
119 
120 /* USB30_CON */
121 #define USB30_CON_POW_SEL_SHIFT		24
122 #define USB30_CON_POW_SEL_MASK		GENMASK(26, USB30_CON_POW_SEL_SHIFT)
123 #define USB30_CON_POW_SEL_IN_U3		BIT(26)
124 #define USB30_CON_POW_SEL_IN_DISCON	0
125 #define USB30_CON_POW_SEL_P2_TO_P0	BIT(25)
126 #define USB30_CON_POW_SEL_P0_TO_P3	BIT(24)
127 #define USB30_CON_POW_SEL_P0_TO_P2	0
128 #define USB30_CON_B3_PLLWAKE		BIT(23)
129 #define USB30_CON_B3_CONNECT		BIT(17)
130 #define USB30_CON_B3_HOTRST_CMP		BIT(1)
131 
132 /* USB_STA */
133 #define USB_STA_SPEED_MASK	(BIT(2) | BIT(1))
134 #define USB_STA_SPEED_HS	BIT(2)
135 #define USB_STA_SPEED_FS	BIT(1)
136 #define USB_STA_SPEED_SS	0
137 #define USB_STA_VBUS_STA	BIT(0)
138 
139 /* DRD_CON */
140 #define DRD_CON_PERI_RST	BIT(31)		/* rzv2m only */
141 #define DRD_CON_HOST_RST	BIT(30)		/* rzv2m only */
142 #define DRD_CON_PERI_CON	BIT(24)
143 #define DRD_CON_VBOUT		BIT(0)
144 
145 /* USB_INT_ENA_1 and USB_INT_STA_1 */
146 #define USB_INT_1_B3_PLLWKUP	BIT(31)
147 #define USB_INT_1_B3_LUPSUCS	BIT(30)
148 #define USB_INT_1_B3_DISABLE	BIT(27)
149 #define USB_INT_1_B3_WRMRST	BIT(21)
150 #define USB_INT_1_B3_HOTRST	BIT(20)
151 #define USB_INT_1_B2_USBRST	BIT(12)
152 #define USB_INT_1_B2_L1SPND	BIT(11)
153 #define USB_INT_1_B2_SPND	BIT(9)
154 #define USB_INT_1_B2_RSUM	BIT(8)
155 #define USB_INT_1_SPEED		BIT(1)
156 #define USB_INT_1_VBUS_CNG	BIT(0)
157 
158 /* USB_INT_ENA_2 and USB_INT_STA_2 */
159 #define USB_INT_2_PIPE(n)	BIT(n)
160 
161 /* USB_OTG_STA, USB_OTG_INT_STA and USB_OTG_INT_ENA */
162 #define USB_OTG_IDMON(p)	((p)->is_rzv2m ? BIT(0) : BIT(4))
163 
164 /* P0_MOD */
165 #define P0_MOD_DIR		BIT(6)
166 
167 /* P0_CON and PN_CON */
168 #define PX_CON_BYTE_EN_MASK		(BIT(10) | BIT(9))
169 #define PX_CON_BYTE_EN_SHIFT		9
170 #define PX_CON_BYTE_EN_BYTES(n)		(((n) << PX_CON_BYTE_EN_SHIFT) & \
171 					 PX_CON_BYTE_EN_MASK)
172 #define PX_CON_SEND			BIT(8)
173 
174 /* P0_CON */
175 #define P0_CON_ST_RES_MASK		(BIT(27) | BIT(26))
176 #define P0_CON_ST_RES_FORCE_STALL	BIT(27)
177 #define P0_CON_ST_RES_NORMAL		BIT(26)
178 #define P0_CON_ST_RES_FORCE_NRDY	0
179 #define P0_CON_OT_RES_MASK		(BIT(25) | BIT(24))
180 #define P0_CON_OT_RES_FORCE_STALL	BIT(25)
181 #define P0_CON_OT_RES_NORMAL		BIT(24)
182 #define P0_CON_OT_RES_FORCE_NRDY	0
183 #define P0_CON_IN_RES_MASK		(BIT(17) | BIT(16))
184 #define P0_CON_IN_RES_FORCE_STALL	BIT(17)
185 #define P0_CON_IN_RES_NORMAL		BIT(16)
186 #define P0_CON_IN_RES_FORCE_NRDY	0
187 #define P0_CON_RES_WEN			BIT(7)
188 #define P0_CON_BCLR			BIT(1)
189 
190 /* P0_STA and PN_STA */
191 #define PX_STA_BUFSTS		BIT(0)
192 
193 /* P0_INT_ENA and P0_INT_STA */
194 #define P0_INT_STSED		BIT(18)
195 #define P0_INT_STSST		BIT(17)
196 #define P0_INT_SETUP		BIT(16)
197 #define P0_INT_RCVNL		BIT(8)
198 #define P0_INT_ERDY		BIT(7)
199 #define P0_INT_FLOW		BIT(6)
200 #define P0_INT_STALL		BIT(2)
201 #define P0_INT_NRDY		BIT(1)
202 #define P0_INT_BFRDY		BIT(0)
203 #define P0_INT_ALL_BITS		(P0_INT_STSED | P0_INT_SETUP | P0_INT_BFRDY)
204 
205 /* PN_MOD */
206 #define PN_MOD_DIR		BIT(6)
207 #define PN_MOD_TYPE_SHIFT	4
208 #define PN_MOD_TYPE_MASK	GENMASK(5, PN_MOD_TYPE_SHIFT)
209 #define PN_MOD_TYPE(n)		(((n) << PN_MOD_TYPE_SHIFT) & \
210 				 PN_MOD_TYPE_MASK)
211 #define PN_MOD_EPNUM_MASK	GENMASK(3, 0)
212 #define PN_MOD_EPNUM(n)		((n) & PN_MOD_EPNUM_MASK)
213 
214 /* PN_RAMMAP */
215 #define PN_RAMMAP_RAMAREA_SHIFT	29
216 #define PN_RAMMAP_RAMAREA_MASK	GENMASK(31, PN_RAMMAP_RAMAREA_SHIFT)
217 #define PN_RAMMAP_RAMAREA_16KB	BIT(31)
218 #define PN_RAMMAP_RAMAREA_8KB	(BIT(30) | BIT(29))
219 #define PN_RAMMAP_RAMAREA_4KB	BIT(30)
220 #define PN_RAMMAP_RAMAREA_2KB	BIT(29)
221 #define PN_RAMMAP_RAMAREA_1KB	0
222 #define PN_RAMMAP_MPKT_SHIFT	16
223 #define PN_RAMMAP_MPKT_MASK	GENMASK(26, PN_RAMMAP_MPKT_SHIFT)
224 #define PN_RAMMAP_MPKT(n)	(((n) << PN_RAMMAP_MPKT_SHIFT) & \
225 				 PN_RAMMAP_MPKT_MASK)
226 #define PN_RAMMAP_RAMIF_SHIFT	14
227 #define PN_RAMMAP_RAMIF_MASK	GENMASK(15, PN_RAMMAP_RAMIF_SHIFT)
228 #define PN_RAMMAP_RAMIF(n)	(((n) << PN_RAMMAP_RAMIF_SHIFT) & \
229 				 PN_RAMMAP_RAMIF_MASK)
230 #define PN_RAMMAP_BASEAD_MASK	GENMASK(13, 0)
231 #define PN_RAMMAP_BASEAD(offs)	(((offs) >> 3) & PN_RAMMAP_BASEAD_MASK)
232 #define PN_RAMMAP_DATA(area, ramif, basead)	((PN_RAMMAP_##area) | \
233 						 (PN_RAMMAP_RAMIF(ramif)) | \
234 						 (PN_RAMMAP_BASEAD(basead)))
235 
236 /* PN_CON */
237 #define PN_CON_EN		BIT(31)
238 #define PN_CON_DATAIF_EN	BIT(30)
239 #define PN_CON_RES_MASK		(BIT(17) | BIT(16))
240 #define PN_CON_RES_FORCE_STALL	BIT(17)
241 #define PN_CON_RES_NORMAL	BIT(16)
242 #define PN_CON_RES_FORCE_NRDY	0
243 #define PN_CON_LAST		BIT(11)
244 #define PN_CON_RES_WEN		BIT(7)
245 #define PN_CON_CLR		BIT(0)
246 
247 /* PN_INT_STA and PN_INT_ENA */
248 #define PN_INT_LSTTR	BIT(4)
249 #define PN_INT_BFRDY	BIT(0)
250 
251 /* USB3_SSIFCMD */
252 #define SSIFCMD_URES_U2		BIT(9)
253 #define SSIFCMD_URES_U1		BIT(8)
254 #define SSIFCMD_UDIR_U2		BIT(7)
255 #define SSIFCMD_UDIR_U1		BIT(6)
256 #define SSIFCMD_UREQ_U2		BIT(5)
257 #define SSIFCMD_UREQ_U1		BIT(4)
258 
259 #define USB3_EP0_SS_MAX_PACKET_SIZE	512
260 #define USB3_EP0_HSFS_MAX_PACKET_SIZE	64
261 #define USB3_EP0_BUF_SIZE		8
262 #define USB3_MAX_NUM_PIPES(p)		((p)->is_rzv2m ? 16 : 6)	/* This includes PIPE 0 */
263 #define USB3_WAIT_US			3
264 #define USB3_DMA_NUM_SETTING_AREA	4
265 /*
266  * To avoid double-meaning of "0" (xferred 65536 bytes or received zlp if
267  * buffer size is 65536), this driver uses the maximum size per a entry is
268  * 32768 bytes.
269  */
270 #define USB3_DMA_MAX_XFER_SIZE		32768
271 #define USB3_DMA_PRD_SIZE		4096
272 
273 struct renesas_usb3;
274 
275 /* Physical Region Descriptor Table */
276 struct renesas_usb3_prd {
277 	u32 word1;
278 #define USB3_PRD1_E		BIT(30)		/* the end of chain */
279 #define USB3_PRD1_U		BIT(29)		/* completion of transfer */
280 #define USB3_PRD1_D		BIT(28)		/* Error occurred */
281 #define USB3_PRD1_INT		BIT(27)		/* Interrupt occurred */
282 #define USB3_PRD1_LST		BIT(26)		/* Last Packet */
283 #define USB3_PRD1_B_INC		BIT(24)
284 #define USB3_PRD1_MPS_8		0
285 #define USB3_PRD1_MPS_16	BIT(21)
286 #define USB3_PRD1_MPS_32	BIT(22)
287 #define USB3_PRD1_MPS_64	(BIT(22) | BIT(21))
288 #define USB3_PRD1_MPS_512	BIT(23)
289 #define USB3_PRD1_MPS_1024	(BIT(23) | BIT(21))
290 #define USB3_PRD1_MPS_RESERVED	(BIT(23) | BIT(22) | BIT(21))
291 #define USB3_PRD1_SIZE_MASK	GENMASK(15, 0)
292 
293 	u32 bap;
294 };
295 #define USB3_DMA_NUM_PRD_ENTRIES	(USB3_DMA_PRD_SIZE / \
296 					  sizeof(struct renesas_usb3_prd))
297 #define USB3_DMA_MAX_XFER_SIZE_ALL_PRDS	(USB3_DMA_PRD_SIZE / \
298 					 sizeof(struct renesas_usb3_prd) * \
299 					 USB3_DMA_MAX_XFER_SIZE)
300 
301 struct renesas_usb3_dma {
302 	struct renesas_usb3_prd *prd;
303 	dma_addr_t prd_dma;
304 	int num;	/* Setting area number (from 1 to 4) */
305 	bool used;
306 };
307 
308 struct renesas_usb3_request {
309 	struct usb_request	req;
310 	struct list_head	queue;
311 };
312 
313 #define USB3_EP_NAME_SIZE	8
314 struct renesas_usb3_ep {
315 	struct usb_ep ep;
316 	struct renesas_usb3 *usb3;
317 	struct renesas_usb3_dma *dma;
318 	int num;
319 	char ep_name[USB3_EP_NAME_SIZE];
320 	struct list_head queue;
321 	u32 rammap_val;
322 	bool dir_in;
323 	bool halt;
324 	bool wedge;
325 	bool started;
326 };
327 
328 struct renesas_usb3_priv {
329 	int ramsize_per_ramif;		/* unit = bytes */
330 	int num_ramif;
331 	int ramsize_per_pipe;		/* unit = bytes */
332 	bool workaround_for_vbus;	/* if true, don't check vbus signal */
333 	bool is_rzv2m;			/* if true, RZ/V2M SoC */
334 };
335 
336 struct renesas_usb3 {
337 	void __iomem *reg;
338 	void __iomem *drd_reg;
339 	struct reset_control *usbp_rstc;
340 
341 	struct usb_gadget gadget;
342 	struct usb_gadget_driver *driver;
343 	struct extcon_dev *extcon;
344 	struct work_struct extcon_work;
345 	struct phy *phy;
346 	struct dentry *dentry;
347 
348 	struct usb_role_switch *role_sw;
349 	struct device *host_dev;
350 	struct work_struct role_work;
351 	enum usb_role role;
352 
353 	struct renesas_usb3_ep *usb3_ep;
354 	int num_usb3_eps;
355 
356 	struct renesas_usb3_dma dma[USB3_DMA_NUM_SETTING_AREA];
357 
358 	spinlock_t lock;
359 	int disabled_count;
360 
361 	struct usb_request *ep0_req;
362 
363 	enum usb_role connection_state;
364 	u16 test_mode;
365 	u8 ep0_buf[USB3_EP0_BUF_SIZE];
366 	bool softconnect;
367 	bool workaround_for_vbus;
368 	bool extcon_host;		/* check id and set EXTCON_USB_HOST */
369 	bool extcon_usb;		/* check vbus and set EXTCON_USB */
370 	bool forced_b_device;
371 	bool start_to_connect;
372 	bool role_sw_by_connector;
373 	bool is_rzv2m;
374 };
375 
376 #define gadget_to_renesas_usb3(_gadget)	\
377 		container_of(_gadget, struct renesas_usb3, gadget)
378 #define renesas_usb3_to_gadget(renesas_usb3) (&renesas_usb3->gadget)
379 #define usb3_to_dev(_usb3)	(_usb3->gadget.dev.parent)
380 
381 #define usb_ep_to_usb3_ep(_ep) container_of(_ep, struct renesas_usb3_ep, ep)
382 #define usb3_ep_to_usb3(_usb3_ep) (_usb3_ep->usb3)
383 #define usb_req_to_usb3_req(_req) container_of(_req, \
384 					    struct renesas_usb3_request, req)
385 
386 #define usb3_get_ep(usb3, n) ((usb3)->usb3_ep + (n))
387 #define usb3_for_each_ep(usb3_ep, usb3, i)			\
388 		for ((i) = 0, usb3_ep = usb3_get_ep(usb3, (i));	\
389 		     (i) < (usb3)->num_usb3_eps;		\
390 		     (i)++, usb3_ep = usb3_get_ep(usb3, (i)))
391 
392 #define usb3_get_dma(usb3, i)	(&(usb3)->dma[i])
393 #define usb3_for_each_dma(usb3, dma, i)				\
394 		for ((i) = 0, dma = usb3_get_dma((usb3), (i));	\
395 		     (i) < USB3_DMA_NUM_SETTING_AREA;		\
396 		     (i)++, dma = usb3_get_dma((usb3), (i)))
397 
398 static const char udc_name[] = "renesas_usb3";
399 
400 static bool use_dma = 1;
401 module_param(use_dma, bool, 0644);
402 MODULE_PARM_DESC(use_dma, "use dedicated DMAC");
403 
404 static void usb3_write(struct renesas_usb3 *usb3, u32 data, u32 offs)
405 {
406 	iowrite32(data, usb3->reg + offs);
407 }
408 
409 static u32 usb3_read(struct renesas_usb3 *usb3, u32 offs)
410 {
411 	return ioread32(usb3->reg + offs);
412 }
413 
414 static void usb3_set_bit(struct renesas_usb3 *usb3, u32 bits, u32 offs)
415 {
416 	u32 val = usb3_read(usb3, offs);
417 
418 	val |= bits;
419 	usb3_write(usb3, val, offs);
420 }
421 
422 static void usb3_clear_bit(struct renesas_usb3 *usb3, u32 bits, u32 offs)
423 {
424 	u32 val = usb3_read(usb3, offs);
425 
426 	val &= ~bits;
427 	usb3_write(usb3, val, offs);
428 }
429 
430 static void usb3_drd_write(struct renesas_usb3 *usb3, u32 data, u32 offs)
431 {
432 	void __iomem *reg;
433 
434 	if (usb3->is_rzv2m)
435 		reg = usb3->drd_reg + offs - USB3_DRD_CON(usb3);
436 	else
437 		reg = usb3->reg + offs;
438 
439 	iowrite32(data, reg);
440 }
441 
442 static u32 usb3_drd_read(struct renesas_usb3 *usb3, u32 offs)
443 {
444 	void __iomem *reg;
445 
446 	if (usb3->is_rzv2m)
447 		reg = usb3->drd_reg + offs - USB3_DRD_CON(usb3);
448 	else
449 		reg = usb3->reg + offs;
450 
451 	return ioread32(reg);
452 }
453 
454 static void usb3_drd_set_bit(struct renesas_usb3 *usb3, u32 bits, u32 offs)
455 {
456 	u32 val = usb3_drd_read(usb3, offs);
457 
458 	val |= bits;
459 	usb3_drd_write(usb3, val, offs);
460 }
461 
462 static void usb3_drd_clear_bit(struct renesas_usb3 *usb3, u32 bits, u32 offs)
463 {
464 	u32 val = usb3_drd_read(usb3, offs);
465 
466 	val &= ~bits;
467 	usb3_drd_write(usb3, val, offs);
468 }
469 
470 static int usb3_wait(struct renesas_usb3 *usb3, u32 reg, u32 mask,
471 		     u32 expected)
472 {
473 	int i;
474 
475 	for (i = 0; i < USB3_WAIT_US; i++) {
476 		if ((usb3_read(usb3, reg) & mask) == expected)
477 			return 0;
478 		udelay(1);
479 	}
480 
481 	dev_dbg(usb3_to_dev(usb3), "%s: timed out (%8x, %08x, %08x)\n",
482 		__func__, reg, mask, expected);
483 
484 	return -EBUSY;
485 }
486 
487 static void renesas_usb3_extcon_work(struct work_struct *work)
488 {
489 	struct renesas_usb3 *usb3 = container_of(work, struct renesas_usb3,
490 						 extcon_work);
491 
492 	extcon_set_state_sync(usb3->extcon, EXTCON_USB_HOST, usb3->extcon_host);
493 	extcon_set_state_sync(usb3->extcon, EXTCON_USB, usb3->extcon_usb);
494 }
495 
496 static void usb3_enable_irq_1(struct renesas_usb3 *usb3, u32 bits)
497 {
498 	usb3_set_bit(usb3, bits, USB3_USB_INT_ENA_1);
499 }
500 
501 static void usb3_disable_irq_1(struct renesas_usb3 *usb3, u32 bits)
502 {
503 	usb3_clear_bit(usb3, bits, USB3_USB_INT_ENA_1);
504 }
505 
506 static void usb3_enable_pipe_irq(struct renesas_usb3 *usb3, int num)
507 {
508 	usb3_set_bit(usb3, USB_INT_2_PIPE(num), USB3_USB_INT_ENA_2);
509 }
510 
511 static void usb3_disable_pipe_irq(struct renesas_usb3 *usb3, int num)
512 {
513 	usb3_clear_bit(usb3, USB_INT_2_PIPE(num), USB3_USB_INT_ENA_2);
514 }
515 
516 static bool usb3_is_host(struct renesas_usb3 *usb3)
517 {
518 	return !(usb3_drd_read(usb3, USB3_DRD_CON(usb3)) & DRD_CON_PERI_CON);
519 }
520 
521 static void usb3_init_axi_bridge(struct renesas_usb3 *usb3)
522 {
523 	/* Set AXI_INT */
524 	usb3_write(usb3, ~0, USB3_DMA_INT_STA);
525 	usb3_write(usb3, 0, USB3_DMA_INT_ENA);
526 	usb3_set_bit(usb3, AXI_INT_DMAINT | AXI_INT_EPCINT, USB3_AXI_INT_ENA);
527 }
528 
529 static void usb3_init_epc_registers(struct renesas_usb3 *usb3)
530 {
531 	usb3_write(usb3, ~0, USB3_USB_INT_STA_1);
532 	if (!usb3->workaround_for_vbus)
533 		usb3_enable_irq_1(usb3, USB_INT_1_VBUS_CNG);
534 }
535 
536 static bool usb3_wakeup_usb2_phy(struct renesas_usb3 *usb3)
537 {
538 	if (!(usb3_read(usb3, USB3_USB20_CON) & USB20_CON_B2_SUSPEND))
539 		return true;	/* already waked it up */
540 
541 	usb3_clear_bit(usb3, USB20_CON_B2_SUSPEND, USB3_USB20_CON);
542 	usb3_enable_irq_1(usb3, USB_INT_1_B2_RSUM);
543 
544 	return false;
545 }
546 
547 static void usb3_usb2_pullup(struct renesas_usb3 *usb3, int pullup)
548 {
549 	u32 bits = USB20_CON_B2_PUE | USB20_CON_B2_CONNECT;
550 
551 	if (usb3->softconnect && pullup)
552 		usb3_set_bit(usb3, bits, USB3_USB20_CON);
553 	else
554 		usb3_clear_bit(usb3, bits, USB3_USB20_CON);
555 }
556 
557 static void usb3_set_test_mode(struct renesas_usb3 *usb3)
558 {
559 	u32 val = usb3_read(usb3, USB3_USB20_CON);
560 
561 	val &= ~USB20_CON_B2_TSTMOD_MASK;
562 	val |= USB20_CON_B2_TSTMOD(usb3->test_mode);
563 	usb3_write(usb3, val | USB20_CON_B2_TSTMOD_EN, USB3_USB20_CON);
564 	if (!usb3->test_mode)
565 		usb3_clear_bit(usb3, USB20_CON_B2_TSTMOD_EN, USB3_USB20_CON);
566 }
567 
568 static void usb3_start_usb2_connection(struct renesas_usb3 *usb3)
569 {
570 	usb3->disabled_count++;
571 	usb3_set_bit(usb3, USB_COM_CON_EP0_EN, USB3_USB_COM_CON);
572 	usb3_set_bit(usb3, USB_COM_CON_SPD_MODE, USB3_USB_COM_CON);
573 	usb3_usb2_pullup(usb3, 1);
574 }
575 
576 static int usb3_is_usb3_phy_in_u3(struct renesas_usb3 *usb3)
577 {
578 	return usb3_read(usb3, USB3_USB30_CON) & USB30_CON_POW_SEL_IN_U3;
579 }
580 
581 static bool usb3_wakeup_usb3_phy(struct renesas_usb3 *usb3)
582 {
583 	if (!usb3_is_usb3_phy_in_u3(usb3))
584 		return true;	/* already waked it up */
585 
586 	usb3_set_bit(usb3, USB30_CON_B3_PLLWAKE, USB3_USB30_CON);
587 	usb3_enable_irq_1(usb3, USB_INT_1_B3_PLLWKUP);
588 
589 	return false;
590 }
591 
592 static u16 usb3_feature_get_un_enabled(struct renesas_usb3 *usb3)
593 {
594 	u32 mask_u2 = SSIFCMD_UDIR_U2 | SSIFCMD_UREQ_U2;
595 	u32 mask_u1 = SSIFCMD_UDIR_U1 | SSIFCMD_UREQ_U1;
596 	u32 val = usb3_read(usb3, USB3_SSIFCMD);
597 	u16 ret = 0;
598 
599 	/* Enables {U2,U1} if the bits of UDIR and UREQ are set to 0 */
600 	if (!(val & mask_u2))
601 		ret |= 1 << USB_DEV_STAT_U2_ENABLED;
602 	if (!(val & mask_u1))
603 		ret |= 1 << USB_DEV_STAT_U1_ENABLED;
604 
605 	return ret;
606 }
607 
608 static void usb3_feature_u2_enable(struct renesas_usb3 *usb3, bool enable)
609 {
610 	u32 bits = SSIFCMD_UDIR_U2 | SSIFCMD_UREQ_U2;
611 
612 	/* Enables U2 if the bits of UDIR and UREQ are set to 0 */
613 	if (enable)
614 		usb3_clear_bit(usb3, bits, USB3_SSIFCMD);
615 	else
616 		usb3_set_bit(usb3, bits, USB3_SSIFCMD);
617 }
618 
619 static void usb3_feature_u1_enable(struct renesas_usb3 *usb3, bool enable)
620 {
621 	u32 bits = SSIFCMD_UDIR_U1 | SSIFCMD_UREQ_U1;
622 
623 	/* Enables U1 if the bits of UDIR and UREQ are set to 0 */
624 	if (enable)
625 		usb3_clear_bit(usb3, bits, USB3_SSIFCMD);
626 	else
627 		usb3_set_bit(usb3, bits, USB3_SSIFCMD);
628 }
629 
630 static void usb3_start_operation_for_usb3(struct renesas_usb3 *usb3)
631 {
632 	usb3_set_bit(usb3, USB_COM_CON_EP0_EN, USB3_USB_COM_CON);
633 	usb3_clear_bit(usb3, USB_COM_CON_SPD_MODE, USB3_USB_COM_CON);
634 	usb3_set_bit(usb3, USB30_CON_B3_CONNECT, USB3_USB30_CON);
635 }
636 
637 static void usb3_start_usb3_connection(struct renesas_usb3 *usb3)
638 {
639 	usb3_start_operation_for_usb3(usb3);
640 	usb3_set_bit(usb3, USB_COM_CON_RX_DETECTION, USB3_USB_COM_CON);
641 
642 	usb3_enable_irq_1(usb3, USB_INT_1_B3_LUPSUCS | USB_INT_1_B3_DISABLE |
643 			  USB_INT_1_SPEED);
644 }
645 
646 static void usb3_stop_usb3_connection(struct renesas_usb3 *usb3)
647 {
648 	usb3_clear_bit(usb3, USB30_CON_B3_CONNECT, USB3_USB30_CON);
649 }
650 
651 static void usb3_transition_to_default_state(struct renesas_usb3 *usb3,
652 					     bool is_usb3)
653 {
654 	usb3_set_bit(usb3, USB_INT_2_PIPE(0), USB3_USB_INT_ENA_2);
655 	usb3_write(usb3, P0_INT_ALL_BITS, USB3_P0_INT_STA);
656 	usb3_set_bit(usb3, P0_INT_ALL_BITS, USB3_P0_INT_ENA);
657 
658 	if (is_usb3)
659 		usb3_enable_irq_1(usb3, USB_INT_1_B3_WRMRST |
660 				  USB_INT_1_B3_HOTRST);
661 	else
662 		usb3_enable_irq_1(usb3, USB_INT_1_B2_SPND |
663 				  USB_INT_1_B2_L1SPND | USB_INT_1_B2_USBRST);
664 }
665 
666 static void usb3_connect(struct renesas_usb3 *usb3)
667 {
668 	if (usb3_wakeup_usb3_phy(usb3))
669 		usb3_start_usb3_connection(usb3);
670 }
671 
672 static void usb3_reset_epc(struct renesas_usb3 *usb3)
673 {
674 	usb3_clear_bit(usb3, USB_COM_CON_CONF, USB3_USB_COM_CON);
675 	usb3_clear_bit(usb3, USB_COM_CON_EP0_EN, USB3_USB_COM_CON);
676 	usb3_set_bit(usb3, USB_COM_CON_PIPE_CLR, USB3_USB_COM_CON);
677 	usb3->test_mode = 0;
678 	usb3_set_test_mode(usb3);
679 }
680 
681 static void usb3_disconnect(struct renesas_usb3 *usb3)
682 {
683 	usb3->disabled_count = 0;
684 	usb3_usb2_pullup(usb3, 0);
685 	usb3_clear_bit(usb3, USB30_CON_B3_CONNECT, USB3_USB30_CON);
686 	usb3_reset_epc(usb3);
687 	usb3_disable_irq_1(usb3, USB_INT_1_B2_RSUM | USB_INT_1_B3_PLLWKUP |
688 			   USB_INT_1_B3_LUPSUCS | USB_INT_1_B3_DISABLE |
689 			   USB_INT_1_SPEED | USB_INT_1_B3_WRMRST |
690 			   USB_INT_1_B3_HOTRST | USB_INT_1_B2_SPND |
691 			   USB_INT_1_B2_L1SPND | USB_INT_1_B2_USBRST);
692 	usb3_clear_bit(usb3, USB_COM_CON_SPD_MODE, USB3_USB_COM_CON);
693 	usb3_init_epc_registers(usb3);
694 
695 	if (usb3->driver)
696 		usb3->driver->disconnect(&usb3->gadget);
697 }
698 
699 static void usb3_check_vbus(struct renesas_usb3 *usb3)
700 {
701 	if (usb3->workaround_for_vbus) {
702 		usb3_connect(usb3);
703 	} else {
704 		usb3->extcon_usb = !!(usb3_read(usb3, USB3_USB_STA) &
705 							USB_STA_VBUS_STA);
706 		if (usb3->extcon_usb)
707 			usb3_connect(usb3);
708 		else
709 			usb3_disconnect(usb3);
710 
711 		schedule_work(&usb3->extcon_work);
712 	}
713 }
714 
715 static void renesas_usb3_role_work(struct work_struct *work)
716 {
717 	struct renesas_usb3 *usb3 =
718 			container_of(work, struct renesas_usb3, role_work);
719 
720 	usb_role_switch_set_role(usb3->role_sw, usb3->role);
721 }
722 
723 static void usb3_set_mode(struct renesas_usb3 *usb3, bool host)
724 {
725 	if (usb3->is_rzv2m) {
726 		if (host) {
727 			usb3_drd_set_bit(usb3, DRD_CON_PERI_RST, USB3_DRD_CON(usb3));
728 			usb3_drd_clear_bit(usb3, DRD_CON_HOST_RST, USB3_DRD_CON(usb3));
729 		} else {
730 			usb3_drd_set_bit(usb3, DRD_CON_HOST_RST, USB3_DRD_CON(usb3));
731 			usb3_drd_clear_bit(usb3, DRD_CON_PERI_RST, USB3_DRD_CON(usb3));
732 		}
733 	}
734 
735 	if (host)
736 		usb3_drd_clear_bit(usb3, DRD_CON_PERI_CON, USB3_DRD_CON(usb3));
737 	else
738 		usb3_drd_set_bit(usb3, DRD_CON_PERI_CON, USB3_DRD_CON(usb3));
739 }
740 
741 static void usb3_set_mode_by_role_sw(struct renesas_usb3 *usb3, bool host)
742 {
743 	if (usb3->role_sw) {
744 		usb3->role = host ? USB_ROLE_HOST : USB_ROLE_DEVICE;
745 		schedule_work(&usb3->role_work);
746 	} else {
747 		usb3_set_mode(usb3, host);
748 	}
749 }
750 
751 static void usb3_vbus_out(struct renesas_usb3 *usb3, bool enable)
752 {
753 	if (enable)
754 		usb3_drd_set_bit(usb3, DRD_CON_VBOUT, USB3_DRD_CON(usb3));
755 	else
756 		usb3_drd_clear_bit(usb3, DRD_CON_VBOUT, USB3_DRD_CON(usb3));
757 }
758 
759 static void usb3_mode_config(struct renesas_usb3 *usb3, bool host, bool a_dev)
760 {
761 	unsigned long flags;
762 
763 	spin_lock_irqsave(&usb3->lock, flags);
764 	if (!usb3->role_sw_by_connector ||
765 	    usb3->connection_state != USB_ROLE_NONE) {
766 		usb3_set_mode_by_role_sw(usb3, host);
767 		usb3_vbus_out(usb3, a_dev);
768 	}
769 	/* for A-Peripheral or forced B-device mode */
770 	if ((!host && a_dev) || usb3->start_to_connect)
771 		usb3_connect(usb3);
772 	spin_unlock_irqrestore(&usb3->lock, flags);
773 }
774 
775 static bool usb3_is_a_device(struct renesas_usb3 *usb3)
776 {
777 	return !(usb3_drd_read(usb3, USB3_USB_OTG_STA(usb3)) & USB_OTG_IDMON(usb3));
778 }
779 
780 static void usb3_check_id(struct renesas_usb3 *usb3)
781 {
782 	usb3->extcon_host = usb3_is_a_device(usb3);
783 
784 	if ((!usb3->role_sw_by_connector && usb3->extcon_host &&
785 	     !usb3->forced_b_device) || usb3->connection_state == USB_ROLE_HOST)
786 		usb3_mode_config(usb3, true, true);
787 	else
788 		usb3_mode_config(usb3, false, false);
789 
790 	schedule_work(&usb3->extcon_work);
791 }
792 
793 static void renesas_usb3_init_controller(struct renesas_usb3 *usb3)
794 {
795 	usb3_init_axi_bridge(usb3);
796 	usb3_init_epc_registers(usb3);
797 	usb3_set_bit(usb3, USB_COM_CON_PN_WDATAIF_NL |
798 		     USB_COM_CON_PN_RDATAIF_NL | USB_COM_CON_PN_LSTTR_PP,
799 		     USB3_USB_COM_CON);
800 	usb3_drd_write(usb3, USB_OTG_IDMON(usb3), USB3_USB_OTG_INT_STA(usb3));
801 	usb3_drd_write(usb3, USB_OTG_IDMON(usb3), USB3_USB_OTG_INT_ENA(usb3));
802 
803 	usb3_check_id(usb3);
804 	usb3_check_vbus(usb3);
805 }
806 
807 static void renesas_usb3_stop_controller(struct renesas_usb3 *usb3)
808 {
809 	usb3_disconnect(usb3);
810 	usb3_write(usb3, 0, USB3_P0_INT_ENA);
811 	usb3_drd_write(usb3, 0, USB3_USB_OTG_INT_ENA(usb3));
812 	usb3_write(usb3, 0, USB3_USB_INT_ENA_1);
813 	usb3_write(usb3, 0, USB3_USB_INT_ENA_2);
814 	usb3_write(usb3, 0, USB3_AXI_INT_ENA);
815 }
816 
817 static void usb3_irq_epc_int_1_pll_wakeup(struct renesas_usb3 *usb3)
818 {
819 	usb3_disable_irq_1(usb3, USB_INT_1_B3_PLLWKUP);
820 	usb3_clear_bit(usb3, USB30_CON_B3_PLLWAKE, USB3_USB30_CON);
821 	usb3_start_usb3_connection(usb3);
822 }
823 
824 static void usb3_irq_epc_int_1_linkup_success(struct renesas_usb3 *usb3)
825 {
826 	usb3_transition_to_default_state(usb3, true);
827 }
828 
829 static void usb3_irq_epc_int_1_resume(struct renesas_usb3 *usb3)
830 {
831 	usb3_disable_irq_1(usb3, USB_INT_1_B2_RSUM);
832 	usb3_start_usb2_connection(usb3);
833 	usb3_transition_to_default_state(usb3, false);
834 }
835 
836 static void usb3_irq_epc_int_1_suspend(struct renesas_usb3 *usb3)
837 {
838 	usb3_disable_irq_1(usb3, USB_INT_1_B2_SPND);
839 
840 	if (usb3->gadget.speed != USB_SPEED_UNKNOWN &&
841 	    usb3->gadget.state != USB_STATE_NOTATTACHED) {
842 		if (usb3->driver && usb3->driver->suspend)
843 			usb3->driver->suspend(&usb3->gadget);
844 		usb_gadget_set_state(&usb3->gadget, USB_STATE_SUSPENDED);
845 	}
846 }
847 
848 static void usb3_irq_epc_int_1_disable(struct renesas_usb3 *usb3)
849 {
850 	usb3_stop_usb3_connection(usb3);
851 	if (usb3_wakeup_usb2_phy(usb3))
852 		usb3_irq_epc_int_1_resume(usb3);
853 }
854 
855 static void usb3_irq_epc_int_1_bus_reset(struct renesas_usb3 *usb3)
856 {
857 	usb3_reset_epc(usb3);
858 	if (usb3->disabled_count < 3)
859 		usb3_start_usb3_connection(usb3);
860 	else
861 		usb3_start_usb2_connection(usb3);
862 }
863 
864 static void usb3_irq_epc_int_1_vbus_change(struct renesas_usb3 *usb3)
865 {
866 	usb3_check_vbus(usb3);
867 }
868 
869 static void usb3_irq_epc_int_1_hot_reset(struct renesas_usb3 *usb3)
870 {
871 	usb3_reset_epc(usb3);
872 	usb3_set_bit(usb3, USB_COM_CON_EP0_EN, USB3_USB_COM_CON);
873 
874 	/* This bit shall be set within 12ms from the start of HotReset */
875 	usb3_set_bit(usb3, USB30_CON_B3_HOTRST_CMP, USB3_USB30_CON);
876 }
877 
878 static void usb3_irq_epc_int_1_warm_reset(struct renesas_usb3 *usb3)
879 {
880 	usb3_reset_epc(usb3);
881 	usb3_set_bit(usb3, USB_COM_CON_EP0_EN, USB3_USB_COM_CON);
882 
883 	usb3_start_operation_for_usb3(usb3);
884 	usb3_enable_irq_1(usb3, USB_INT_1_SPEED);
885 }
886 
887 static void usb3_irq_epc_int_1_speed(struct renesas_usb3 *usb3)
888 {
889 	u32 speed = usb3_read(usb3, USB3_USB_STA) & USB_STA_SPEED_MASK;
890 
891 	switch (speed) {
892 	case USB_STA_SPEED_SS:
893 		usb3->gadget.speed = USB_SPEED_SUPER;
894 		usb3->gadget.ep0->maxpacket = USB3_EP0_SS_MAX_PACKET_SIZE;
895 		break;
896 	case USB_STA_SPEED_HS:
897 		usb3->gadget.speed = USB_SPEED_HIGH;
898 		usb3->gadget.ep0->maxpacket = USB3_EP0_HSFS_MAX_PACKET_SIZE;
899 		break;
900 	case USB_STA_SPEED_FS:
901 		usb3->gadget.speed = USB_SPEED_FULL;
902 		usb3->gadget.ep0->maxpacket = USB3_EP0_HSFS_MAX_PACKET_SIZE;
903 		break;
904 	default:
905 		usb3->gadget.speed = USB_SPEED_UNKNOWN;
906 		break;
907 	}
908 }
909 
910 static void usb3_irq_epc_int_1(struct renesas_usb3 *usb3, u32 int_sta_1)
911 {
912 	if (int_sta_1 & USB_INT_1_B3_PLLWKUP)
913 		usb3_irq_epc_int_1_pll_wakeup(usb3);
914 
915 	if (int_sta_1 & USB_INT_1_B3_LUPSUCS)
916 		usb3_irq_epc_int_1_linkup_success(usb3);
917 
918 	if (int_sta_1 & USB_INT_1_B3_HOTRST)
919 		usb3_irq_epc_int_1_hot_reset(usb3);
920 
921 	if (int_sta_1 & USB_INT_1_B3_WRMRST)
922 		usb3_irq_epc_int_1_warm_reset(usb3);
923 
924 	if (int_sta_1 & USB_INT_1_B3_DISABLE)
925 		usb3_irq_epc_int_1_disable(usb3);
926 
927 	if (int_sta_1 & USB_INT_1_B2_USBRST)
928 		usb3_irq_epc_int_1_bus_reset(usb3);
929 
930 	if (int_sta_1 & USB_INT_1_B2_RSUM)
931 		usb3_irq_epc_int_1_resume(usb3);
932 
933 	if (int_sta_1 & USB_INT_1_B2_SPND)
934 		usb3_irq_epc_int_1_suspend(usb3);
935 
936 	if (int_sta_1 & USB_INT_1_SPEED)
937 		usb3_irq_epc_int_1_speed(usb3);
938 
939 	if (int_sta_1 & USB_INT_1_VBUS_CNG)
940 		usb3_irq_epc_int_1_vbus_change(usb3);
941 }
942 
943 static struct renesas_usb3_request *__usb3_get_request(struct renesas_usb3_ep
944 						       *usb3_ep)
945 {
946 	return list_first_entry_or_null(&usb3_ep->queue,
947 					struct renesas_usb3_request, queue);
948 }
949 
950 static struct renesas_usb3_request *usb3_get_request(struct renesas_usb3_ep
951 						     *usb3_ep)
952 {
953 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
954 	struct renesas_usb3_request *usb3_req;
955 	unsigned long flags;
956 
957 	spin_lock_irqsave(&usb3->lock, flags);
958 	usb3_req = __usb3_get_request(usb3_ep);
959 	spin_unlock_irqrestore(&usb3->lock, flags);
960 
961 	return usb3_req;
962 }
963 
964 static void __usb3_request_done(struct renesas_usb3_ep *usb3_ep,
965 				struct renesas_usb3_request *usb3_req,
966 				int status)
967 {
968 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
969 
970 	dev_dbg(usb3_to_dev(usb3), "giveback: ep%2d, %u, %u, %d\n",
971 		usb3_ep->num, usb3_req->req.length, usb3_req->req.actual,
972 		status);
973 	usb3_req->req.status = status;
974 	usb3_ep->started = false;
975 	list_del_init(&usb3_req->queue);
976 	spin_unlock(&usb3->lock);
977 	usb_gadget_giveback_request(&usb3_ep->ep, &usb3_req->req);
978 	spin_lock(&usb3->lock);
979 }
980 
981 static void usb3_request_done(struct renesas_usb3_ep *usb3_ep,
982 			      struct renesas_usb3_request *usb3_req, int status)
983 {
984 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
985 	unsigned long flags;
986 
987 	spin_lock_irqsave(&usb3->lock, flags);
988 	__usb3_request_done(usb3_ep, usb3_req, status);
989 	spin_unlock_irqrestore(&usb3->lock, flags);
990 }
991 
992 static void usb3_irq_epc_pipe0_status_end(struct renesas_usb3 *usb3)
993 {
994 	struct renesas_usb3_ep *usb3_ep = usb3_get_ep(usb3, 0);
995 	struct renesas_usb3_request *usb3_req = usb3_get_request(usb3_ep);
996 
997 	if (usb3_req)
998 		usb3_request_done(usb3_ep, usb3_req, 0);
999 	if (usb3->test_mode)
1000 		usb3_set_test_mode(usb3);
1001 }
1002 
1003 static void usb3_get_setup_data(struct renesas_usb3 *usb3,
1004 				struct usb_ctrlrequest *ctrl)
1005 {
1006 	struct renesas_usb3_ep *usb3_ep = usb3_get_ep(usb3, 0);
1007 	u32 *data = (u32 *)ctrl;
1008 
1009 	*data++ = usb3_read(usb3, USB3_STUP_DAT_0);
1010 	*data = usb3_read(usb3, USB3_STUP_DAT_1);
1011 
1012 	/* update this driver's flag */
1013 	usb3_ep->dir_in = !!(ctrl->bRequestType & USB_DIR_IN);
1014 }
1015 
1016 static void usb3_set_p0_con_update_res(struct renesas_usb3 *usb3, u32 res)
1017 {
1018 	u32 val = usb3_read(usb3, USB3_P0_CON);
1019 
1020 	val &= ~(P0_CON_ST_RES_MASK | P0_CON_OT_RES_MASK | P0_CON_IN_RES_MASK);
1021 	val |= res | P0_CON_RES_WEN;
1022 	usb3_write(usb3, val, USB3_P0_CON);
1023 }
1024 
1025 static void usb3_set_p0_con_for_ctrl_read_data(struct renesas_usb3 *usb3)
1026 {
1027 	usb3_set_p0_con_update_res(usb3, P0_CON_ST_RES_FORCE_NRDY |
1028 				   P0_CON_OT_RES_FORCE_STALL |
1029 				   P0_CON_IN_RES_NORMAL);
1030 }
1031 
1032 static void usb3_set_p0_con_for_ctrl_read_status(struct renesas_usb3 *usb3)
1033 {
1034 	usb3_set_p0_con_update_res(usb3, P0_CON_ST_RES_NORMAL |
1035 				   P0_CON_OT_RES_FORCE_STALL |
1036 				   P0_CON_IN_RES_NORMAL);
1037 }
1038 
1039 static void usb3_set_p0_con_for_ctrl_write_data(struct renesas_usb3 *usb3)
1040 {
1041 	usb3_set_p0_con_update_res(usb3, P0_CON_ST_RES_FORCE_NRDY |
1042 				   P0_CON_OT_RES_NORMAL |
1043 				   P0_CON_IN_RES_FORCE_STALL);
1044 }
1045 
1046 static void usb3_set_p0_con_for_ctrl_write_status(struct renesas_usb3 *usb3)
1047 {
1048 	usb3_set_p0_con_update_res(usb3, P0_CON_ST_RES_NORMAL |
1049 				   P0_CON_OT_RES_NORMAL |
1050 				   P0_CON_IN_RES_FORCE_STALL);
1051 }
1052 
1053 static void usb3_set_p0_con_for_no_data(struct renesas_usb3 *usb3)
1054 {
1055 	usb3_set_p0_con_update_res(usb3, P0_CON_ST_RES_NORMAL |
1056 				   P0_CON_OT_RES_FORCE_STALL |
1057 				   P0_CON_IN_RES_FORCE_STALL);
1058 }
1059 
1060 static void usb3_set_p0_con_stall(struct renesas_usb3 *usb3)
1061 {
1062 	usb3_set_p0_con_update_res(usb3, P0_CON_ST_RES_FORCE_STALL |
1063 				   P0_CON_OT_RES_FORCE_STALL |
1064 				   P0_CON_IN_RES_FORCE_STALL);
1065 }
1066 
1067 static void usb3_set_p0_con_stop(struct renesas_usb3 *usb3)
1068 {
1069 	usb3_set_p0_con_update_res(usb3, P0_CON_ST_RES_FORCE_NRDY |
1070 				   P0_CON_OT_RES_FORCE_NRDY |
1071 				   P0_CON_IN_RES_FORCE_NRDY);
1072 }
1073 
1074 static int usb3_pn_change(struct renesas_usb3 *usb3, int num)
1075 {
1076 	if (num == 0 || num > usb3->num_usb3_eps)
1077 		return -ENXIO;
1078 
1079 	usb3_write(usb3, num, USB3_PIPE_COM);
1080 
1081 	return 0;
1082 }
1083 
1084 static void usb3_set_pn_con_update_res(struct renesas_usb3 *usb3, u32 res)
1085 {
1086 	u32 val = usb3_read(usb3, USB3_PN_CON);
1087 
1088 	val &= ~PN_CON_RES_MASK;
1089 	val |= res & PN_CON_RES_MASK;
1090 	val |= PN_CON_RES_WEN;
1091 	usb3_write(usb3, val, USB3_PN_CON);
1092 }
1093 
1094 static void usb3_pn_start(struct renesas_usb3 *usb3)
1095 {
1096 	usb3_set_pn_con_update_res(usb3, PN_CON_RES_NORMAL);
1097 }
1098 
1099 static void usb3_pn_stop(struct renesas_usb3 *usb3)
1100 {
1101 	usb3_set_pn_con_update_res(usb3, PN_CON_RES_FORCE_NRDY);
1102 }
1103 
1104 static void usb3_pn_stall(struct renesas_usb3 *usb3)
1105 {
1106 	usb3_set_pn_con_update_res(usb3, PN_CON_RES_FORCE_STALL);
1107 }
1108 
1109 static int usb3_pn_con_clear(struct renesas_usb3 *usb3)
1110 {
1111 	usb3_set_bit(usb3, PN_CON_CLR, USB3_PN_CON);
1112 
1113 	return usb3_wait(usb3, USB3_PN_CON, PN_CON_CLR, 0);
1114 }
1115 
1116 static bool usb3_is_transfer_complete(struct renesas_usb3_ep *usb3_ep,
1117 				      struct renesas_usb3_request *usb3_req)
1118 {
1119 	struct usb_request *req = &usb3_req->req;
1120 
1121 	if ((!req->zero && req->actual == req->length) ||
1122 	    (req->actual % usb3_ep->ep.maxpacket) || (req->length == 0))
1123 		return true;
1124 	else
1125 		return false;
1126 }
1127 
1128 static int usb3_wait_pipe_status(struct renesas_usb3_ep *usb3_ep, u32 mask)
1129 {
1130 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1131 	u32 sta_reg = usb3_ep->num ? USB3_PN_STA : USB3_P0_STA;
1132 
1133 	return usb3_wait(usb3, sta_reg, mask, mask);
1134 }
1135 
1136 static void usb3_set_px_con_send(struct renesas_usb3_ep *usb3_ep, int bytes,
1137 				 bool last)
1138 {
1139 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1140 	u32 con_reg = usb3_ep->num ? USB3_PN_CON : USB3_P0_CON;
1141 	u32 val = usb3_read(usb3, con_reg);
1142 
1143 	val |= PX_CON_SEND | PX_CON_BYTE_EN_BYTES(bytes);
1144 	val |= (usb3_ep->num && last) ? PN_CON_LAST : 0;
1145 	usb3_write(usb3, val, con_reg);
1146 }
1147 
1148 static int usb3_write_pipe(struct renesas_usb3_ep *usb3_ep,
1149 			   struct renesas_usb3_request *usb3_req,
1150 			   u32 fifo_reg)
1151 {
1152 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1153 	int i;
1154 	int len = min_t(unsigned, usb3_req->req.length - usb3_req->req.actual,
1155 			usb3_ep->ep.maxpacket);
1156 	u8 *buf = usb3_req->req.buf + usb3_req->req.actual;
1157 	u32 tmp = 0;
1158 	bool is_last = !len ? true : false;
1159 
1160 	if (usb3_wait_pipe_status(usb3_ep, PX_STA_BUFSTS) < 0)
1161 		return -EBUSY;
1162 
1163 	/* Update gadget driver parameter */
1164 	usb3_req->req.actual += len;
1165 
1166 	/* Write data to the register */
1167 	if (len >= 4) {
1168 		iowrite32_rep(usb3->reg + fifo_reg, buf, len / 4);
1169 		buf += (len / 4) * 4;
1170 		len %= 4;	/* update len to use usb3_set_pX_con_send() */
1171 	}
1172 
1173 	if (len) {
1174 		for (i = 0; i < len; i++)
1175 			tmp |= buf[i] << (8 * i);
1176 		usb3_write(usb3, tmp, fifo_reg);
1177 	}
1178 
1179 	if (!is_last)
1180 		is_last = usb3_is_transfer_complete(usb3_ep, usb3_req);
1181 	/* Send the data */
1182 	usb3_set_px_con_send(usb3_ep, len, is_last);
1183 
1184 	return is_last ? 0 : -EAGAIN;
1185 }
1186 
1187 static u32 usb3_get_received_length(struct renesas_usb3_ep *usb3_ep)
1188 {
1189 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1190 	u32 lng_reg = usb3_ep->num ? USB3_PN_LNG : USB3_P0_LNG;
1191 
1192 	return usb3_read(usb3, lng_reg);
1193 }
1194 
1195 static int usb3_read_pipe(struct renesas_usb3_ep *usb3_ep,
1196 			  struct renesas_usb3_request *usb3_req, u32 fifo_reg)
1197 {
1198 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1199 	int i;
1200 	int len = min_t(unsigned, usb3_req->req.length - usb3_req->req.actual,
1201 			usb3_get_received_length(usb3_ep));
1202 	u8 *buf = usb3_req->req.buf + usb3_req->req.actual;
1203 	u32 tmp = 0;
1204 
1205 	if (!len)
1206 		return 0;
1207 
1208 	/* Update gadget driver parameter */
1209 	usb3_req->req.actual += len;
1210 
1211 	/* Read data from the register */
1212 	if (len >= 4) {
1213 		ioread32_rep(usb3->reg + fifo_reg, buf, len / 4);
1214 		buf += (len / 4) * 4;
1215 		len %= 4;
1216 	}
1217 
1218 	if (len) {
1219 		tmp = usb3_read(usb3, fifo_reg);
1220 		for (i = 0; i < len; i++)
1221 			buf[i] = (tmp >> (8 * i)) & 0xff;
1222 	}
1223 
1224 	return usb3_is_transfer_complete(usb3_ep, usb3_req) ? 0 : -EAGAIN;
1225 }
1226 
1227 static void usb3_set_status_stage(struct renesas_usb3_ep *usb3_ep,
1228 				  struct renesas_usb3_request *usb3_req)
1229 {
1230 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1231 
1232 	if (usb3_ep->dir_in) {
1233 		usb3_set_p0_con_for_ctrl_read_status(usb3);
1234 	} else {
1235 		if (!usb3_req->req.length)
1236 			usb3_set_p0_con_for_no_data(usb3);
1237 		else
1238 			usb3_set_p0_con_for_ctrl_write_status(usb3);
1239 	}
1240 }
1241 
1242 static void usb3_p0_xfer(struct renesas_usb3_ep *usb3_ep,
1243 			 struct renesas_usb3_request *usb3_req)
1244 {
1245 	int ret;
1246 
1247 	if (usb3_ep->dir_in)
1248 		ret = usb3_write_pipe(usb3_ep, usb3_req, USB3_P0_WRITE);
1249 	else
1250 		ret = usb3_read_pipe(usb3_ep, usb3_req, USB3_P0_READ);
1251 
1252 	if (!ret)
1253 		usb3_set_status_stage(usb3_ep, usb3_req);
1254 }
1255 
1256 static void usb3_start_pipe0(struct renesas_usb3_ep *usb3_ep,
1257 			     struct renesas_usb3_request *usb3_req)
1258 {
1259 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1260 
1261 	if (usb3_ep->started)
1262 		return;
1263 
1264 	usb3_ep->started = true;
1265 
1266 	if (usb3_ep->dir_in) {
1267 		usb3_set_bit(usb3, P0_MOD_DIR, USB3_P0_MOD);
1268 		usb3_set_p0_con_for_ctrl_read_data(usb3);
1269 	} else {
1270 		usb3_clear_bit(usb3, P0_MOD_DIR, USB3_P0_MOD);
1271 		if (usb3_req->req.length)
1272 			usb3_set_p0_con_for_ctrl_write_data(usb3);
1273 	}
1274 
1275 	usb3_p0_xfer(usb3_ep, usb3_req);
1276 }
1277 
1278 static void usb3_enable_dma_pipen(struct renesas_usb3 *usb3)
1279 {
1280 	usb3_set_bit(usb3, PN_CON_DATAIF_EN, USB3_PN_CON);
1281 }
1282 
1283 static void usb3_disable_dma_pipen(struct renesas_usb3 *usb3)
1284 {
1285 	usb3_clear_bit(usb3, PN_CON_DATAIF_EN, USB3_PN_CON);
1286 }
1287 
1288 static void usb3_enable_dma_irq(struct renesas_usb3 *usb3, int num)
1289 {
1290 	usb3_set_bit(usb3, DMA_INT(num), USB3_DMA_INT_ENA);
1291 }
1292 
1293 static void usb3_disable_dma_irq(struct renesas_usb3 *usb3, int num)
1294 {
1295 	usb3_clear_bit(usb3, DMA_INT(num), USB3_DMA_INT_ENA);
1296 }
1297 
1298 static u32 usb3_dma_mps_to_prd_word1(struct renesas_usb3_ep *usb3_ep)
1299 {
1300 	switch (usb3_ep->ep.maxpacket) {
1301 	case 8:
1302 		return USB3_PRD1_MPS_8;
1303 	case 16:
1304 		return USB3_PRD1_MPS_16;
1305 	case 32:
1306 		return USB3_PRD1_MPS_32;
1307 	case 64:
1308 		return USB3_PRD1_MPS_64;
1309 	case 512:
1310 		return USB3_PRD1_MPS_512;
1311 	case 1024:
1312 		return USB3_PRD1_MPS_1024;
1313 	default:
1314 		return USB3_PRD1_MPS_RESERVED;
1315 	}
1316 }
1317 
1318 static bool usb3_dma_get_setting_area(struct renesas_usb3_ep *usb3_ep,
1319 				      struct renesas_usb3_request *usb3_req)
1320 {
1321 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1322 	struct renesas_usb3_dma *dma;
1323 	int i;
1324 	bool ret = false;
1325 
1326 	if (usb3_req->req.length > USB3_DMA_MAX_XFER_SIZE_ALL_PRDS) {
1327 		dev_dbg(usb3_to_dev(usb3), "%s: the length is too big (%d)\n",
1328 			__func__, usb3_req->req.length);
1329 		return false;
1330 	}
1331 
1332 	/* The driver doesn't handle zero-length packet via dmac */
1333 	if (!usb3_req->req.length)
1334 		return false;
1335 
1336 	if (usb3_dma_mps_to_prd_word1(usb3_ep) == USB3_PRD1_MPS_RESERVED)
1337 		return false;
1338 
1339 	usb3_for_each_dma(usb3, dma, i) {
1340 		if (dma->used)
1341 			continue;
1342 
1343 		if (usb_gadget_map_request(&usb3->gadget, &usb3_req->req,
1344 					   usb3_ep->dir_in) < 0)
1345 			break;
1346 
1347 		dma->used = true;
1348 		usb3_ep->dma = dma;
1349 		ret = true;
1350 		break;
1351 	}
1352 
1353 	return ret;
1354 }
1355 
1356 static void usb3_dma_put_setting_area(struct renesas_usb3_ep *usb3_ep,
1357 				      struct renesas_usb3_request *usb3_req)
1358 {
1359 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1360 	int i;
1361 	struct renesas_usb3_dma *dma;
1362 
1363 	usb3_for_each_dma(usb3, dma, i) {
1364 		if (usb3_ep->dma == dma) {
1365 			usb_gadget_unmap_request(&usb3->gadget, &usb3_req->req,
1366 						 usb3_ep->dir_in);
1367 			dma->used = false;
1368 			usb3_ep->dma = NULL;
1369 			break;
1370 		}
1371 	}
1372 }
1373 
1374 static void usb3_dma_fill_prd(struct renesas_usb3_ep *usb3_ep,
1375 			      struct renesas_usb3_request *usb3_req)
1376 {
1377 	struct renesas_usb3_prd *cur_prd = usb3_ep->dma->prd;
1378 	u32 remain = usb3_req->req.length;
1379 	u32 dma = usb3_req->req.dma;
1380 	u32 len;
1381 	int i = 0;
1382 
1383 	do {
1384 		len = min_t(u32, remain, USB3_DMA_MAX_XFER_SIZE) &
1385 			    USB3_PRD1_SIZE_MASK;
1386 		cur_prd->word1 = usb3_dma_mps_to_prd_word1(usb3_ep) |
1387 				 USB3_PRD1_B_INC | len;
1388 		cur_prd->bap = dma;
1389 		remain -= len;
1390 		dma += len;
1391 		if (!remain || (i + 1) < USB3_DMA_NUM_PRD_ENTRIES)
1392 			break;
1393 
1394 		cur_prd++;
1395 		i++;
1396 	} while (1);
1397 
1398 	cur_prd->word1 |= USB3_PRD1_E | USB3_PRD1_INT;
1399 	if (usb3_ep->dir_in)
1400 		cur_prd->word1 |= USB3_PRD1_LST;
1401 }
1402 
1403 static void usb3_dma_kick_prd(struct renesas_usb3_ep *usb3_ep)
1404 {
1405 	struct renesas_usb3_dma *dma = usb3_ep->dma;
1406 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1407 	u32 dma_con = DMA_COM_PIPE_NO(usb3_ep->num) | DMA_CON_PRD_EN;
1408 
1409 	if (usb3_ep->dir_in)
1410 		dma_con |= DMA_CON_PIPE_DIR;
1411 
1412 	wmb();	/* prd entries should be in system memory here */
1413 
1414 	usb3_write(usb3, 1 << usb3_ep->num, USB3_DMA_INT_STA);
1415 	usb3_write(usb3, AXI_INT_PRDEN_CLR_STA(dma->num) |
1416 		   AXI_INT_PRDERR_STA(dma->num), USB3_AXI_INT_STA);
1417 
1418 	usb3_write(usb3, dma->prd_dma, USB3_DMA_CH0_PRD_ADR(dma->num));
1419 	usb3_write(usb3, dma_con, USB3_DMA_CH0_CON(dma->num));
1420 	usb3_enable_dma_irq(usb3, usb3_ep->num);
1421 }
1422 
1423 static void usb3_dma_stop_prd(struct renesas_usb3_ep *usb3_ep)
1424 {
1425 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1426 	struct renesas_usb3_dma *dma = usb3_ep->dma;
1427 
1428 	usb3_disable_dma_irq(usb3, usb3_ep->num);
1429 	usb3_write(usb3, 0, USB3_DMA_CH0_CON(dma->num));
1430 }
1431 
1432 static int usb3_dma_update_status(struct renesas_usb3_ep *usb3_ep,
1433 				  struct renesas_usb3_request *usb3_req)
1434 {
1435 	struct renesas_usb3_prd *cur_prd = usb3_ep->dma->prd;
1436 	struct usb_request *req = &usb3_req->req;
1437 	u32 remain, len;
1438 	int i = 0;
1439 	int status = 0;
1440 
1441 	rmb();	/* The controller updated prd entries */
1442 
1443 	do {
1444 		if (cur_prd->word1 & USB3_PRD1_D)
1445 			status = -EIO;
1446 		if (cur_prd->word1 & USB3_PRD1_E)
1447 			len = req->length % USB3_DMA_MAX_XFER_SIZE;
1448 		else
1449 			len = USB3_DMA_MAX_XFER_SIZE;
1450 		remain = cur_prd->word1 & USB3_PRD1_SIZE_MASK;
1451 		req->actual += len - remain;
1452 
1453 		if (cur_prd->word1 & USB3_PRD1_E ||
1454 		    (i + 1) < USB3_DMA_NUM_PRD_ENTRIES)
1455 			break;
1456 
1457 		cur_prd++;
1458 		i++;
1459 	} while (1);
1460 
1461 	return status;
1462 }
1463 
1464 static bool usb3_dma_try_start(struct renesas_usb3_ep *usb3_ep,
1465 			       struct renesas_usb3_request *usb3_req)
1466 {
1467 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1468 
1469 	if (!use_dma)
1470 		return false;
1471 
1472 	if (usb3_dma_get_setting_area(usb3_ep, usb3_req)) {
1473 		usb3_pn_stop(usb3);
1474 		usb3_enable_dma_pipen(usb3);
1475 		usb3_dma_fill_prd(usb3_ep, usb3_req);
1476 		usb3_dma_kick_prd(usb3_ep);
1477 		usb3_pn_start(usb3);
1478 		return true;
1479 	}
1480 
1481 	return false;
1482 }
1483 
1484 static int usb3_dma_try_stop(struct renesas_usb3_ep *usb3_ep,
1485 			     struct renesas_usb3_request *usb3_req)
1486 {
1487 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1488 	unsigned long flags;
1489 	int status = 0;
1490 
1491 	spin_lock_irqsave(&usb3->lock, flags);
1492 	if (!usb3_ep->dma)
1493 		goto out;
1494 
1495 	if (!usb3_pn_change(usb3, usb3_ep->num))
1496 		usb3_disable_dma_pipen(usb3);
1497 	usb3_dma_stop_prd(usb3_ep);
1498 	status = usb3_dma_update_status(usb3_ep, usb3_req);
1499 	usb3_dma_put_setting_area(usb3_ep, usb3_req);
1500 
1501 out:
1502 	spin_unlock_irqrestore(&usb3->lock, flags);
1503 	return status;
1504 }
1505 
1506 static int renesas_usb3_dma_free_prd(struct renesas_usb3 *usb3,
1507 				     struct device *dev)
1508 {
1509 	int i;
1510 	struct renesas_usb3_dma *dma;
1511 
1512 	usb3_for_each_dma(usb3, dma, i) {
1513 		if (dma->prd) {
1514 			dma_free_coherent(dev, USB3_DMA_PRD_SIZE,
1515 					  dma->prd, dma->prd_dma);
1516 			dma->prd = NULL;
1517 		}
1518 	}
1519 
1520 	return 0;
1521 }
1522 
1523 static int renesas_usb3_dma_alloc_prd(struct renesas_usb3 *usb3,
1524 				      struct device *dev)
1525 {
1526 	int i;
1527 	struct renesas_usb3_dma *dma;
1528 
1529 	if (!use_dma)
1530 		return 0;
1531 
1532 	usb3_for_each_dma(usb3, dma, i) {
1533 		dma->prd = dma_alloc_coherent(dev, USB3_DMA_PRD_SIZE,
1534 					      &dma->prd_dma, GFP_KERNEL);
1535 		if (!dma->prd) {
1536 			renesas_usb3_dma_free_prd(usb3, dev);
1537 			return -ENOMEM;
1538 		}
1539 		dma->num = i + 1;
1540 	}
1541 
1542 	return 0;
1543 }
1544 
1545 static void usb3_start_pipen(struct renesas_usb3_ep *usb3_ep,
1546 			     struct renesas_usb3_request *usb3_req)
1547 {
1548 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1549 	struct renesas_usb3_request *usb3_req_first;
1550 	unsigned long flags;
1551 	int ret = -EAGAIN;
1552 	u32 enable_bits = 0;
1553 
1554 	spin_lock_irqsave(&usb3->lock, flags);
1555 	if (usb3_ep->halt || usb3_ep->started)
1556 		goto out;
1557 	usb3_req_first = __usb3_get_request(usb3_ep);
1558 	if (!usb3_req_first || usb3_req != usb3_req_first)
1559 		goto out;
1560 
1561 	if (usb3_pn_change(usb3, usb3_ep->num) < 0)
1562 		goto out;
1563 
1564 	usb3_ep->started = true;
1565 
1566 	if (usb3_dma_try_start(usb3_ep, usb3_req))
1567 		goto out;
1568 
1569 	usb3_pn_start(usb3);
1570 
1571 	if (usb3_ep->dir_in) {
1572 		ret = usb3_write_pipe(usb3_ep, usb3_req, USB3_PN_WRITE);
1573 		enable_bits |= PN_INT_LSTTR;
1574 	}
1575 
1576 	if (ret < 0)
1577 		enable_bits |= PN_INT_BFRDY;
1578 
1579 	if (enable_bits) {
1580 		usb3_set_bit(usb3, enable_bits, USB3_PN_INT_ENA);
1581 		usb3_enable_pipe_irq(usb3, usb3_ep->num);
1582 	}
1583 out:
1584 	spin_unlock_irqrestore(&usb3->lock, flags);
1585 }
1586 
1587 static int renesas_usb3_ep_queue(struct usb_ep *_ep, struct usb_request *_req,
1588 				 gfp_t gfp_flags)
1589 {
1590 	struct renesas_usb3_ep *usb3_ep = usb_ep_to_usb3_ep(_ep);
1591 	struct renesas_usb3_request *usb3_req = usb_req_to_usb3_req(_req);
1592 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1593 	unsigned long flags;
1594 
1595 	dev_dbg(usb3_to_dev(usb3), "ep_queue: ep%2d, %u\n", usb3_ep->num,
1596 		_req->length);
1597 
1598 	_req->status = -EINPROGRESS;
1599 	_req->actual = 0;
1600 	spin_lock_irqsave(&usb3->lock, flags);
1601 	list_add_tail(&usb3_req->queue, &usb3_ep->queue);
1602 	spin_unlock_irqrestore(&usb3->lock, flags);
1603 
1604 	if (!usb3_ep->num)
1605 		usb3_start_pipe0(usb3_ep, usb3_req);
1606 	else
1607 		usb3_start_pipen(usb3_ep, usb3_req);
1608 
1609 	return 0;
1610 }
1611 
1612 static void usb3_set_device_address(struct renesas_usb3 *usb3, u16 addr)
1613 {
1614 	/* DEV_ADDR bit field is cleared by WarmReset, HotReset and BusReset */
1615 	usb3_set_bit(usb3, USB_COM_CON_DEV_ADDR(addr), USB3_USB_COM_CON);
1616 }
1617 
1618 static bool usb3_std_req_set_address(struct renesas_usb3 *usb3,
1619 				     struct usb_ctrlrequest *ctrl)
1620 {
1621 	if (le16_to_cpu(ctrl->wValue) >= 128)
1622 		return true;	/* stall */
1623 
1624 	usb3_set_device_address(usb3, le16_to_cpu(ctrl->wValue));
1625 	usb3_set_p0_con_for_no_data(usb3);
1626 
1627 	return false;
1628 }
1629 
1630 static void usb3_pipe0_internal_xfer(struct renesas_usb3 *usb3,
1631 				     void *tx_data, size_t len,
1632 				     void (*complete)(struct usb_ep *ep,
1633 						      struct usb_request *req))
1634 {
1635 	struct renesas_usb3_ep *usb3_ep = usb3_get_ep(usb3, 0);
1636 
1637 	if (tx_data)
1638 		memcpy(usb3->ep0_buf, tx_data,
1639 		       min_t(size_t, len, USB3_EP0_BUF_SIZE));
1640 
1641 	usb3->ep0_req->buf = &usb3->ep0_buf;
1642 	usb3->ep0_req->length = len;
1643 	usb3->ep0_req->complete = complete;
1644 	renesas_usb3_ep_queue(&usb3_ep->ep, usb3->ep0_req, GFP_ATOMIC);
1645 }
1646 
1647 static void usb3_pipe0_get_status_completion(struct usb_ep *ep,
1648 					     struct usb_request *req)
1649 {
1650 }
1651 
1652 static bool usb3_std_req_get_status(struct renesas_usb3 *usb3,
1653 				    struct usb_ctrlrequest *ctrl)
1654 {
1655 	bool stall = false;
1656 	struct renesas_usb3_ep *usb3_ep;
1657 	int num;
1658 	u16 status = 0;
1659 	__le16 tx_data;
1660 
1661 	switch (ctrl->bRequestType & USB_RECIP_MASK) {
1662 	case USB_RECIP_DEVICE:
1663 		if (usb3->gadget.is_selfpowered)
1664 			status |= 1 << USB_DEVICE_SELF_POWERED;
1665 		if (usb3->gadget.speed == USB_SPEED_SUPER)
1666 			status |= usb3_feature_get_un_enabled(usb3);
1667 		break;
1668 	case USB_RECIP_INTERFACE:
1669 		break;
1670 	case USB_RECIP_ENDPOINT:
1671 		num = le16_to_cpu(ctrl->wIndex) & USB_ENDPOINT_NUMBER_MASK;
1672 		usb3_ep = usb3_get_ep(usb3, num);
1673 		if (usb3_ep->halt)
1674 			status |= 1 << USB_ENDPOINT_HALT;
1675 		break;
1676 	default:
1677 		stall = true;
1678 		break;
1679 	}
1680 
1681 	if (!stall) {
1682 		tx_data = cpu_to_le16(status);
1683 		dev_dbg(usb3_to_dev(usb3), "get_status: req = %p\n",
1684 			usb_req_to_usb3_req(usb3->ep0_req));
1685 		usb3_pipe0_internal_xfer(usb3, &tx_data, sizeof(tx_data),
1686 					 usb3_pipe0_get_status_completion);
1687 	}
1688 
1689 	return stall;
1690 }
1691 
1692 static bool usb3_std_req_feature_device(struct renesas_usb3 *usb3,
1693 					struct usb_ctrlrequest *ctrl, bool set)
1694 {
1695 	bool stall = true;
1696 	u16 w_value = le16_to_cpu(ctrl->wValue);
1697 
1698 	switch (w_value) {
1699 	case USB_DEVICE_TEST_MODE:
1700 		if (!set)
1701 			break;
1702 		usb3->test_mode = le16_to_cpu(ctrl->wIndex) >> 8;
1703 		stall = false;
1704 		break;
1705 	case USB_DEVICE_U1_ENABLE:
1706 	case USB_DEVICE_U2_ENABLE:
1707 		if (usb3->gadget.speed != USB_SPEED_SUPER)
1708 			break;
1709 		if (w_value == USB_DEVICE_U1_ENABLE)
1710 			usb3_feature_u1_enable(usb3, set);
1711 		if (w_value == USB_DEVICE_U2_ENABLE)
1712 			usb3_feature_u2_enable(usb3, set);
1713 		stall = false;
1714 		break;
1715 	default:
1716 		break;
1717 	}
1718 
1719 	return stall;
1720 }
1721 
1722 static int usb3_set_halt_p0(struct renesas_usb3_ep *usb3_ep, bool halt)
1723 {
1724 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1725 
1726 	if (unlikely(usb3_ep->num))
1727 		return -EINVAL;
1728 
1729 	usb3_ep->halt = halt;
1730 	if (halt)
1731 		usb3_set_p0_con_stall(usb3);
1732 	else
1733 		usb3_set_p0_con_stop(usb3);
1734 
1735 	return 0;
1736 }
1737 
1738 static int usb3_set_halt_pn(struct renesas_usb3_ep *usb3_ep, bool halt,
1739 			    bool is_clear_feature)
1740 {
1741 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
1742 	unsigned long flags;
1743 
1744 	spin_lock_irqsave(&usb3->lock, flags);
1745 	if (!usb3_pn_change(usb3, usb3_ep->num)) {
1746 		usb3_ep->halt = halt;
1747 		if (halt) {
1748 			usb3_pn_stall(usb3);
1749 		} else if (!is_clear_feature || !usb3_ep->wedge) {
1750 			usb3_pn_con_clear(usb3);
1751 			usb3_set_bit(usb3, PN_CON_EN, USB3_PN_CON);
1752 			usb3_pn_stop(usb3);
1753 		}
1754 	}
1755 	spin_unlock_irqrestore(&usb3->lock, flags);
1756 
1757 	return 0;
1758 }
1759 
1760 static int usb3_set_halt(struct renesas_usb3_ep *usb3_ep, bool halt,
1761 			 bool is_clear_feature)
1762 {
1763 	int ret = 0;
1764 
1765 	if (halt && usb3_ep->started)
1766 		return -EAGAIN;
1767 
1768 	if (usb3_ep->num)
1769 		ret = usb3_set_halt_pn(usb3_ep, halt, is_clear_feature);
1770 	else
1771 		ret = usb3_set_halt_p0(usb3_ep, halt);
1772 
1773 	return ret;
1774 }
1775 
1776 static bool usb3_std_req_feature_endpoint(struct renesas_usb3 *usb3,
1777 					  struct usb_ctrlrequest *ctrl,
1778 					  bool set)
1779 {
1780 	int num = le16_to_cpu(ctrl->wIndex) & USB_ENDPOINT_NUMBER_MASK;
1781 	struct renesas_usb3_ep *usb3_ep;
1782 	struct renesas_usb3_request *usb3_req;
1783 
1784 	if (le16_to_cpu(ctrl->wValue) != USB_ENDPOINT_HALT)
1785 		return true;	/* stall */
1786 
1787 	usb3_ep = usb3_get_ep(usb3, num);
1788 	usb3_set_halt(usb3_ep, set, true);
1789 
1790 	/* Restarts a queue if clear feature */
1791 	if (!set) {
1792 		usb3_ep->started = false;
1793 		usb3_req = usb3_get_request(usb3_ep);
1794 		if (usb3_req)
1795 			usb3_start_pipen(usb3_ep, usb3_req);
1796 	}
1797 
1798 	return false;
1799 }
1800 
1801 static bool usb3_std_req_feature(struct renesas_usb3 *usb3,
1802 				 struct usb_ctrlrequest *ctrl, bool set)
1803 {
1804 	bool stall = false;
1805 
1806 	switch (ctrl->bRequestType & USB_RECIP_MASK) {
1807 	case USB_RECIP_DEVICE:
1808 		stall = usb3_std_req_feature_device(usb3, ctrl, set);
1809 		break;
1810 	case USB_RECIP_INTERFACE:
1811 		break;
1812 	case USB_RECIP_ENDPOINT:
1813 		stall = usb3_std_req_feature_endpoint(usb3, ctrl, set);
1814 		break;
1815 	default:
1816 		stall = true;
1817 		break;
1818 	}
1819 
1820 	if (!stall)
1821 		usb3_set_p0_con_for_no_data(usb3);
1822 
1823 	return stall;
1824 }
1825 
1826 static void usb3_pipe0_set_sel_completion(struct usb_ep *ep,
1827 					  struct usb_request *req)
1828 {
1829 	/* TODO */
1830 }
1831 
1832 static bool usb3_std_req_set_sel(struct renesas_usb3 *usb3,
1833 				 struct usb_ctrlrequest *ctrl)
1834 {
1835 	u16 w_length = le16_to_cpu(ctrl->wLength);
1836 
1837 	if (w_length != 6)
1838 		return true;	/* stall */
1839 
1840 	dev_dbg(usb3_to_dev(usb3), "set_sel: req = %p\n",
1841 		usb_req_to_usb3_req(usb3->ep0_req));
1842 	usb3_pipe0_internal_xfer(usb3, NULL, 6, usb3_pipe0_set_sel_completion);
1843 
1844 	return false;
1845 }
1846 
1847 static bool usb3_std_req_set_configuration(struct renesas_usb3 *usb3,
1848 					   struct usb_ctrlrequest *ctrl)
1849 {
1850 	if (le16_to_cpu(ctrl->wValue) > 0)
1851 		usb3_set_bit(usb3, USB_COM_CON_CONF, USB3_USB_COM_CON);
1852 	else
1853 		usb3_clear_bit(usb3, USB_COM_CON_CONF, USB3_USB_COM_CON);
1854 
1855 	return false;
1856 }
1857 
1858 /**
1859  * usb3_handle_standard_request - handle some standard requests
1860  * @usb3: the renesas_usb3 pointer
1861  * @ctrl: a pointer of setup data
1862  *
1863  * Returns true if this function handled a standard request
1864  */
1865 static bool usb3_handle_standard_request(struct renesas_usb3 *usb3,
1866 					 struct usb_ctrlrequest *ctrl)
1867 {
1868 	bool ret = false;
1869 	bool stall = false;
1870 
1871 	if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1872 		switch (ctrl->bRequest) {
1873 		case USB_REQ_SET_ADDRESS:
1874 			stall = usb3_std_req_set_address(usb3, ctrl);
1875 			ret = true;
1876 			break;
1877 		case USB_REQ_GET_STATUS:
1878 			stall = usb3_std_req_get_status(usb3, ctrl);
1879 			ret = true;
1880 			break;
1881 		case USB_REQ_CLEAR_FEATURE:
1882 			stall = usb3_std_req_feature(usb3, ctrl, false);
1883 			ret = true;
1884 			break;
1885 		case USB_REQ_SET_FEATURE:
1886 			stall = usb3_std_req_feature(usb3, ctrl, true);
1887 			ret = true;
1888 			break;
1889 		case USB_REQ_SET_SEL:
1890 			stall = usb3_std_req_set_sel(usb3, ctrl);
1891 			ret = true;
1892 			break;
1893 		case USB_REQ_SET_ISOCH_DELAY:
1894 			/* This hardware doesn't support Isochronous xfer */
1895 			stall = true;
1896 			ret = true;
1897 			break;
1898 		case USB_REQ_SET_CONFIGURATION:
1899 			usb3_std_req_set_configuration(usb3, ctrl);
1900 			break;
1901 		default:
1902 			break;
1903 		}
1904 	}
1905 
1906 	if (stall)
1907 		usb3_set_p0_con_stall(usb3);
1908 
1909 	return ret;
1910 }
1911 
1912 static int usb3_p0_con_clear_buffer(struct renesas_usb3 *usb3)
1913 {
1914 	usb3_set_bit(usb3, P0_CON_BCLR, USB3_P0_CON);
1915 
1916 	return usb3_wait(usb3, USB3_P0_CON, P0_CON_BCLR, 0);
1917 }
1918 
1919 static void usb3_irq_epc_pipe0_setup(struct renesas_usb3 *usb3)
1920 {
1921 	struct usb_ctrlrequest ctrl;
1922 	struct renesas_usb3_ep *usb3_ep = usb3_get_ep(usb3, 0);
1923 
1924 	/* Call giveback function if previous transfer is not completed */
1925 	if (usb3_ep->started)
1926 		usb3_request_done(usb3_ep, usb3_get_request(usb3_ep),
1927 				  -ECONNRESET);
1928 
1929 	usb3_p0_con_clear_buffer(usb3);
1930 	usb3_get_setup_data(usb3, &ctrl);
1931 	if (!usb3_handle_standard_request(usb3, &ctrl))
1932 		if (usb3->driver->setup(&usb3->gadget, &ctrl) < 0)
1933 			usb3_set_p0_con_stall(usb3);
1934 }
1935 
1936 static void usb3_irq_epc_pipe0_bfrdy(struct renesas_usb3 *usb3)
1937 {
1938 	struct renesas_usb3_ep *usb3_ep = usb3_get_ep(usb3, 0);
1939 	struct renesas_usb3_request *usb3_req = usb3_get_request(usb3_ep);
1940 
1941 	if (!usb3_req)
1942 		return;
1943 
1944 	usb3_p0_xfer(usb3_ep, usb3_req);
1945 }
1946 
1947 static void usb3_irq_epc_pipe0(struct renesas_usb3 *usb3)
1948 {
1949 	u32 p0_int_sta = usb3_read(usb3, USB3_P0_INT_STA);
1950 
1951 	p0_int_sta &= usb3_read(usb3, USB3_P0_INT_ENA);
1952 	usb3_write(usb3, p0_int_sta, USB3_P0_INT_STA);
1953 	if (p0_int_sta & P0_INT_STSED)
1954 		usb3_irq_epc_pipe0_status_end(usb3);
1955 	if (p0_int_sta & P0_INT_SETUP)
1956 		usb3_irq_epc_pipe0_setup(usb3);
1957 	if (p0_int_sta & P0_INT_BFRDY)
1958 		usb3_irq_epc_pipe0_bfrdy(usb3);
1959 }
1960 
1961 static void usb3_request_done_pipen(struct renesas_usb3 *usb3,
1962 				    struct renesas_usb3_ep *usb3_ep,
1963 				    struct renesas_usb3_request *usb3_req,
1964 				    int status)
1965 {
1966 	unsigned long flags;
1967 
1968 	spin_lock_irqsave(&usb3->lock, flags);
1969 	if (usb3_pn_change(usb3, usb3_ep->num))
1970 		usb3_pn_stop(usb3);
1971 	spin_unlock_irqrestore(&usb3->lock, flags);
1972 
1973 	usb3_disable_pipe_irq(usb3, usb3_ep->num);
1974 	usb3_request_done(usb3_ep, usb3_req, status);
1975 
1976 	/* get next usb3_req */
1977 	usb3_req = usb3_get_request(usb3_ep);
1978 	if (usb3_req)
1979 		usb3_start_pipen(usb3_ep, usb3_req);
1980 }
1981 
1982 static void usb3_irq_epc_pipen_lsttr(struct renesas_usb3 *usb3, int num)
1983 {
1984 	struct renesas_usb3_ep *usb3_ep = usb3_get_ep(usb3, num);
1985 	struct renesas_usb3_request *usb3_req = usb3_get_request(usb3_ep);
1986 
1987 	if (!usb3_req)
1988 		return;
1989 
1990 	if (usb3_ep->dir_in) {
1991 		dev_dbg(usb3_to_dev(usb3), "%s: len = %u, actual = %u\n",
1992 			__func__, usb3_req->req.length, usb3_req->req.actual);
1993 		usb3_request_done_pipen(usb3, usb3_ep, usb3_req, 0);
1994 	}
1995 }
1996 
1997 static void usb3_irq_epc_pipen_bfrdy(struct renesas_usb3 *usb3, int num)
1998 {
1999 	struct renesas_usb3_ep *usb3_ep = usb3_get_ep(usb3, num);
2000 	struct renesas_usb3_request *usb3_req = usb3_get_request(usb3_ep);
2001 	bool done = false;
2002 
2003 	if (!usb3_req)
2004 		return;
2005 
2006 	spin_lock(&usb3->lock);
2007 	if (usb3_pn_change(usb3, num))
2008 		goto out;
2009 
2010 	if (usb3_ep->dir_in) {
2011 		/* Do not stop the IN pipe here to detect LSTTR interrupt */
2012 		if (!usb3_write_pipe(usb3_ep, usb3_req, USB3_PN_WRITE))
2013 			usb3_clear_bit(usb3, PN_INT_BFRDY, USB3_PN_INT_ENA);
2014 	} else {
2015 		if (!usb3_read_pipe(usb3_ep, usb3_req, USB3_PN_READ))
2016 			done = true;
2017 	}
2018 
2019 out:
2020 	/* need to unlock because usb3_request_done_pipen() locks it */
2021 	spin_unlock(&usb3->lock);
2022 
2023 	if (done)
2024 		usb3_request_done_pipen(usb3, usb3_ep, usb3_req, 0);
2025 }
2026 
2027 static void usb3_irq_epc_pipen(struct renesas_usb3 *usb3, int num)
2028 {
2029 	u32 pn_int_sta;
2030 
2031 	spin_lock(&usb3->lock);
2032 	if (usb3_pn_change(usb3, num) < 0) {
2033 		spin_unlock(&usb3->lock);
2034 		return;
2035 	}
2036 
2037 	pn_int_sta = usb3_read(usb3, USB3_PN_INT_STA);
2038 	pn_int_sta &= usb3_read(usb3, USB3_PN_INT_ENA);
2039 	usb3_write(usb3, pn_int_sta, USB3_PN_INT_STA);
2040 	spin_unlock(&usb3->lock);
2041 	if (pn_int_sta & PN_INT_LSTTR)
2042 		usb3_irq_epc_pipen_lsttr(usb3, num);
2043 	if (pn_int_sta & PN_INT_BFRDY)
2044 		usb3_irq_epc_pipen_bfrdy(usb3, num);
2045 }
2046 
2047 static void usb3_irq_epc_int_2(struct renesas_usb3 *usb3, u32 int_sta_2)
2048 {
2049 	int i;
2050 
2051 	for (i = 0; i < usb3->num_usb3_eps; i++) {
2052 		if (int_sta_2 & USB_INT_2_PIPE(i)) {
2053 			if (!i)
2054 				usb3_irq_epc_pipe0(usb3);
2055 			else
2056 				usb3_irq_epc_pipen(usb3, i);
2057 		}
2058 	}
2059 }
2060 
2061 static void usb3_irq_idmon_change(struct renesas_usb3 *usb3)
2062 {
2063 	usb3_check_id(usb3);
2064 }
2065 
2066 static void usb3_irq_otg_int(struct renesas_usb3 *usb3)
2067 {
2068 	u32 otg_int_sta = usb3_drd_read(usb3, USB3_USB_OTG_INT_STA(usb3));
2069 
2070 	otg_int_sta &= usb3_drd_read(usb3, USB3_USB_OTG_INT_ENA(usb3));
2071 	if (otg_int_sta)
2072 		usb3_drd_write(usb3, otg_int_sta, USB3_USB_OTG_INT_STA(usb3));
2073 
2074 	if (otg_int_sta & USB_OTG_IDMON(usb3))
2075 		usb3_irq_idmon_change(usb3);
2076 }
2077 
2078 static void usb3_irq_epc(struct renesas_usb3 *usb3)
2079 {
2080 	u32 int_sta_1 = usb3_read(usb3, USB3_USB_INT_STA_1);
2081 	u32 int_sta_2 = usb3_read(usb3, USB3_USB_INT_STA_2);
2082 
2083 	int_sta_1 &= usb3_read(usb3, USB3_USB_INT_ENA_1);
2084 	if (int_sta_1) {
2085 		usb3_write(usb3, int_sta_1, USB3_USB_INT_STA_1);
2086 		usb3_irq_epc_int_1(usb3, int_sta_1);
2087 	}
2088 
2089 	int_sta_2 &= usb3_read(usb3, USB3_USB_INT_ENA_2);
2090 	if (int_sta_2)
2091 		usb3_irq_epc_int_2(usb3, int_sta_2);
2092 
2093 	if (!usb3->is_rzv2m)
2094 		usb3_irq_otg_int(usb3);
2095 }
2096 
2097 static void usb3_irq_dma_int(struct renesas_usb3 *usb3, u32 dma_sta)
2098 {
2099 	struct renesas_usb3_ep *usb3_ep;
2100 	struct renesas_usb3_request *usb3_req;
2101 	int i, status;
2102 
2103 	for (i = 0; i < usb3->num_usb3_eps; i++) {
2104 		if (!(dma_sta & DMA_INT(i)))
2105 			continue;
2106 
2107 		usb3_ep = usb3_get_ep(usb3, i);
2108 		if (!(usb3_read(usb3, USB3_AXI_INT_STA) &
2109 		    AXI_INT_PRDEN_CLR_STA(usb3_ep->dma->num)))
2110 			continue;
2111 
2112 		usb3_req = usb3_get_request(usb3_ep);
2113 		status = usb3_dma_try_stop(usb3_ep, usb3_req);
2114 		usb3_request_done_pipen(usb3, usb3_ep, usb3_req, status);
2115 	}
2116 }
2117 
2118 static void usb3_irq_dma(struct renesas_usb3 *usb3)
2119 {
2120 	u32 dma_sta = usb3_read(usb3, USB3_DMA_INT_STA);
2121 
2122 	dma_sta &= usb3_read(usb3, USB3_DMA_INT_ENA);
2123 	if (dma_sta) {
2124 		usb3_write(usb3, dma_sta, USB3_DMA_INT_STA);
2125 		usb3_irq_dma_int(usb3, dma_sta);
2126 	}
2127 }
2128 
2129 static irqreturn_t renesas_usb3_irq(int irq, void *_usb3)
2130 {
2131 	struct renesas_usb3 *usb3 = _usb3;
2132 	irqreturn_t ret = IRQ_NONE;
2133 	u32 axi_int_sta = usb3_read(usb3, USB3_AXI_INT_STA);
2134 
2135 	if (axi_int_sta & AXI_INT_DMAINT) {
2136 		usb3_irq_dma(usb3);
2137 		ret = IRQ_HANDLED;
2138 	}
2139 
2140 	if (axi_int_sta & AXI_INT_EPCINT) {
2141 		usb3_irq_epc(usb3);
2142 		ret = IRQ_HANDLED;
2143 	}
2144 
2145 	return ret;
2146 }
2147 
2148 static irqreturn_t renesas_usb3_otg_irq(int irq, void *_usb3)
2149 {
2150 	struct renesas_usb3 *usb3 = _usb3;
2151 
2152 	usb3_irq_otg_int(usb3);
2153 
2154 	return IRQ_HANDLED;
2155 }
2156 
2157 static void usb3_write_pn_mod(struct renesas_usb3_ep *usb3_ep,
2158 			      const struct usb_endpoint_descriptor *desc)
2159 {
2160 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
2161 	u32 val = 0;
2162 
2163 	val |= usb3_ep->dir_in ? PN_MOD_DIR : 0;
2164 	val |= PN_MOD_TYPE(usb_endpoint_type(desc));
2165 	val |= PN_MOD_EPNUM(usb_endpoint_num(desc));
2166 	usb3_write(usb3, val, USB3_PN_MOD);
2167 }
2168 
2169 static u32 usb3_calc_ramarea(int ram_size)
2170 {
2171 	WARN_ON(ram_size > SZ_16K);
2172 
2173 	if (ram_size <= SZ_1K)
2174 		return PN_RAMMAP_RAMAREA_1KB;
2175 	else if (ram_size <= SZ_2K)
2176 		return PN_RAMMAP_RAMAREA_2KB;
2177 	else if (ram_size <= SZ_4K)
2178 		return PN_RAMMAP_RAMAREA_4KB;
2179 	else if (ram_size <= SZ_8K)
2180 		return PN_RAMMAP_RAMAREA_8KB;
2181 	else
2182 		return PN_RAMMAP_RAMAREA_16KB;
2183 }
2184 
2185 static u32 usb3_calc_rammap_val(struct renesas_usb3_ep *usb3_ep,
2186 				const struct usb_endpoint_descriptor *desc)
2187 {
2188 	int i;
2189 	static const u32 max_packet_array[] = {8, 16, 32, 64, 512};
2190 	u32 mpkt = PN_RAMMAP_MPKT(1024);
2191 
2192 	for (i = 0; i < ARRAY_SIZE(max_packet_array); i++) {
2193 		if (usb_endpoint_maxp(desc) <= max_packet_array[i])
2194 			mpkt = PN_RAMMAP_MPKT(max_packet_array[i]);
2195 	}
2196 
2197 	return usb3_ep->rammap_val | mpkt;
2198 }
2199 
2200 static int usb3_enable_pipe_n(struct renesas_usb3_ep *usb3_ep,
2201 			      const struct usb_endpoint_descriptor *desc)
2202 {
2203 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
2204 	unsigned long flags;
2205 
2206 	usb3_ep->dir_in = usb_endpoint_dir_in(desc);
2207 
2208 	spin_lock_irqsave(&usb3->lock, flags);
2209 	if (!usb3_pn_change(usb3, usb3_ep->num)) {
2210 		usb3_write_pn_mod(usb3_ep, desc);
2211 		usb3_write(usb3, usb3_calc_rammap_val(usb3_ep, desc),
2212 			   USB3_PN_RAMMAP);
2213 		usb3_pn_con_clear(usb3);
2214 		usb3_set_bit(usb3, PN_CON_EN, USB3_PN_CON);
2215 	}
2216 	spin_unlock_irqrestore(&usb3->lock, flags);
2217 
2218 	return 0;
2219 }
2220 
2221 static int usb3_disable_pipe_n(struct renesas_usb3_ep *usb3_ep)
2222 {
2223 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
2224 	unsigned long flags;
2225 
2226 	usb3_ep->halt = false;
2227 
2228 	spin_lock_irqsave(&usb3->lock, flags);
2229 	if (!usb3_pn_change(usb3, usb3_ep->num)) {
2230 		usb3_write(usb3, 0, USB3_PN_INT_ENA);
2231 		usb3_write(usb3, 0, USB3_PN_RAMMAP);
2232 		usb3_clear_bit(usb3, PN_CON_EN, USB3_PN_CON);
2233 	}
2234 	spin_unlock_irqrestore(&usb3->lock, flags);
2235 
2236 	return 0;
2237 }
2238 
2239 /*------- usb_ep_ops -----------------------------------------------------*/
2240 static int renesas_usb3_ep_enable(struct usb_ep *_ep,
2241 				  const struct usb_endpoint_descriptor *desc)
2242 {
2243 	struct renesas_usb3_ep *usb3_ep = usb_ep_to_usb3_ep(_ep);
2244 
2245 	return usb3_enable_pipe_n(usb3_ep, desc);
2246 }
2247 
2248 static int renesas_usb3_ep_disable(struct usb_ep *_ep)
2249 {
2250 	struct renesas_usb3_ep *usb3_ep = usb_ep_to_usb3_ep(_ep);
2251 	struct renesas_usb3_request *usb3_req;
2252 
2253 	do {
2254 		usb3_req = usb3_get_request(usb3_ep);
2255 		if (!usb3_req)
2256 			break;
2257 		usb3_dma_try_stop(usb3_ep, usb3_req);
2258 		usb3_request_done(usb3_ep, usb3_req, -ESHUTDOWN);
2259 	} while (1);
2260 
2261 	return usb3_disable_pipe_n(usb3_ep);
2262 }
2263 
2264 static struct usb_request *__renesas_usb3_ep_alloc_request(gfp_t gfp_flags)
2265 {
2266 	struct renesas_usb3_request *usb3_req;
2267 
2268 	usb3_req = kzalloc(sizeof(struct renesas_usb3_request), gfp_flags);
2269 	if (!usb3_req)
2270 		return NULL;
2271 
2272 	INIT_LIST_HEAD(&usb3_req->queue);
2273 
2274 	return &usb3_req->req;
2275 }
2276 
2277 static void __renesas_usb3_ep_free_request(struct usb_request *_req)
2278 {
2279 	struct renesas_usb3_request *usb3_req = usb_req_to_usb3_req(_req);
2280 
2281 	kfree(usb3_req);
2282 }
2283 
2284 static struct usb_request *renesas_usb3_ep_alloc_request(struct usb_ep *_ep,
2285 							 gfp_t gfp_flags)
2286 {
2287 	return __renesas_usb3_ep_alloc_request(gfp_flags);
2288 }
2289 
2290 static void renesas_usb3_ep_free_request(struct usb_ep *_ep,
2291 					 struct usb_request *_req)
2292 {
2293 	__renesas_usb3_ep_free_request(_req);
2294 }
2295 
2296 static int renesas_usb3_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
2297 {
2298 	struct renesas_usb3_ep *usb3_ep = usb_ep_to_usb3_ep(_ep);
2299 	struct renesas_usb3_request *usb3_req = usb_req_to_usb3_req(_req);
2300 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
2301 
2302 	dev_dbg(usb3_to_dev(usb3), "ep_dequeue: ep%2d, %u\n", usb3_ep->num,
2303 		_req->length);
2304 
2305 	usb3_dma_try_stop(usb3_ep, usb3_req);
2306 	usb3_request_done_pipen(usb3, usb3_ep, usb3_req, -ECONNRESET);
2307 
2308 	return 0;
2309 }
2310 
2311 static int renesas_usb3_ep_set_halt(struct usb_ep *_ep, int value)
2312 {
2313 	return usb3_set_halt(usb_ep_to_usb3_ep(_ep), !!value, false);
2314 }
2315 
2316 static int renesas_usb3_ep_set_wedge(struct usb_ep *_ep)
2317 {
2318 	struct renesas_usb3_ep *usb3_ep = usb_ep_to_usb3_ep(_ep);
2319 
2320 	usb3_ep->wedge = true;
2321 	return usb3_set_halt(usb3_ep, true, false);
2322 }
2323 
2324 static void renesas_usb3_ep_fifo_flush(struct usb_ep *_ep)
2325 {
2326 	struct renesas_usb3_ep *usb3_ep = usb_ep_to_usb3_ep(_ep);
2327 	struct renesas_usb3 *usb3 = usb3_ep_to_usb3(usb3_ep);
2328 	unsigned long flags;
2329 
2330 	if (usb3_ep->num) {
2331 		spin_lock_irqsave(&usb3->lock, flags);
2332 		if (!usb3_pn_change(usb3, usb3_ep->num)) {
2333 			usb3_pn_con_clear(usb3);
2334 			usb3_set_bit(usb3, PN_CON_EN, USB3_PN_CON);
2335 		}
2336 		spin_unlock_irqrestore(&usb3->lock, flags);
2337 	} else {
2338 		usb3_p0_con_clear_buffer(usb3);
2339 	}
2340 }
2341 
2342 static const struct usb_ep_ops renesas_usb3_ep_ops = {
2343 	.enable		= renesas_usb3_ep_enable,
2344 	.disable	= renesas_usb3_ep_disable,
2345 
2346 	.alloc_request	= renesas_usb3_ep_alloc_request,
2347 	.free_request	= renesas_usb3_ep_free_request,
2348 
2349 	.queue		= renesas_usb3_ep_queue,
2350 	.dequeue	= renesas_usb3_ep_dequeue,
2351 
2352 	.set_halt	= renesas_usb3_ep_set_halt,
2353 	.set_wedge	= renesas_usb3_ep_set_wedge,
2354 	.fifo_flush	= renesas_usb3_ep_fifo_flush,
2355 };
2356 
2357 /*------- usb_gadget_ops -------------------------------------------------*/
2358 static int renesas_usb3_start(struct usb_gadget *gadget,
2359 			      struct usb_gadget_driver *driver)
2360 {
2361 	struct renesas_usb3 *usb3;
2362 
2363 	if (!driver || driver->max_speed < USB_SPEED_FULL ||
2364 	    !driver->setup)
2365 		return -EINVAL;
2366 
2367 	usb3 = gadget_to_renesas_usb3(gadget);
2368 
2369 	if (usb3->is_rzv2m && usb3_is_a_device(usb3))
2370 		return -EBUSY;
2371 
2372 	/* hook up the driver */
2373 	usb3->driver = driver;
2374 
2375 	if (usb3->phy)
2376 		phy_init(usb3->phy);
2377 
2378 	pm_runtime_get_sync(usb3_to_dev(usb3));
2379 
2380 	/* Peripheral Reset */
2381 	if (usb3->is_rzv2m)
2382 		rzv2m_usb3drd_reset(usb3_to_dev(usb3)->parent, false);
2383 
2384 	renesas_usb3_init_controller(usb3);
2385 
2386 	return 0;
2387 }
2388 
2389 static int renesas_usb3_stop(struct usb_gadget *gadget)
2390 {
2391 	struct renesas_usb3 *usb3 = gadget_to_renesas_usb3(gadget);
2392 
2393 	usb3->softconnect = false;
2394 	usb3->gadget.speed = USB_SPEED_UNKNOWN;
2395 	usb3->driver = NULL;
2396 	if (usb3->is_rzv2m)
2397 		rzv2m_usb3drd_reset(usb3_to_dev(usb3)->parent, false);
2398 
2399 	renesas_usb3_stop_controller(usb3);
2400 	if (usb3->phy)
2401 		phy_exit(usb3->phy);
2402 
2403 	pm_runtime_put(usb3_to_dev(usb3));
2404 
2405 	return 0;
2406 }
2407 
2408 static int renesas_usb3_get_frame(struct usb_gadget *_gadget)
2409 {
2410 	return -EOPNOTSUPP;
2411 }
2412 
2413 static int renesas_usb3_pullup(struct usb_gadget *gadget, int is_on)
2414 {
2415 	struct renesas_usb3 *usb3 = gadget_to_renesas_usb3(gadget);
2416 
2417 	usb3->softconnect = !!is_on;
2418 
2419 	return 0;
2420 }
2421 
2422 static int renesas_usb3_set_selfpowered(struct usb_gadget *gadget, int is_self)
2423 {
2424 	gadget->is_selfpowered = !!is_self;
2425 
2426 	return 0;
2427 }
2428 
2429 static const struct usb_gadget_ops renesas_usb3_gadget_ops = {
2430 	.get_frame		= renesas_usb3_get_frame,
2431 	.udc_start		= renesas_usb3_start,
2432 	.udc_stop		= renesas_usb3_stop,
2433 	.pullup			= renesas_usb3_pullup,
2434 	.set_selfpowered	= renesas_usb3_set_selfpowered,
2435 };
2436 
2437 static enum usb_role renesas_usb3_role_switch_get(struct usb_role_switch *sw)
2438 {
2439 	struct renesas_usb3 *usb3 = usb_role_switch_get_drvdata(sw);
2440 	enum usb_role cur_role;
2441 
2442 	pm_runtime_get_sync(usb3_to_dev(usb3));
2443 	cur_role = usb3_is_host(usb3) ? USB_ROLE_HOST : USB_ROLE_DEVICE;
2444 	pm_runtime_put(usb3_to_dev(usb3));
2445 
2446 	return cur_role;
2447 }
2448 
2449 static void handle_ext_role_switch_states(struct device *dev,
2450 					    enum usb_role role)
2451 {
2452 	struct renesas_usb3 *usb3 = dev_get_drvdata(dev);
2453 	struct device *host = usb3->host_dev;
2454 	enum usb_role cur_role = renesas_usb3_role_switch_get(usb3->role_sw);
2455 
2456 	switch (role) {
2457 	case USB_ROLE_NONE:
2458 		usb3->connection_state = USB_ROLE_NONE;
2459 		if (!usb3->is_rzv2m && cur_role == USB_ROLE_HOST)
2460 			device_release_driver(host);
2461 		if (usb3->driver) {
2462 			if (usb3->is_rzv2m)
2463 				rzv2m_usb3drd_reset(dev->parent, false);
2464 			usb3_disconnect(usb3);
2465 		}
2466 		usb3_vbus_out(usb3, false);
2467 
2468 		if (usb3->is_rzv2m) {
2469 			rzv2m_usb3drd_reset(dev->parent, true);
2470 			device_release_driver(host);
2471 		}
2472 		break;
2473 	case USB_ROLE_DEVICE:
2474 		if (usb3->connection_state == USB_ROLE_NONE) {
2475 			usb3->connection_state = USB_ROLE_DEVICE;
2476 			usb3_set_mode(usb3, false);
2477 			if (usb3->driver) {
2478 				if (usb3->is_rzv2m)
2479 					renesas_usb3_init_controller(usb3);
2480 				usb3_connect(usb3);
2481 			}
2482 		} else if (cur_role == USB_ROLE_HOST)  {
2483 			device_release_driver(host);
2484 			usb3_set_mode(usb3, false);
2485 			if (usb3->driver)
2486 				usb3_connect(usb3);
2487 		}
2488 		usb3_vbus_out(usb3, false);
2489 		break;
2490 	case USB_ROLE_HOST:
2491 		if (usb3->connection_state == USB_ROLE_NONE) {
2492 			if (usb3->driver) {
2493 				if (usb3->is_rzv2m)
2494 					rzv2m_usb3drd_reset(dev->parent, false);
2495 				usb3_disconnect(usb3);
2496 			}
2497 
2498 			usb3->connection_state = USB_ROLE_HOST;
2499 			usb3_set_mode(usb3, true);
2500 			usb3_vbus_out(usb3, true);
2501 			if (device_attach(host) < 0)
2502 				dev_err(dev, "device_attach(host) failed\n");
2503 		} else if (cur_role == USB_ROLE_DEVICE) {
2504 			usb3_disconnect(usb3);
2505 			/* Must set the mode before device_attach of the host */
2506 			usb3_set_mode(usb3, true);
2507 			/* This device_attach() might sleep */
2508 			if (device_attach(host) < 0)
2509 				dev_err(dev, "device_attach(host) failed\n");
2510 		}
2511 		break;
2512 	default:
2513 		break;
2514 	}
2515 }
2516 
2517 static void handle_role_switch_states(struct device *dev,
2518 					    enum usb_role role)
2519 {
2520 	struct renesas_usb3 *usb3 = dev_get_drvdata(dev);
2521 	struct device *host = usb3->host_dev;
2522 	enum usb_role cur_role = renesas_usb3_role_switch_get(usb3->role_sw);
2523 
2524 	if (cur_role == USB_ROLE_HOST && role == USB_ROLE_DEVICE) {
2525 		device_release_driver(host);
2526 		usb3_set_mode(usb3, false);
2527 	} else if (cur_role == USB_ROLE_DEVICE && role == USB_ROLE_HOST) {
2528 		/* Must set the mode before device_attach of the host */
2529 		usb3_set_mode(usb3, true);
2530 		/* This device_attach() might sleep */
2531 		if (device_attach(host) < 0)
2532 			dev_err(dev, "device_attach(host) failed\n");
2533 	}
2534 }
2535 
2536 static int renesas_usb3_role_switch_set(struct usb_role_switch *sw,
2537 					enum usb_role role)
2538 {
2539 	struct renesas_usb3 *usb3 = usb_role_switch_get_drvdata(sw);
2540 
2541 	pm_runtime_get_sync(usb3_to_dev(usb3));
2542 
2543 	if (usb3->role_sw_by_connector)
2544 		handle_ext_role_switch_states(usb3_to_dev(usb3), role);
2545 	else
2546 		handle_role_switch_states(usb3_to_dev(usb3), role);
2547 
2548 	pm_runtime_put(usb3_to_dev(usb3));
2549 
2550 	return 0;
2551 }
2552 
2553 static ssize_t role_store(struct device *dev, struct device_attribute *attr,
2554 			  const char *buf, size_t count)
2555 {
2556 	struct renesas_usb3 *usb3 = dev_get_drvdata(dev);
2557 	bool new_mode_is_host;
2558 
2559 	if (!usb3->driver)
2560 		return -ENODEV;
2561 
2562 	if (usb3->forced_b_device)
2563 		return -EBUSY;
2564 
2565 	if (sysfs_streq(buf, "host"))
2566 		new_mode_is_host = true;
2567 	else if (sysfs_streq(buf, "peripheral"))
2568 		new_mode_is_host = false;
2569 	else
2570 		return -EINVAL;
2571 
2572 	if (new_mode_is_host == usb3_is_host(usb3))
2573 		return -EINVAL;
2574 
2575 	usb3_mode_config(usb3, new_mode_is_host, usb3_is_a_device(usb3));
2576 
2577 	return count;
2578 }
2579 
2580 static ssize_t role_show(struct device *dev, struct device_attribute *attr,
2581 			 char *buf)
2582 {
2583 	struct renesas_usb3 *usb3 = dev_get_drvdata(dev);
2584 
2585 	if (!usb3->driver)
2586 		return -ENODEV;
2587 
2588 	return sprintf(buf, "%s\n", usb3_is_host(usb3) ? "host" : "peripheral");
2589 }
2590 static DEVICE_ATTR_RW(role);
2591 
2592 static int renesas_usb3_b_device_show(struct seq_file *s, void *unused)
2593 {
2594 	struct renesas_usb3 *usb3 = s->private;
2595 
2596 	seq_printf(s, "%d\n", usb3->forced_b_device);
2597 
2598 	return 0;
2599 }
2600 
2601 static int renesas_usb3_b_device_open(struct inode *inode, struct file *file)
2602 {
2603 	return single_open(file, renesas_usb3_b_device_show, inode->i_private);
2604 }
2605 
2606 static ssize_t renesas_usb3_b_device_write(struct file *file,
2607 					   const char __user *ubuf,
2608 					   size_t count, loff_t *ppos)
2609 {
2610 	struct seq_file *s = file->private_data;
2611 	struct renesas_usb3 *usb3 = s->private;
2612 	char buf[32];
2613 
2614 	if (!usb3->driver)
2615 		return -ENODEV;
2616 
2617 	if (copy_from_user(&buf, ubuf, min_t(size_t, sizeof(buf) - 1, count)))
2618 		return -EFAULT;
2619 
2620 	usb3->start_to_connect = false;
2621 	if (usb3->workaround_for_vbus && usb3->forced_b_device &&
2622 	    !strncmp(buf, "2", 1))
2623 		usb3->start_to_connect = true;
2624 	else if (!strncmp(buf, "1", 1))
2625 		usb3->forced_b_device = true;
2626 	else
2627 		usb3->forced_b_device = false;
2628 
2629 	if (usb3->workaround_for_vbus)
2630 		usb3_disconnect(usb3);
2631 
2632 	/* Let this driver call usb3_connect() if needed */
2633 	usb3_check_id(usb3);
2634 
2635 	return count;
2636 }
2637 
2638 static const struct file_operations renesas_usb3_b_device_fops = {
2639 	.open = renesas_usb3_b_device_open,
2640 	.write = renesas_usb3_b_device_write,
2641 	.read = seq_read,
2642 	.llseek = seq_lseek,
2643 	.release = single_release,
2644 };
2645 
2646 static void renesas_usb3_debugfs_init(struct renesas_usb3 *usb3,
2647 				      struct device *dev)
2648 {
2649 	usb3->dentry = debugfs_create_dir(dev_name(dev), usb_debug_root);
2650 
2651 	debugfs_create_file("b_device", 0644, usb3->dentry, usb3,
2652 			    &renesas_usb3_b_device_fops);
2653 }
2654 
2655 /*------- platform_driver ------------------------------------------------*/
2656 static int renesas_usb3_remove(struct platform_device *pdev)
2657 {
2658 	struct renesas_usb3 *usb3 = platform_get_drvdata(pdev);
2659 
2660 	debugfs_remove_recursive(usb3->dentry);
2661 	device_remove_file(&pdev->dev, &dev_attr_role);
2662 
2663 	cancel_work_sync(&usb3->role_work);
2664 	usb_role_switch_unregister(usb3->role_sw);
2665 
2666 	usb_del_gadget_udc(&usb3->gadget);
2667 	reset_control_assert(usb3->usbp_rstc);
2668 	renesas_usb3_dma_free_prd(usb3, &pdev->dev);
2669 
2670 	__renesas_usb3_ep_free_request(usb3->ep0_req);
2671 	pm_runtime_disable(&pdev->dev);
2672 
2673 	return 0;
2674 }
2675 
2676 static int renesas_usb3_init_ep(struct renesas_usb3 *usb3, struct device *dev,
2677 				const struct renesas_usb3_priv *priv)
2678 {
2679 	struct renesas_usb3_ep *usb3_ep;
2680 	int i;
2681 
2682 	/* calculate num_usb3_eps from renesas_usb3_priv */
2683 	usb3->num_usb3_eps = priv->ramsize_per_ramif * priv->num_ramif * 2 /
2684 			     priv->ramsize_per_pipe + 1;
2685 
2686 	if (usb3->num_usb3_eps > USB3_MAX_NUM_PIPES(usb3))
2687 		usb3->num_usb3_eps = USB3_MAX_NUM_PIPES(usb3);
2688 
2689 	usb3->usb3_ep = devm_kcalloc(dev,
2690 				     usb3->num_usb3_eps, sizeof(*usb3_ep),
2691 				     GFP_KERNEL);
2692 	if (!usb3->usb3_ep)
2693 		return -ENOMEM;
2694 
2695 	dev_dbg(dev, "%s: num_usb3_eps = %d\n", __func__, usb3->num_usb3_eps);
2696 	/*
2697 	 * This driver prepares pipes as follows:
2698 	 *  - odd pipes = IN pipe
2699 	 *  - even pipes = OUT pipe (except pipe 0)
2700 	 */
2701 	usb3_for_each_ep(usb3_ep, usb3, i) {
2702 		snprintf(usb3_ep->ep_name, sizeof(usb3_ep->ep_name), "ep%d", i);
2703 		usb3_ep->usb3 = usb3;
2704 		usb3_ep->num = i;
2705 		usb3_ep->ep.name = usb3_ep->ep_name;
2706 		usb3_ep->ep.ops = &renesas_usb3_ep_ops;
2707 		INIT_LIST_HEAD(&usb3_ep->queue);
2708 		INIT_LIST_HEAD(&usb3_ep->ep.ep_list);
2709 		if (!i) {
2710 			/* for control pipe */
2711 			usb3->gadget.ep0 = &usb3_ep->ep;
2712 			usb_ep_set_maxpacket_limit(&usb3_ep->ep,
2713 						USB3_EP0_SS_MAX_PACKET_SIZE);
2714 			usb3_ep->ep.caps.type_control = true;
2715 			usb3_ep->ep.caps.dir_in = true;
2716 			usb3_ep->ep.caps.dir_out = true;
2717 			continue;
2718 		}
2719 
2720 		/* for bulk or interrupt pipe */
2721 		usb_ep_set_maxpacket_limit(&usb3_ep->ep, ~0);
2722 		list_add_tail(&usb3_ep->ep.ep_list, &usb3->gadget.ep_list);
2723 		usb3_ep->ep.caps.type_bulk = true;
2724 		usb3_ep->ep.caps.type_int = true;
2725 		if (i & 1)
2726 			usb3_ep->ep.caps.dir_in = true;
2727 		else
2728 			usb3_ep->ep.caps.dir_out = true;
2729 	}
2730 
2731 	return 0;
2732 }
2733 
2734 static void renesas_usb3_init_ram(struct renesas_usb3 *usb3, struct device *dev,
2735 				  const struct renesas_usb3_priv *priv)
2736 {
2737 	struct renesas_usb3_ep *usb3_ep;
2738 	int i;
2739 	u32 ramif[2], basead[2];	/* index 0 = for IN pipes */
2740 	u32 *cur_ramif, *cur_basead;
2741 	u32 val;
2742 
2743 	memset(ramif, 0, sizeof(ramif));
2744 	memset(basead, 0, sizeof(basead));
2745 
2746 	/*
2747 	 * This driver prepares pipes as follows:
2748 	 *  - all pipes = the same size as "ramsize_per_pipe"
2749 	 * Please refer to the "Method of Specifying RAM Mapping"
2750 	 */
2751 	usb3_for_each_ep(usb3_ep, usb3, i) {
2752 		if (!i)
2753 			continue;	/* out of scope if ep num = 0 */
2754 		if (usb3_ep->ep.caps.dir_in) {
2755 			cur_ramif = &ramif[0];
2756 			cur_basead = &basead[0];
2757 		} else {
2758 			cur_ramif = &ramif[1];
2759 			cur_basead = &basead[1];
2760 		}
2761 
2762 		if (*cur_basead > priv->ramsize_per_ramif)
2763 			continue;	/* out of memory for IN or OUT pipe */
2764 
2765 		/* calculate rammap_val */
2766 		val = PN_RAMMAP_RAMIF(*cur_ramif);
2767 		val |= usb3_calc_ramarea(priv->ramsize_per_pipe);
2768 		val |= PN_RAMMAP_BASEAD(*cur_basead);
2769 		usb3_ep->rammap_val = val;
2770 
2771 		dev_dbg(dev, "ep%2d: val = %08x, ramif = %d, base = %x\n",
2772 			i, val, *cur_ramif, *cur_basead);
2773 
2774 		/* update current ramif */
2775 		if (*cur_ramif + 1 == priv->num_ramif) {
2776 			*cur_ramif = 0;
2777 			*cur_basead += priv->ramsize_per_pipe;
2778 		} else {
2779 			(*cur_ramif)++;
2780 		}
2781 	}
2782 }
2783 
2784 static const struct renesas_usb3_priv renesas_usb3_priv_gen3 = {
2785 	.ramsize_per_ramif = SZ_16K,
2786 	.num_ramif = 4,
2787 	.ramsize_per_pipe = SZ_4K,
2788 };
2789 
2790 static const struct renesas_usb3_priv renesas_usb3_priv_r8a77990 = {
2791 	.ramsize_per_ramif = SZ_16K,
2792 	.num_ramif = 4,
2793 	.ramsize_per_pipe = SZ_4K,
2794 	.workaround_for_vbus = true,
2795 };
2796 
2797 static const struct renesas_usb3_priv renesas_usb3_priv_rzv2m = {
2798 	.ramsize_per_ramif = SZ_16K,
2799 	.num_ramif = 1,
2800 	.ramsize_per_pipe = SZ_4K,
2801 	.is_rzv2m = true,
2802 };
2803 
2804 static const struct of_device_id usb3_of_match[] = {
2805 	{
2806 		.compatible = "renesas,r8a774c0-usb3-peri",
2807 		.data = &renesas_usb3_priv_r8a77990,
2808 	}, {
2809 		.compatible = "renesas,r8a7795-usb3-peri",
2810 		.data = &renesas_usb3_priv_gen3,
2811 	}, {
2812 		.compatible = "renesas,r8a77990-usb3-peri",
2813 		.data = &renesas_usb3_priv_r8a77990,
2814 	}, {
2815 		.compatible = "renesas,rzv2m-usb3-peri",
2816 		.data = &renesas_usb3_priv_rzv2m,
2817 	}, {
2818 		.compatible = "renesas,rcar-gen3-usb3-peri",
2819 		.data = &renesas_usb3_priv_gen3,
2820 	},
2821 	{ },
2822 };
2823 MODULE_DEVICE_TABLE(of, usb3_of_match);
2824 
2825 static const unsigned int renesas_usb3_cable[] = {
2826 	EXTCON_USB,
2827 	EXTCON_USB_HOST,
2828 	EXTCON_NONE,
2829 };
2830 
2831 static struct usb_role_switch_desc renesas_usb3_role_switch_desc = {
2832 	.set = renesas_usb3_role_switch_set,
2833 	.get = renesas_usb3_role_switch_get,
2834 	.allow_userspace_control = true,
2835 };
2836 
2837 static int renesas_usb3_probe(struct platform_device *pdev)
2838 {
2839 	struct renesas_usb3 *usb3;
2840 	int irq, ret;
2841 	const struct renesas_usb3_priv *priv;
2842 
2843 	priv = of_device_get_match_data(&pdev->dev);
2844 
2845 	irq = platform_get_irq(pdev, 0);
2846 	if (irq < 0)
2847 		return irq;
2848 
2849 	usb3 = devm_kzalloc(&pdev->dev, sizeof(*usb3), GFP_KERNEL);
2850 	if (!usb3)
2851 		return -ENOMEM;
2852 
2853 	usb3->is_rzv2m = priv->is_rzv2m;
2854 
2855 	usb3->reg = devm_platform_ioremap_resource(pdev, 0);
2856 	if (IS_ERR(usb3->reg))
2857 		return PTR_ERR(usb3->reg);
2858 
2859 	platform_set_drvdata(pdev, usb3);
2860 	spin_lock_init(&usb3->lock);
2861 
2862 	usb3->gadget.ops = &renesas_usb3_gadget_ops;
2863 	usb3->gadget.name = udc_name;
2864 	usb3->gadget.max_speed = USB_SPEED_SUPER;
2865 	INIT_LIST_HEAD(&usb3->gadget.ep_list);
2866 	ret = renesas_usb3_init_ep(usb3, &pdev->dev, priv);
2867 	if (ret < 0)
2868 		return ret;
2869 	renesas_usb3_init_ram(usb3, &pdev->dev, priv);
2870 
2871 	ret = devm_request_irq(&pdev->dev, irq, renesas_usb3_irq, 0,
2872 			       dev_name(&pdev->dev), usb3);
2873 	if (ret < 0)
2874 		return ret;
2875 
2876 	if (usb3->is_rzv2m) {
2877 		struct rzv2m_usb3drd *ddata = dev_get_drvdata(pdev->dev.parent);
2878 
2879 		usb3->drd_reg = ddata->reg;
2880 		ret = devm_request_irq(ddata->dev, ddata->drd_irq,
2881 				       renesas_usb3_otg_irq, 0,
2882 				       dev_name(ddata->dev), usb3);
2883 		if (ret < 0)
2884 			return ret;
2885 	}
2886 
2887 	INIT_WORK(&usb3->extcon_work, renesas_usb3_extcon_work);
2888 	usb3->extcon = devm_extcon_dev_allocate(&pdev->dev, renesas_usb3_cable);
2889 	if (IS_ERR(usb3->extcon))
2890 		return PTR_ERR(usb3->extcon);
2891 
2892 	ret = devm_extcon_dev_register(&pdev->dev, usb3->extcon);
2893 	if (ret < 0) {
2894 		dev_err(&pdev->dev, "Failed to register extcon\n");
2895 		return ret;
2896 	}
2897 
2898 	/* for ep0 handling */
2899 	usb3->ep0_req = __renesas_usb3_ep_alloc_request(GFP_KERNEL);
2900 	if (!usb3->ep0_req)
2901 		return -ENOMEM;
2902 
2903 	ret = renesas_usb3_dma_alloc_prd(usb3, &pdev->dev);
2904 	if (ret < 0)
2905 		goto err_alloc_prd;
2906 
2907 	/*
2908 	 * This is optional. So, if this driver cannot get a phy,
2909 	 * this driver will not handle a phy anymore.
2910 	 */
2911 	usb3->phy = devm_phy_optional_get(&pdev->dev, "usb");
2912 	if (IS_ERR(usb3->phy)) {
2913 		ret = PTR_ERR(usb3->phy);
2914 		goto err_add_udc;
2915 	}
2916 
2917 	usb3->usbp_rstc = devm_reset_control_get_optional_shared(&pdev->dev,
2918 								 NULL);
2919 	if (IS_ERR(usb3->usbp_rstc)) {
2920 		ret = PTR_ERR(usb3->usbp_rstc);
2921 		goto err_add_udc;
2922 	}
2923 
2924 	reset_control_deassert(usb3->usbp_rstc);
2925 
2926 	pm_runtime_enable(&pdev->dev);
2927 	ret = usb_add_gadget_udc(&pdev->dev, &usb3->gadget);
2928 	if (ret < 0)
2929 		goto err_reset;
2930 
2931 	ret = device_create_file(&pdev->dev, &dev_attr_role);
2932 	if (ret < 0)
2933 		goto err_dev_create;
2934 
2935 	if (device_property_read_bool(&pdev->dev, "usb-role-switch")) {
2936 		usb3->role_sw_by_connector = true;
2937 		renesas_usb3_role_switch_desc.fwnode = dev_fwnode(&pdev->dev);
2938 	}
2939 
2940 	renesas_usb3_role_switch_desc.driver_data = usb3;
2941 
2942 	INIT_WORK(&usb3->role_work, renesas_usb3_role_work);
2943 	usb3->role_sw = usb_role_switch_register(&pdev->dev,
2944 					&renesas_usb3_role_switch_desc);
2945 	if (!IS_ERR(usb3->role_sw)) {
2946 		usb3->host_dev = usb_of_get_companion_dev(&pdev->dev);
2947 		if (!usb3->host_dev) {
2948 			/* If not found, this driver will not use a role sw */
2949 			usb_role_switch_unregister(usb3->role_sw);
2950 			usb3->role_sw = NULL;
2951 		}
2952 	} else {
2953 		usb3->role_sw = NULL;
2954 	}
2955 
2956 	usb3->workaround_for_vbus = priv->workaround_for_vbus;
2957 
2958 	renesas_usb3_debugfs_init(usb3, &pdev->dev);
2959 
2960 	dev_info(&pdev->dev, "probed%s\n", usb3->phy ? " with phy" : "");
2961 
2962 	return 0;
2963 
2964 err_dev_create:
2965 	usb_del_gadget_udc(&usb3->gadget);
2966 
2967 err_reset:
2968 	reset_control_assert(usb3->usbp_rstc);
2969 
2970 err_add_udc:
2971 	renesas_usb3_dma_free_prd(usb3, &pdev->dev);
2972 
2973 err_alloc_prd:
2974 	__renesas_usb3_ep_free_request(usb3->ep0_req);
2975 
2976 	return ret;
2977 }
2978 
2979 #ifdef CONFIG_PM_SLEEP
2980 static int renesas_usb3_suspend(struct device *dev)
2981 {
2982 	struct renesas_usb3 *usb3 = dev_get_drvdata(dev);
2983 
2984 	/* Not started */
2985 	if (!usb3->driver)
2986 		return 0;
2987 
2988 	renesas_usb3_stop_controller(usb3);
2989 	if (usb3->phy)
2990 		phy_exit(usb3->phy);
2991 	pm_runtime_put(dev);
2992 
2993 	return 0;
2994 }
2995 
2996 static int renesas_usb3_resume(struct device *dev)
2997 {
2998 	struct renesas_usb3 *usb3 = dev_get_drvdata(dev);
2999 
3000 	/* Not started */
3001 	if (!usb3->driver)
3002 		return 0;
3003 
3004 	if (usb3->phy)
3005 		phy_init(usb3->phy);
3006 	pm_runtime_get_sync(dev);
3007 	renesas_usb3_init_controller(usb3);
3008 
3009 	return 0;
3010 }
3011 #endif
3012 
3013 static SIMPLE_DEV_PM_OPS(renesas_usb3_pm_ops, renesas_usb3_suspend,
3014 			renesas_usb3_resume);
3015 
3016 static struct platform_driver renesas_usb3_driver = {
3017 	.probe		= renesas_usb3_probe,
3018 	.remove		= renesas_usb3_remove,
3019 	.driver		= {
3020 		.name =	udc_name,
3021 		.pm		= &renesas_usb3_pm_ops,
3022 		.of_match_table = usb3_of_match,
3023 	},
3024 };
3025 module_platform_driver(renesas_usb3_driver);
3026 
3027 MODULE_DESCRIPTION("Renesas USB3.0 Peripheral driver");
3028 MODULE_LICENSE("GPL v2");
3029 MODULE_AUTHOR("Yoshihiro Shimoda <yoshihiro.shimoda.uh@renesas.com>");
3030 MODULE_ALIAS("platform:renesas_usb3");
3031