1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * NVIDIA Tegra XUSB device mode controller
4  *
5  * Copyright (c) 2013-2022, NVIDIA CORPORATION.  All rights reserved.
6  * Copyright (c) 2015, Google Inc.
7  */
8 
9 #include <linux/clk.h>
10 #include <linux/completion.h>
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmapool.h>
14 #include <linux/interrupt.h>
15 #include <linux/iopoll.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/of.h>
19 #include <linux/phy/phy.h>
20 #include <linux/phy/tegra/xusb.h>
21 #include <linux/pm_domain.h>
22 #include <linux/platform_device.h>
23 #include <linux/pm_runtime.h>
24 #include <linux/regulator/consumer.h>
25 #include <linux/reset.h>
26 #include <linux/usb/ch9.h>
27 #include <linux/usb/gadget.h>
28 #include <linux/usb/otg.h>
29 #include <linux/usb/role.h>
30 #include <linux/usb/phy.h>
31 #include <linux/workqueue.h>
32 
33 /* XUSB_DEV registers */
34 #define DB 0x004
35 #define  DB_TARGET_MASK GENMASK(15, 8)
36 #define  DB_TARGET(x) (((x) << 8) & DB_TARGET_MASK)
37 #define  DB_STREAMID_MASK GENMASK(31, 16)
38 #define  DB_STREAMID(x) (((x) << 16) & DB_STREAMID_MASK)
39 #define ERSTSZ 0x008
40 #define  ERSTSZ_ERSTXSZ_SHIFT(x) ((x) * 16)
41 #define  ERSTSZ_ERSTXSZ_MASK GENMASK(15, 0)
42 #define ERSTXBALO(x) (0x010 + 8 * (x))
43 #define ERSTXBAHI(x) (0x014 + 8 * (x))
44 #define ERDPLO 0x020
45 #define  ERDPLO_EHB BIT(3)
46 #define ERDPHI 0x024
47 #define EREPLO 0x028
48 #define  EREPLO_ECS BIT(0)
49 #define  EREPLO_SEGI BIT(1)
50 #define EREPHI 0x02c
51 #define CTRL 0x030
52 #define  CTRL_RUN BIT(0)
53 #define  CTRL_LSE BIT(1)
54 #define  CTRL_IE BIT(4)
55 #define  CTRL_SMI_EVT BIT(5)
56 #define  CTRL_SMI_DSE BIT(6)
57 #define  CTRL_EWE BIT(7)
58 #define  CTRL_DEVADDR_MASK GENMASK(30, 24)
59 #define  CTRL_DEVADDR(x) (((x) << 24) & CTRL_DEVADDR_MASK)
60 #define  CTRL_ENABLE BIT(31)
61 #define ST 0x034
62 #define  ST_RC BIT(0)
63 #define  ST_IP BIT(4)
64 #define RT_IMOD	0x038
65 #define  RT_IMOD_IMODI_MASK GENMASK(15, 0)
66 #define  RT_IMOD_IMODI(x) ((x) & RT_IMOD_IMODI_MASK)
67 #define  RT_IMOD_IMODC_MASK GENMASK(31, 16)
68 #define  RT_IMOD_IMODC(x) (((x) << 16) & RT_IMOD_IMODC_MASK)
69 #define PORTSC 0x03c
70 #define  PORTSC_CCS BIT(0)
71 #define  PORTSC_PED BIT(1)
72 #define  PORTSC_PR BIT(4)
73 #define  PORTSC_PLS_SHIFT 5
74 #define  PORTSC_PLS_MASK GENMASK(8, 5)
75 #define  PORTSC_PLS_U0 0x0
76 #define  PORTSC_PLS_U2 0x2
77 #define  PORTSC_PLS_U3 0x3
78 #define  PORTSC_PLS_DISABLED 0x4
79 #define  PORTSC_PLS_RXDETECT 0x5
80 #define  PORTSC_PLS_INACTIVE 0x6
81 #define  PORTSC_PLS_RESUME 0xf
82 #define  PORTSC_PLS(x) (((x) << PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK)
83 #define  PORTSC_PS_SHIFT 10
84 #define  PORTSC_PS_MASK GENMASK(13, 10)
85 #define  PORTSC_PS_UNDEFINED 0x0
86 #define  PORTSC_PS_FS 0x1
87 #define  PORTSC_PS_LS 0x2
88 #define  PORTSC_PS_HS 0x3
89 #define  PORTSC_PS_SS 0x4
90 #define  PORTSC_LWS BIT(16)
91 #define  PORTSC_CSC BIT(17)
92 #define  PORTSC_WRC BIT(19)
93 #define  PORTSC_PRC BIT(21)
94 #define  PORTSC_PLC BIT(22)
95 #define  PORTSC_CEC BIT(23)
96 #define  PORTSC_WPR BIT(30)
97 #define  PORTSC_CHANGE_MASK (PORTSC_CSC | PORTSC_WRC | PORTSC_PRC | \
98 			     PORTSC_PLC | PORTSC_CEC)
99 #define ECPLO 0x040
100 #define ECPHI 0x044
101 #define MFINDEX 0x048
102 #define  MFINDEX_FRAME_SHIFT 3
103 #define  MFINDEX_FRAME_MASK GENMASK(13, 3)
104 #define PORTPM 0x04c
105 #define  PORTPM_L1S_MASK GENMASK(1, 0)
106 #define  PORTPM_L1S_DROP 0x0
107 #define  PORTPM_L1S_ACCEPT 0x1
108 #define  PORTPM_L1S_NYET 0x2
109 #define  PORTPM_L1S_STALL 0x3
110 #define  PORTPM_L1S(x) ((x) & PORTPM_L1S_MASK)
111 #define  PORTPM_RWE BIT(3)
112 #define  PORTPM_U2TIMEOUT_MASK GENMASK(15, 8)
113 #define  PORTPM_U1TIMEOUT_MASK GENMASK(23, 16)
114 #define  PORTPM_FLA BIT(24)
115 #define  PORTPM_VBA BIT(25)
116 #define  PORTPM_WOC BIT(26)
117 #define  PORTPM_WOD BIT(27)
118 #define  PORTPM_U1E BIT(28)
119 #define  PORTPM_U2E BIT(29)
120 #define  PORTPM_FRWE BIT(30)
121 #define  PORTPM_PNG_CYA BIT(31)
122 #define EP_HALT 0x050
123 #define EP_PAUSE 0x054
124 #define EP_RELOAD 0x058
125 #define EP_STCHG 0x05c
126 #define DEVNOTIF_LO 0x064
127 #define  DEVNOTIF_LO_TRIG BIT(0)
128 #define  DEVNOTIF_LO_TYPE_MASK GENMASK(7, 4)
129 #define  DEVNOTIF_LO_TYPE(x) (((x) << 4)  & DEVNOTIF_LO_TYPE_MASK)
130 #define  DEVNOTIF_LO_TYPE_FUNCTION_WAKE 0x1
131 #define DEVNOTIF_HI 0x068
132 #define PORTHALT 0x06c
133 #define  PORTHALT_HALT_LTSSM BIT(0)
134 #define  PORTHALT_HALT_REJECT BIT(1)
135 #define  PORTHALT_STCHG_REQ BIT(20)
136 #define  PORTHALT_STCHG_INTR_EN BIT(24)
137 #define PORT_TM	0x070
138 #define EP_THREAD_ACTIVE 0x074
139 #define EP_STOPPED 0x078
140 #define HSFSPI_COUNT0 0x100
141 #define HSFSPI_COUNT13 0x134
142 #define  HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK GENMASK(29, 0)
143 #define  HSFSPI_COUNT13_U2_RESUME_K_DURATION(x) ((x) & \
144 				HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK)
145 #define BLCG 0x840
146 #define SSPX_CORE_CNT0 0x610
147 #define  SSPX_CORE_CNT0_PING_TBURST_MASK GENMASK(7, 0)
148 #define  SSPX_CORE_CNT0_PING_TBURST(x) ((x) & SSPX_CORE_CNT0_PING_TBURST_MASK)
149 #define SSPX_CORE_CNT30 0x688
150 #define  SSPX_CORE_CNT30_LMPITP_TIMER_MASK GENMASK(19, 0)
151 #define  SSPX_CORE_CNT30_LMPITP_TIMER(x) ((x) & \
152 					SSPX_CORE_CNT30_LMPITP_TIMER_MASK)
153 #define SSPX_CORE_CNT32 0x690
154 #define  SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK GENMASK(7, 0)
155 #define  SSPX_CORE_CNT32_POLL_TBURST_MAX(x) ((x) & \
156 					SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK)
157 #define SSPX_CORE_CNT56 0x6fc
158 #define  SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK GENMASK(19, 0)
159 #define  SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(x) ((x) & \
160 				SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK)
161 #define SSPX_CORE_CNT57 0x700
162 #define  SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK GENMASK(19, 0)
163 #define  SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(x) ((x) & \
164 				SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK)
165 #define SSPX_CORE_CNT65 0x720
166 #define  SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK GENMASK(19, 0)
167 #define  SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(x) ((x) & \
168 				SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK)
169 #define SSPX_CORE_CNT66 0x724
170 #define  SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK GENMASK(19, 0)
171 #define  SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(x) ((x) & \
172 				SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK)
173 #define SSPX_CORE_CNT67 0x728
174 #define  SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK GENMASK(19, 0)
175 #define  SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(x) ((x) & \
176 				SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK)
177 #define SSPX_CORE_CNT72 0x73c
178 #define  SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK GENMASK(19, 0)
179 #define  SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(x) ((x) & \
180 				SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK)
181 #define SSPX_CORE_PADCTL4 0x750
182 #define  SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK GENMASK(19, 0)
183 #define  SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(x) ((x) & \
184 				SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK)
185 #define  BLCG_DFPCI BIT(0)
186 #define  BLCG_UFPCI BIT(1)
187 #define  BLCG_FE BIT(2)
188 #define  BLCG_COREPLL_PWRDN BIT(8)
189 #define  BLCG_IOPLL_0_PWRDN BIT(9)
190 #define  BLCG_IOPLL_1_PWRDN BIT(10)
191 #define  BLCG_IOPLL_2_PWRDN BIT(11)
192 #define  BLCG_ALL 0x1ff
193 #define CFG_DEV_SSPI_XFER 0x858
194 #define  CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK GENMASK(31, 0)
195 #define  CFG_DEV_SSPI_XFER_ACKTIMEOUT(x) ((x) & \
196 					CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK)
197 #define CFG_DEV_FE 0x85c
198 #define  CFG_DEV_FE_PORTREGSEL_MASK GENMASK(1, 0)
199 #define  CFG_DEV_FE_PORTREGSEL_SS_PI 1
200 #define  CFG_DEV_FE_PORTREGSEL_HSFS_PI 2
201 #define  CFG_DEV_FE_PORTREGSEL(x) ((x) & CFG_DEV_FE_PORTREGSEL_MASK)
202 #define  CFG_DEV_FE_INFINITE_SS_RETRY BIT(29)
203 
204 /* FPCI registers */
205 #define XUSB_DEV_CFG_1 0x004
206 #define  XUSB_DEV_CFG_1_IO_SPACE_EN BIT(0)
207 #define  XUSB_DEV_CFG_1_MEMORY_SPACE_EN BIT(1)
208 #define  XUSB_DEV_CFG_1_BUS_MASTER_EN BIT(2)
209 #define XUSB_DEV_CFG_4 0x010
210 #define  XUSB_DEV_CFG_4_BASE_ADDR_MASK GENMASK(31, 15)
211 #define XUSB_DEV_CFG_5 0x014
212 
213 /* IPFS registers */
214 #define XUSB_DEV_CONFIGURATION_0 0x180
215 #define  XUSB_DEV_CONFIGURATION_0_EN_FPCI BIT(0)
216 #define XUSB_DEV_INTR_MASK_0 0x188
217 #define  XUSB_DEV_INTR_MASK_0_IP_INT_MASK BIT(16)
218 
219 struct tegra_xudc_ep_context {
220 	__le32 info0;
221 	__le32 info1;
222 	__le32 deq_lo;
223 	__le32 deq_hi;
224 	__le32 tx_info;
225 	__le32 rsvd[11];
226 };
227 
228 #define EP_STATE_DISABLED 0
229 #define EP_STATE_RUNNING 1
230 #define EP_STATE_HALTED 2
231 #define EP_STATE_STOPPED 3
232 #define EP_STATE_ERROR 4
233 
234 #define EP_TYPE_INVALID 0
235 #define EP_TYPE_ISOCH_OUT 1
236 #define EP_TYPE_BULK_OUT 2
237 #define EP_TYPE_INTERRUPT_OUT 3
238 #define EP_TYPE_CONTROL 4
239 #define EP_TYPE_ISCOH_IN 5
240 #define EP_TYPE_BULK_IN 6
241 #define EP_TYPE_INTERRUPT_IN 7
242 
243 #define BUILD_EP_CONTEXT_RW(name, member, shift, mask)			\
244 static inline u32 ep_ctx_read_##name(struct tegra_xudc_ep_context *ctx)	\
245 {									\
246 	return (le32_to_cpu(ctx->member) >> (shift)) & (mask);		\
247 }									\
248 static inline void							\
249 ep_ctx_write_##name(struct tegra_xudc_ep_context *ctx, u32 val)		\
250 {									\
251 	u32 tmp;							\
252 									\
253 	tmp = le32_to_cpu(ctx->member) & ~((mask) << (shift));		\
254 	tmp |= (val & (mask)) << (shift);				\
255 	ctx->member = cpu_to_le32(tmp);					\
256 }
257 
258 BUILD_EP_CONTEXT_RW(state, info0, 0, 0x7)
259 BUILD_EP_CONTEXT_RW(mult, info0, 8, 0x3)
260 BUILD_EP_CONTEXT_RW(max_pstreams, info0, 10, 0x1f)
261 BUILD_EP_CONTEXT_RW(lsa, info0, 15, 0x1)
262 BUILD_EP_CONTEXT_RW(interval, info0, 16, 0xff)
263 BUILD_EP_CONTEXT_RW(cerr, info1, 1, 0x3)
264 BUILD_EP_CONTEXT_RW(type, info1, 3, 0x7)
265 BUILD_EP_CONTEXT_RW(hid, info1, 7, 0x1)
266 BUILD_EP_CONTEXT_RW(max_burst_size, info1, 8, 0xff)
267 BUILD_EP_CONTEXT_RW(max_packet_size, info1, 16, 0xffff)
268 BUILD_EP_CONTEXT_RW(dcs, deq_lo, 0, 0x1)
269 BUILD_EP_CONTEXT_RW(deq_lo, deq_lo, 4, 0xfffffff)
270 BUILD_EP_CONTEXT_RW(deq_hi, deq_hi, 0, 0xffffffff)
271 BUILD_EP_CONTEXT_RW(avg_trb_len, tx_info, 0, 0xffff)
272 BUILD_EP_CONTEXT_RW(max_esit_payload, tx_info, 16, 0xffff)
273 BUILD_EP_CONTEXT_RW(edtla, rsvd[0], 0, 0xffffff)
274 BUILD_EP_CONTEXT_RW(rsvd, rsvd[0], 24, 0x1)
275 BUILD_EP_CONTEXT_RW(partial_td, rsvd[0], 25, 0x1)
276 BUILD_EP_CONTEXT_RW(splitxstate, rsvd[0], 26, 0x1)
277 BUILD_EP_CONTEXT_RW(seq_num, rsvd[0], 27, 0x1f)
278 BUILD_EP_CONTEXT_RW(cerrcnt, rsvd[1], 18, 0x3)
279 BUILD_EP_CONTEXT_RW(data_offset, rsvd[2], 0, 0x1ffff)
280 BUILD_EP_CONTEXT_RW(numtrbs, rsvd[2], 22, 0x1f)
281 BUILD_EP_CONTEXT_RW(devaddr, rsvd[6], 0, 0x7f)
282 
283 static inline u64 ep_ctx_read_deq_ptr(struct tegra_xudc_ep_context *ctx)
284 {
285 	return ((u64)ep_ctx_read_deq_hi(ctx) << 32) |
286 		(ep_ctx_read_deq_lo(ctx) << 4);
287 }
288 
289 static inline void
290 ep_ctx_write_deq_ptr(struct tegra_xudc_ep_context *ctx, u64 addr)
291 {
292 	ep_ctx_write_deq_lo(ctx, lower_32_bits(addr) >> 4);
293 	ep_ctx_write_deq_hi(ctx, upper_32_bits(addr));
294 }
295 
296 struct tegra_xudc_trb {
297 	__le32 data_lo;
298 	__le32 data_hi;
299 	__le32 status;
300 	__le32 control;
301 };
302 
303 #define TRB_TYPE_RSVD 0
304 #define TRB_TYPE_NORMAL 1
305 #define TRB_TYPE_SETUP_STAGE 2
306 #define TRB_TYPE_DATA_STAGE 3
307 #define TRB_TYPE_STATUS_STAGE 4
308 #define TRB_TYPE_ISOCH 5
309 #define TRB_TYPE_LINK 6
310 #define TRB_TYPE_TRANSFER_EVENT 32
311 #define TRB_TYPE_PORT_STATUS_CHANGE_EVENT 34
312 #define TRB_TYPE_STREAM 48
313 #define TRB_TYPE_SETUP_PACKET_EVENT 63
314 
315 #define TRB_CMPL_CODE_INVALID 0
316 #define TRB_CMPL_CODE_SUCCESS 1
317 #define TRB_CMPL_CODE_DATA_BUFFER_ERR 2
318 #define TRB_CMPL_CODE_BABBLE_DETECTED_ERR 3
319 #define TRB_CMPL_CODE_USB_TRANS_ERR 4
320 #define TRB_CMPL_CODE_TRB_ERR 5
321 #define TRB_CMPL_CODE_STALL 6
322 #define TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR 10
323 #define TRB_CMPL_CODE_SHORT_PACKET 13
324 #define TRB_CMPL_CODE_RING_UNDERRUN 14
325 #define TRB_CMPL_CODE_RING_OVERRUN 15
326 #define TRB_CMPL_CODE_EVENT_RING_FULL_ERR 21
327 #define TRB_CMPL_CODE_STOPPED 26
328 #define TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN 31
329 #define TRB_CMPL_CODE_STREAM_NUMP_ERROR 219
330 #define TRB_CMPL_CODE_PRIME_PIPE_RECEIVED 220
331 #define TRB_CMPL_CODE_HOST_REJECTED 221
332 #define TRB_CMPL_CODE_CTRL_DIR_ERR 222
333 #define TRB_CMPL_CODE_CTRL_SEQNUM_ERR 223
334 
335 #define BUILD_TRB_RW(name, member, shift, mask)				\
336 static inline u32 trb_read_##name(struct tegra_xudc_trb *trb)		\
337 {									\
338 	return (le32_to_cpu(trb->member) >> (shift)) & (mask);		\
339 }									\
340 static inline void							\
341 trb_write_##name(struct tegra_xudc_trb *trb, u32 val)			\
342 {									\
343 	u32 tmp;							\
344 									\
345 	tmp = le32_to_cpu(trb->member) & ~((mask) << (shift));		\
346 	tmp |= (val & (mask)) << (shift);				\
347 	trb->member = cpu_to_le32(tmp);					\
348 }
349 
350 BUILD_TRB_RW(data_lo, data_lo, 0, 0xffffffff)
351 BUILD_TRB_RW(data_hi, data_hi, 0, 0xffffffff)
352 BUILD_TRB_RW(seq_num, status, 0, 0xffff)
353 BUILD_TRB_RW(transfer_len, status, 0, 0xffffff)
354 BUILD_TRB_RW(td_size, status, 17, 0x1f)
355 BUILD_TRB_RW(cmpl_code, status, 24, 0xff)
356 BUILD_TRB_RW(cycle, control, 0, 0x1)
357 BUILD_TRB_RW(toggle_cycle, control, 1, 0x1)
358 BUILD_TRB_RW(isp, control, 2, 0x1)
359 BUILD_TRB_RW(chain, control, 4, 0x1)
360 BUILD_TRB_RW(ioc, control, 5, 0x1)
361 BUILD_TRB_RW(type, control, 10, 0x3f)
362 BUILD_TRB_RW(stream_id, control, 16, 0xffff)
363 BUILD_TRB_RW(endpoint_id, control, 16, 0x1f)
364 BUILD_TRB_RW(tlbpc, control, 16, 0xf)
365 BUILD_TRB_RW(data_stage_dir, control, 16, 0x1)
366 BUILD_TRB_RW(frame_id, control, 20, 0x7ff)
367 BUILD_TRB_RW(sia, control, 31, 0x1)
368 
369 static inline u64 trb_read_data_ptr(struct tegra_xudc_trb *trb)
370 {
371 	return ((u64)trb_read_data_hi(trb) << 32) |
372 		trb_read_data_lo(trb);
373 }
374 
375 static inline void trb_write_data_ptr(struct tegra_xudc_trb *trb, u64 addr)
376 {
377 	trb_write_data_lo(trb, lower_32_bits(addr));
378 	trb_write_data_hi(trb, upper_32_bits(addr));
379 }
380 
381 struct tegra_xudc_request {
382 	struct usb_request usb_req;
383 
384 	size_t buf_queued;
385 	unsigned int trbs_queued;
386 	unsigned int trbs_needed;
387 	bool need_zlp;
388 
389 	struct tegra_xudc_trb *first_trb;
390 	struct tegra_xudc_trb *last_trb;
391 
392 	struct list_head list;
393 };
394 
395 struct tegra_xudc_ep {
396 	struct tegra_xudc *xudc;
397 	struct usb_ep usb_ep;
398 	unsigned int index;
399 	char name[8];
400 
401 	struct tegra_xudc_ep_context *context;
402 
403 #define XUDC_TRANSFER_RING_SIZE 64
404 	struct tegra_xudc_trb *transfer_ring;
405 	dma_addr_t transfer_ring_phys;
406 
407 	unsigned int enq_ptr;
408 	unsigned int deq_ptr;
409 	bool pcs;
410 	bool ring_full;
411 	bool stream_rejected;
412 
413 	struct list_head queue;
414 	const struct usb_endpoint_descriptor *desc;
415 	const struct usb_ss_ep_comp_descriptor *comp_desc;
416 };
417 
418 struct tegra_xudc_sel_timing {
419 	__u8 u1sel;
420 	__u8 u1pel;
421 	__le16 u2sel;
422 	__le16 u2pel;
423 };
424 
425 enum tegra_xudc_setup_state {
426 	WAIT_FOR_SETUP,
427 	DATA_STAGE_XFER,
428 	DATA_STAGE_RECV,
429 	STATUS_STAGE_XFER,
430 	STATUS_STAGE_RECV,
431 };
432 
433 struct tegra_xudc_setup_packet {
434 	struct usb_ctrlrequest ctrl_req;
435 	unsigned int seq_num;
436 };
437 
438 struct tegra_xudc_save_regs {
439 	u32 ctrl;
440 	u32 portpm;
441 };
442 
443 struct tegra_xudc {
444 	struct device *dev;
445 	const struct tegra_xudc_soc *soc;
446 	struct tegra_xusb_padctl *padctl;
447 
448 	spinlock_t lock;
449 
450 	struct usb_gadget gadget;
451 	struct usb_gadget_driver *driver;
452 
453 #define XUDC_NR_EVENT_RINGS 2
454 #define XUDC_EVENT_RING_SIZE 4096
455 	struct tegra_xudc_trb *event_ring[XUDC_NR_EVENT_RINGS];
456 	dma_addr_t event_ring_phys[XUDC_NR_EVENT_RINGS];
457 	unsigned int event_ring_index;
458 	unsigned int event_ring_deq_ptr;
459 	bool ccs;
460 
461 #define XUDC_NR_EPS 32
462 	struct tegra_xudc_ep ep[XUDC_NR_EPS];
463 	struct tegra_xudc_ep_context *ep_context;
464 	dma_addr_t ep_context_phys;
465 
466 	struct device *genpd_dev_device;
467 	struct device *genpd_dev_ss;
468 	struct device_link *genpd_dl_device;
469 	struct device_link *genpd_dl_ss;
470 
471 	struct dma_pool *transfer_ring_pool;
472 
473 	bool queued_setup_packet;
474 	struct tegra_xudc_setup_packet setup_packet;
475 	enum tegra_xudc_setup_state setup_state;
476 	u16 setup_seq_num;
477 
478 	u16 dev_addr;
479 	u16 isoch_delay;
480 	struct tegra_xudc_sel_timing sel_timing;
481 	u8 test_mode_pattern;
482 	u16 status_buf;
483 	struct tegra_xudc_request *ep0_req;
484 
485 	bool pullup;
486 
487 	unsigned int nr_enabled_eps;
488 	unsigned int nr_isoch_eps;
489 
490 	unsigned int device_state;
491 	unsigned int resume_state;
492 
493 	int irq;
494 
495 	void __iomem *base;
496 	resource_size_t phys_base;
497 	void __iomem *ipfs;
498 	void __iomem *fpci;
499 
500 	struct regulator_bulk_data *supplies;
501 
502 	struct clk_bulk_data *clks;
503 
504 	bool device_mode;
505 	struct work_struct usb_role_sw_work;
506 
507 	struct phy **usb3_phy;
508 	struct phy *curr_usb3_phy;
509 	struct phy **utmi_phy;
510 	struct phy *curr_utmi_phy;
511 
512 	struct tegra_xudc_save_regs saved_regs;
513 	bool suspended;
514 	bool powergated;
515 
516 	struct usb_phy **usbphy;
517 	struct usb_phy *curr_usbphy;
518 	struct notifier_block vbus_nb;
519 
520 	struct completion disconnect_complete;
521 
522 	bool selfpowered;
523 
524 #define TOGGLE_VBUS_WAIT_MS 100
525 	struct delayed_work plc_reset_work;
526 	bool wait_csc;
527 
528 	struct delayed_work port_reset_war_work;
529 	bool wait_for_sec_prc;
530 };
531 
532 #define XUDC_TRB_MAX_BUFFER_SIZE 65536
533 #define XUDC_MAX_ISOCH_EPS 4
534 #define XUDC_INTERRUPT_MODERATION_US 0
535 
536 static struct usb_endpoint_descriptor tegra_xudc_ep0_desc = {
537 	.bLength = USB_DT_ENDPOINT_SIZE,
538 	.bDescriptorType = USB_DT_ENDPOINT,
539 	.bEndpointAddress = 0,
540 	.bmAttributes = USB_ENDPOINT_XFER_CONTROL,
541 	.wMaxPacketSize = cpu_to_le16(64),
542 };
543 
544 struct tegra_xudc_soc {
545 	const char * const *supply_names;
546 	unsigned int num_supplies;
547 	const char * const *clock_names;
548 	unsigned int num_clks;
549 	unsigned int num_phys;
550 	bool u1_enable;
551 	bool u2_enable;
552 	bool lpm_enable;
553 	bool invalid_seq_num;
554 	bool pls_quirk;
555 	bool port_reset_quirk;
556 	bool port_speed_quirk;
557 	bool has_ipfs;
558 };
559 
560 static inline u32 fpci_readl(struct tegra_xudc *xudc, unsigned int offset)
561 {
562 	return readl(xudc->fpci + offset);
563 }
564 
565 static inline void fpci_writel(struct tegra_xudc *xudc, u32 val,
566 			       unsigned int offset)
567 {
568 	writel(val, xudc->fpci + offset);
569 }
570 
571 static inline u32 ipfs_readl(struct tegra_xudc *xudc, unsigned int offset)
572 {
573 	return readl(xudc->ipfs + offset);
574 }
575 
576 static inline void ipfs_writel(struct tegra_xudc *xudc, u32 val,
577 			       unsigned int offset)
578 {
579 	writel(val, xudc->ipfs + offset);
580 }
581 
582 static inline u32 xudc_readl(struct tegra_xudc *xudc, unsigned int offset)
583 {
584 	return readl(xudc->base + offset);
585 }
586 
587 static inline void xudc_writel(struct tegra_xudc *xudc, u32 val,
588 			       unsigned int offset)
589 {
590 	writel(val, xudc->base + offset);
591 }
592 
593 static inline int xudc_readl_poll(struct tegra_xudc *xudc,
594 				  unsigned int offset, u32 mask, u32 val)
595 {
596 	u32 regval;
597 
598 	return readl_poll_timeout_atomic(xudc->base + offset, regval,
599 					 (regval & mask) == val, 1, 100);
600 }
601 
602 static inline struct tegra_xudc *to_xudc(struct usb_gadget *gadget)
603 {
604 	return container_of(gadget, struct tegra_xudc, gadget);
605 }
606 
607 static inline struct tegra_xudc_ep *to_xudc_ep(struct usb_ep *ep)
608 {
609 	return container_of(ep, struct tegra_xudc_ep, usb_ep);
610 }
611 
612 static inline struct tegra_xudc_request *to_xudc_req(struct usb_request *req)
613 {
614 	return container_of(req, struct tegra_xudc_request, usb_req);
615 }
616 
617 static inline void dump_trb(struct tegra_xudc *xudc, const char *type,
618 			    struct tegra_xudc_trb *trb)
619 {
620 	dev_dbg(xudc->dev,
621 		"%s: %p, lo = %#x, hi = %#x, status = %#x, control = %#x\n",
622 		type, trb, trb->data_lo, trb->data_hi, trb->status,
623 		trb->control);
624 }
625 
626 static void tegra_xudc_limit_port_speed(struct tegra_xudc *xudc)
627 {
628 	u32 val;
629 
630 	/* limit port speed to gen 1 */
631 	val = xudc_readl(xudc, SSPX_CORE_CNT56);
632 	val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
633 	val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x260);
634 	xudc_writel(xudc, val, SSPX_CORE_CNT56);
635 
636 	val = xudc_readl(xudc, SSPX_CORE_CNT57);
637 	val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
638 	val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x6D6);
639 	xudc_writel(xudc, val, SSPX_CORE_CNT57);
640 
641 	val = xudc_readl(xudc, SSPX_CORE_CNT65);
642 	val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
643 	val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0x4B0);
644 	xudc_writel(xudc, val, SSPX_CORE_CNT66);
645 
646 	val = xudc_readl(xudc, SSPX_CORE_CNT66);
647 	val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
648 	val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x4B0);
649 	xudc_writel(xudc, val, SSPX_CORE_CNT66);
650 
651 	val = xudc_readl(xudc, SSPX_CORE_CNT67);
652 	val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
653 	val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x4B0);
654 	xudc_writel(xudc, val, SSPX_CORE_CNT67);
655 
656 	val = xudc_readl(xudc, SSPX_CORE_CNT72);
657 	val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
658 	val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x10);
659 	xudc_writel(xudc, val, SSPX_CORE_CNT72);
660 }
661 
662 static void tegra_xudc_restore_port_speed(struct tegra_xudc *xudc)
663 {
664 	u32 val;
665 
666 	/* restore port speed to gen2 */
667 	val = xudc_readl(xudc, SSPX_CORE_CNT56);
668 	val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
669 	val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x438);
670 	xudc_writel(xudc, val, SSPX_CORE_CNT56);
671 
672 	val = xudc_readl(xudc, SSPX_CORE_CNT57);
673 	val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
674 	val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x528);
675 	xudc_writel(xudc, val, SSPX_CORE_CNT57);
676 
677 	val = xudc_readl(xudc, SSPX_CORE_CNT65);
678 	val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
679 	val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0xE10);
680 	xudc_writel(xudc, val, SSPX_CORE_CNT66);
681 
682 	val = xudc_readl(xudc, SSPX_CORE_CNT66);
683 	val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
684 	val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x348);
685 	xudc_writel(xudc, val, SSPX_CORE_CNT66);
686 
687 	val = xudc_readl(xudc, SSPX_CORE_CNT67);
688 	val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
689 	val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x5a0);
690 	xudc_writel(xudc, val, SSPX_CORE_CNT67);
691 
692 	val = xudc_readl(xudc, SSPX_CORE_CNT72);
693 	val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
694 	val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x1c21);
695 	xudc_writel(xudc, val, SSPX_CORE_CNT72);
696 }
697 
698 static void tegra_xudc_device_mode_on(struct tegra_xudc *xudc)
699 {
700 	int err;
701 
702 	pm_runtime_get_sync(xudc->dev);
703 
704 	tegra_phy_xusb_utmi_pad_power_on(xudc->curr_utmi_phy);
705 
706 	err = phy_power_on(xudc->curr_utmi_phy);
707 	if (err < 0)
708 		dev_err(xudc->dev, "UTMI power on failed: %d\n", err);
709 
710 	err = phy_power_on(xudc->curr_usb3_phy);
711 	if (err < 0)
712 		dev_err(xudc->dev, "USB3 PHY power on failed: %d\n", err);
713 
714 	dev_dbg(xudc->dev, "device mode on\n");
715 
716 	phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
717 			 USB_ROLE_DEVICE);
718 }
719 
720 static void tegra_xudc_device_mode_off(struct tegra_xudc *xudc)
721 {
722 	bool connected = false;
723 	u32 pls, val;
724 	int err;
725 
726 	dev_dbg(xudc->dev, "device mode off\n");
727 
728 	connected = !!(xudc_readl(xudc, PORTSC) & PORTSC_CCS);
729 
730 	reinit_completion(&xudc->disconnect_complete);
731 
732 	if (xudc->soc->port_speed_quirk)
733 		tegra_xudc_restore_port_speed(xudc);
734 
735 	phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG, USB_ROLE_NONE);
736 
737 	pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
738 		PORTSC_PLS_SHIFT;
739 
740 	/* Direct link to U0 if disconnected in RESUME or U2. */
741 	if (xudc->soc->pls_quirk && xudc->gadget.speed == USB_SPEED_SUPER &&
742 	    (pls == PORTSC_PLS_RESUME || pls == PORTSC_PLS_U2)) {
743 		val = xudc_readl(xudc, PORTPM);
744 		val |= PORTPM_FRWE;
745 		xudc_writel(xudc, val, PORTPM);
746 
747 		val = xudc_readl(xudc, PORTSC);
748 		val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
749 		val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
750 		xudc_writel(xudc, val, PORTSC);
751 	}
752 
753 	/* Wait for disconnect event. */
754 	if (connected)
755 		wait_for_completion(&xudc->disconnect_complete);
756 
757 	/* Make sure interrupt handler has completed before powergating. */
758 	synchronize_irq(xudc->irq);
759 
760 	tegra_phy_xusb_utmi_pad_power_down(xudc->curr_utmi_phy);
761 
762 	err = phy_power_off(xudc->curr_utmi_phy);
763 	if (err < 0)
764 		dev_err(xudc->dev, "UTMI PHY power off failed: %d\n", err);
765 
766 	err = phy_power_off(xudc->curr_usb3_phy);
767 	if (err < 0)
768 		dev_err(xudc->dev, "USB3 PHY power off failed: %d\n", err);
769 
770 	pm_runtime_put(xudc->dev);
771 }
772 
773 static void tegra_xudc_usb_role_sw_work(struct work_struct *work)
774 {
775 	struct tegra_xudc *xudc = container_of(work, struct tegra_xudc,
776 					       usb_role_sw_work);
777 
778 	if (xudc->device_mode)
779 		tegra_xudc_device_mode_on(xudc);
780 	else
781 		tegra_xudc_device_mode_off(xudc);
782 }
783 
784 static int tegra_xudc_get_phy_index(struct tegra_xudc *xudc,
785 					      struct usb_phy *usbphy)
786 {
787 	unsigned int i;
788 
789 	for (i = 0; i < xudc->soc->num_phys; i++) {
790 		if (xudc->usbphy[i] && usbphy == xudc->usbphy[i])
791 			return i;
792 	}
793 
794 	dev_info(xudc->dev, "phy index could not be found for shared USB PHY");
795 	return -1;
796 }
797 
798 static void tegra_xudc_update_data_role(struct tegra_xudc *xudc,
799 					      struct usb_phy *usbphy)
800 {
801 	int phy_index;
802 
803 	if ((xudc->device_mode && usbphy->last_event == USB_EVENT_VBUS) ||
804 	    (!xudc->device_mode && usbphy->last_event != USB_EVENT_VBUS)) {
805 		dev_dbg(xudc->dev, "Same role(%d) received. Ignore",
806 			xudc->device_mode);
807 		return;
808 	}
809 
810 	xudc->device_mode = (usbphy->last_event == USB_EVENT_VBUS) ? true :
811 								     false;
812 
813 	phy_index = tegra_xudc_get_phy_index(xudc, usbphy);
814 	dev_dbg(xudc->dev, "%s(): current phy index is %d\n", __func__,
815 		phy_index);
816 
817 	if (!xudc->suspended && phy_index != -1) {
818 		xudc->curr_utmi_phy = xudc->utmi_phy[phy_index];
819 		xudc->curr_usb3_phy = xudc->usb3_phy[phy_index];
820 		xudc->curr_usbphy = usbphy;
821 		schedule_work(&xudc->usb_role_sw_work);
822 	}
823 }
824 
825 static int tegra_xudc_vbus_notify(struct notifier_block *nb,
826 					 unsigned long action, void *data)
827 {
828 	struct tegra_xudc *xudc = container_of(nb, struct tegra_xudc,
829 					       vbus_nb);
830 	struct usb_phy *usbphy = (struct usb_phy *)data;
831 
832 	dev_dbg(xudc->dev, "%s(): event is %d\n", __func__, usbphy->last_event);
833 
834 	tegra_xudc_update_data_role(xudc, usbphy);
835 
836 	return NOTIFY_OK;
837 }
838 
839 static void tegra_xudc_plc_reset_work(struct work_struct *work)
840 {
841 	struct delayed_work *dwork = to_delayed_work(work);
842 	struct tegra_xudc *xudc = container_of(dwork, struct tegra_xudc,
843 					       plc_reset_work);
844 	unsigned long flags;
845 
846 	spin_lock_irqsave(&xudc->lock, flags);
847 
848 	if (xudc->wait_csc) {
849 		u32 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
850 			PORTSC_PLS_SHIFT;
851 
852 		if (pls == PORTSC_PLS_INACTIVE) {
853 			dev_info(xudc->dev, "PLS = Inactive. Toggle VBUS\n");
854 			phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
855 					 USB_ROLE_NONE);
856 			phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
857 					 USB_ROLE_DEVICE);
858 
859 			xudc->wait_csc = false;
860 		}
861 	}
862 
863 	spin_unlock_irqrestore(&xudc->lock, flags);
864 }
865 
866 static void tegra_xudc_port_reset_war_work(struct work_struct *work)
867 {
868 	struct delayed_work *dwork = to_delayed_work(work);
869 	struct tegra_xudc *xudc =
870 		container_of(dwork, struct tegra_xudc, port_reset_war_work);
871 	unsigned long flags;
872 	u32 pls;
873 	int ret;
874 
875 	spin_lock_irqsave(&xudc->lock, flags);
876 
877 	if (xudc->device_mode && xudc->wait_for_sec_prc) {
878 		pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
879 			PORTSC_PLS_SHIFT;
880 		dev_dbg(xudc->dev, "pls = %x\n", pls);
881 
882 		if (pls == PORTSC_PLS_DISABLED) {
883 			dev_dbg(xudc->dev, "toggle vbus\n");
884 			/* PRC doesn't complete in 100ms, toggle the vbus */
885 			ret = tegra_phy_xusb_utmi_port_reset(
886 				xudc->curr_utmi_phy);
887 			if (ret == 1)
888 				xudc->wait_for_sec_prc = 0;
889 		}
890 	}
891 
892 	spin_unlock_irqrestore(&xudc->lock, flags);
893 }
894 
895 static dma_addr_t trb_virt_to_phys(struct tegra_xudc_ep *ep,
896 				   struct tegra_xudc_trb *trb)
897 {
898 	unsigned int index;
899 
900 	index = trb - ep->transfer_ring;
901 
902 	if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
903 		return 0;
904 
905 	return (ep->transfer_ring_phys + index * sizeof(*trb));
906 }
907 
908 static struct tegra_xudc_trb *trb_phys_to_virt(struct tegra_xudc_ep *ep,
909 					       dma_addr_t addr)
910 {
911 	struct tegra_xudc_trb *trb;
912 	unsigned int index;
913 
914 	index = (addr - ep->transfer_ring_phys) / sizeof(*trb);
915 
916 	if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
917 		return NULL;
918 
919 	trb = &ep->transfer_ring[index];
920 
921 	return trb;
922 }
923 
924 static void ep_reload(struct tegra_xudc *xudc, unsigned int ep)
925 {
926 	xudc_writel(xudc, BIT(ep), EP_RELOAD);
927 	xudc_readl_poll(xudc, EP_RELOAD, BIT(ep), 0);
928 }
929 
930 static void ep_pause(struct tegra_xudc *xudc, unsigned int ep)
931 {
932 	u32 val;
933 
934 	val = xudc_readl(xudc, EP_PAUSE);
935 	if (val & BIT(ep))
936 		return;
937 	val |= BIT(ep);
938 
939 	xudc_writel(xudc, val, EP_PAUSE);
940 
941 	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
942 
943 	xudc_writel(xudc, BIT(ep), EP_STCHG);
944 }
945 
946 static void ep_unpause(struct tegra_xudc *xudc, unsigned int ep)
947 {
948 	u32 val;
949 
950 	val = xudc_readl(xudc, EP_PAUSE);
951 	if (!(val & BIT(ep)))
952 		return;
953 	val &= ~BIT(ep);
954 
955 	xudc_writel(xudc, val, EP_PAUSE);
956 
957 	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
958 
959 	xudc_writel(xudc, BIT(ep), EP_STCHG);
960 }
961 
962 static void ep_unpause_all(struct tegra_xudc *xudc)
963 {
964 	u32 val;
965 
966 	val = xudc_readl(xudc, EP_PAUSE);
967 
968 	xudc_writel(xudc, 0, EP_PAUSE);
969 
970 	xudc_readl_poll(xudc, EP_STCHG, val, val);
971 
972 	xudc_writel(xudc, val, EP_STCHG);
973 }
974 
975 static void ep_halt(struct tegra_xudc *xudc, unsigned int ep)
976 {
977 	u32 val;
978 
979 	val = xudc_readl(xudc, EP_HALT);
980 	if (val & BIT(ep))
981 		return;
982 	val |= BIT(ep);
983 	xudc_writel(xudc, val, EP_HALT);
984 
985 	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
986 
987 	xudc_writel(xudc, BIT(ep), EP_STCHG);
988 }
989 
990 static void ep_unhalt(struct tegra_xudc *xudc, unsigned int ep)
991 {
992 	u32 val;
993 
994 	val = xudc_readl(xudc, EP_HALT);
995 	if (!(val & BIT(ep)))
996 		return;
997 	val &= ~BIT(ep);
998 	xudc_writel(xudc, val, EP_HALT);
999 
1000 	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
1001 
1002 	xudc_writel(xudc, BIT(ep), EP_STCHG);
1003 }
1004 
1005 static void ep_unhalt_all(struct tegra_xudc *xudc)
1006 {
1007 	u32 val;
1008 
1009 	val = xudc_readl(xudc, EP_HALT);
1010 	if (!val)
1011 		return;
1012 	xudc_writel(xudc, 0, EP_HALT);
1013 
1014 	xudc_readl_poll(xudc, EP_STCHG, val, val);
1015 
1016 	xudc_writel(xudc, val, EP_STCHG);
1017 }
1018 
1019 static void ep_wait_for_stopped(struct tegra_xudc *xudc, unsigned int ep)
1020 {
1021 	xudc_readl_poll(xudc, EP_STOPPED, BIT(ep), BIT(ep));
1022 	xudc_writel(xudc, BIT(ep), EP_STOPPED);
1023 }
1024 
1025 static void ep_wait_for_inactive(struct tegra_xudc *xudc, unsigned int ep)
1026 {
1027 	xudc_readl_poll(xudc, EP_THREAD_ACTIVE, BIT(ep), 0);
1028 }
1029 
1030 static void tegra_xudc_req_done(struct tegra_xudc_ep *ep,
1031 				struct tegra_xudc_request *req, int status)
1032 {
1033 	struct tegra_xudc *xudc = ep->xudc;
1034 
1035 	dev_dbg(xudc->dev, "completing request %p on EP %u with status %d\n",
1036 		 req, ep->index, status);
1037 
1038 	if (likely(req->usb_req.status == -EINPROGRESS))
1039 		req->usb_req.status = status;
1040 
1041 	list_del_init(&req->list);
1042 
1043 	if (usb_endpoint_xfer_control(ep->desc)) {
1044 		usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
1045 					 (xudc->setup_state ==
1046 					  DATA_STAGE_XFER));
1047 	} else {
1048 		usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
1049 					 usb_endpoint_dir_in(ep->desc));
1050 	}
1051 
1052 	spin_unlock(&xudc->lock);
1053 	usb_gadget_giveback_request(&ep->usb_ep, &req->usb_req);
1054 	spin_lock(&xudc->lock);
1055 }
1056 
1057 static void tegra_xudc_ep_nuke(struct tegra_xudc_ep *ep, int status)
1058 {
1059 	struct tegra_xudc_request *req;
1060 
1061 	while (!list_empty(&ep->queue)) {
1062 		req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1063 				       list);
1064 		tegra_xudc_req_done(ep, req, status);
1065 	}
1066 }
1067 
1068 static unsigned int ep_available_trbs(struct tegra_xudc_ep *ep)
1069 {
1070 	if (ep->ring_full)
1071 		return 0;
1072 
1073 	if (ep->deq_ptr > ep->enq_ptr)
1074 		return ep->deq_ptr - ep->enq_ptr - 1;
1075 
1076 	return XUDC_TRANSFER_RING_SIZE - (ep->enq_ptr - ep->deq_ptr) - 2;
1077 }
1078 
1079 static void tegra_xudc_queue_one_trb(struct tegra_xudc_ep *ep,
1080 				     struct tegra_xudc_request *req,
1081 				     struct tegra_xudc_trb *trb,
1082 				     bool ioc)
1083 {
1084 	struct tegra_xudc *xudc = ep->xudc;
1085 	dma_addr_t buf_addr;
1086 	size_t len;
1087 
1088 	len = min_t(size_t, XUDC_TRB_MAX_BUFFER_SIZE, req->usb_req.length -
1089 		    req->buf_queued);
1090 	if (len > 0)
1091 		buf_addr = req->usb_req.dma + req->buf_queued;
1092 	else
1093 		buf_addr = 0;
1094 
1095 	trb_write_data_ptr(trb, buf_addr);
1096 
1097 	trb_write_transfer_len(trb, len);
1098 	trb_write_td_size(trb, req->trbs_needed - req->trbs_queued - 1);
1099 
1100 	if (req->trbs_queued == req->trbs_needed - 1 ||
1101 		(req->need_zlp && req->trbs_queued == req->trbs_needed - 2))
1102 		trb_write_chain(trb, 0);
1103 	else
1104 		trb_write_chain(trb, 1);
1105 
1106 	trb_write_ioc(trb, ioc);
1107 
1108 	if (usb_endpoint_dir_out(ep->desc) ||
1109 	    (usb_endpoint_xfer_control(ep->desc) &&
1110 	     (xudc->setup_state == DATA_STAGE_RECV)))
1111 		trb_write_isp(trb, 1);
1112 	else
1113 		trb_write_isp(trb, 0);
1114 
1115 	if (usb_endpoint_xfer_control(ep->desc)) {
1116 		if (xudc->setup_state == DATA_STAGE_XFER ||
1117 		    xudc->setup_state == DATA_STAGE_RECV)
1118 			trb_write_type(trb, TRB_TYPE_DATA_STAGE);
1119 		else
1120 			trb_write_type(trb, TRB_TYPE_STATUS_STAGE);
1121 
1122 		if (xudc->setup_state == DATA_STAGE_XFER ||
1123 		    xudc->setup_state == STATUS_STAGE_XFER)
1124 			trb_write_data_stage_dir(trb, 1);
1125 		else
1126 			trb_write_data_stage_dir(trb, 0);
1127 	} else if (usb_endpoint_xfer_isoc(ep->desc)) {
1128 		trb_write_type(trb, TRB_TYPE_ISOCH);
1129 		trb_write_sia(trb, 1);
1130 		trb_write_frame_id(trb, 0);
1131 		trb_write_tlbpc(trb, 0);
1132 	} else if (usb_ss_max_streams(ep->comp_desc)) {
1133 		trb_write_type(trb, TRB_TYPE_STREAM);
1134 		trb_write_stream_id(trb, req->usb_req.stream_id);
1135 	} else {
1136 		trb_write_type(trb, TRB_TYPE_NORMAL);
1137 		trb_write_stream_id(trb, 0);
1138 	}
1139 
1140 	trb_write_cycle(trb, ep->pcs);
1141 
1142 	req->trbs_queued++;
1143 	req->buf_queued += len;
1144 
1145 	dump_trb(xudc, "TRANSFER", trb);
1146 }
1147 
1148 static unsigned int tegra_xudc_queue_trbs(struct tegra_xudc_ep *ep,
1149 					  struct tegra_xudc_request *req)
1150 {
1151 	unsigned int i, count, available;
1152 	bool wait_td = false;
1153 
1154 	available = ep_available_trbs(ep);
1155 	count = req->trbs_needed - req->trbs_queued;
1156 	if (available < count) {
1157 		count = available;
1158 		ep->ring_full = true;
1159 	}
1160 
1161 	/*
1162 	 * To generate zero-length packet on USB bus, SW needs schedule a
1163 	 * standalone zero-length TD. According to HW's behavior, SW needs
1164 	 * to schedule TDs in different ways for different endpoint types.
1165 	 *
1166 	 * For control endpoint:
1167 	 * - Data stage TD (IOC = 1, CH = 0)
1168 	 * - Ring doorbell and wait transfer event
1169 	 * - Data stage TD for ZLP (IOC = 1, CH = 0)
1170 	 * - Ring doorbell
1171 	 *
1172 	 * For bulk and interrupt endpoints:
1173 	 * - Normal transfer TD (IOC = 0, CH = 0)
1174 	 * - Normal transfer TD for ZLP (IOC = 1, CH = 0)
1175 	 * - Ring doorbell
1176 	 */
1177 
1178 	if (req->need_zlp && usb_endpoint_xfer_control(ep->desc) && count > 1)
1179 		wait_td = true;
1180 
1181 	if (!req->first_trb)
1182 		req->first_trb = &ep->transfer_ring[ep->enq_ptr];
1183 
1184 	for (i = 0; i < count; i++) {
1185 		struct tegra_xudc_trb *trb = &ep->transfer_ring[ep->enq_ptr];
1186 		bool ioc = false;
1187 
1188 		if ((i == count - 1) || (wait_td && i == count - 2))
1189 			ioc = true;
1190 
1191 		tegra_xudc_queue_one_trb(ep, req, trb, ioc);
1192 		req->last_trb = trb;
1193 
1194 		ep->enq_ptr++;
1195 		if (ep->enq_ptr == XUDC_TRANSFER_RING_SIZE - 1) {
1196 			trb = &ep->transfer_ring[ep->enq_ptr];
1197 			trb_write_cycle(trb, ep->pcs);
1198 			ep->pcs = !ep->pcs;
1199 			ep->enq_ptr = 0;
1200 		}
1201 
1202 		if (ioc)
1203 			break;
1204 	}
1205 
1206 	return count;
1207 }
1208 
1209 static void tegra_xudc_ep_ring_doorbell(struct tegra_xudc_ep *ep)
1210 {
1211 	struct tegra_xudc *xudc = ep->xudc;
1212 	u32 val;
1213 
1214 	if (list_empty(&ep->queue))
1215 		return;
1216 
1217 	val = DB_TARGET(ep->index);
1218 	if (usb_endpoint_xfer_control(ep->desc)) {
1219 		val |= DB_STREAMID(xudc->setup_seq_num);
1220 	} else if (usb_ss_max_streams(ep->comp_desc) > 0) {
1221 		struct tegra_xudc_request *req;
1222 
1223 		/* Don't ring doorbell if the stream has been rejected. */
1224 		if (ep->stream_rejected)
1225 			return;
1226 
1227 		req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1228 				       list);
1229 		val |= DB_STREAMID(req->usb_req.stream_id);
1230 	}
1231 
1232 	dev_dbg(xudc->dev, "ring doorbell: %#x\n", val);
1233 	xudc_writel(xudc, val, DB);
1234 }
1235 
1236 static void tegra_xudc_ep_kick_queue(struct tegra_xudc_ep *ep)
1237 {
1238 	struct tegra_xudc_request *req;
1239 	bool trbs_queued = false;
1240 
1241 	list_for_each_entry(req, &ep->queue, list) {
1242 		if (ep->ring_full)
1243 			break;
1244 
1245 		if (tegra_xudc_queue_trbs(ep, req) > 0)
1246 			trbs_queued = true;
1247 	}
1248 
1249 	if (trbs_queued)
1250 		tegra_xudc_ep_ring_doorbell(ep);
1251 }
1252 
1253 static int
1254 __tegra_xudc_ep_queue(struct tegra_xudc_ep *ep, struct tegra_xudc_request *req)
1255 {
1256 	struct tegra_xudc *xudc = ep->xudc;
1257 	int err;
1258 
1259 	if (usb_endpoint_xfer_control(ep->desc) && !list_empty(&ep->queue)) {
1260 		dev_err(xudc->dev, "control EP has pending transfers\n");
1261 		return -EINVAL;
1262 	}
1263 
1264 	if (usb_endpoint_xfer_control(ep->desc)) {
1265 		err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1266 					     (xudc->setup_state ==
1267 					      DATA_STAGE_XFER));
1268 	} else {
1269 		err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1270 					     usb_endpoint_dir_in(ep->desc));
1271 	}
1272 
1273 	if (err < 0) {
1274 		dev_err(xudc->dev, "failed to map request: %d\n", err);
1275 		return err;
1276 	}
1277 
1278 	req->first_trb = NULL;
1279 	req->last_trb = NULL;
1280 	req->buf_queued = 0;
1281 	req->trbs_queued = 0;
1282 	req->need_zlp = false;
1283 	req->trbs_needed = DIV_ROUND_UP(req->usb_req.length,
1284 					XUDC_TRB_MAX_BUFFER_SIZE);
1285 	if (req->usb_req.length == 0)
1286 		req->trbs_needed++;
1287 
1288 	if (!usb_endpoint_xfer_isoc(ep->desc) &&
1289 	    req->usb_req.zero && req->usb_req.length &&
1290 	    ((req->usb_req.length % ep->usb_ep.maxpacket) == 0)) {
1291 		req->trbs_needed++;
1292 		req->need_zlp = true;
1293 	}
1294 
1295 	req->usb_req.status = -EINPROGRESS;
1296 	req->usb_req.actual = 0;
1297 
1298 	list_add_tail(&req->list, &ep->queue);
1299 
1300 	tegra_xudc_ep_kick_queue(ep);
1301 
1302 	return 0;
1303 }
1304 
1305 static int
1306 tegra_xudc_ep_queue(struct usb_ep *usb_ep, struct usb_request *usb_req,
1307 		    gfp_t gfp)
1308 {
1309 	struct tegra_xudc_request *req;
1310 	struct tegra_xudc_ep *ep;
1311 	struct tegra_xudc *xudc;
1312 	unsigned long flags;
1313 	int ret;
1314 
1315 	if (!usb_ep || !usb_req)
1316 		return -EINVAL;
1317 
1318 	ep = to_xudc_ep(usb_ep);
1319 	req = to_xudc_req(usb_req);
1320 	xudc = ep->xudc;
1321 
1322 	spin_lock_irqsave(&xudc->lock, flags);
1323 	if (xudc->powergated || !ep->desc) {
1324 		ret = -ESHUTDOWN;
1325 		goto unlock;
1326 	}
1327 
1328 	ret = __tegra_xudc_ep_queue(ep, req);
1329 unlock:
1330 	spin_unlock_irqrestore(&xudc->lock, flags);
1331 
1332 	return ret;
1333 }
1334 
1335 static void squeeze_transfer_ring(struct tegra_xudc_ep *ep,
1336 				  struct tegra_xudc_request *req)
1337 {
1338 	struct tegra_xudc_trb *trb = req->first_trb;
1339 	bool pcs_enq = trb_read_cycle(trb);
1340 	bool pcs;
1341 
1342 	/*
1343 	 * Clear out all the TRBs part of or after the cancelled request,
1344 	 * and must correct trb cycle bit to the last un-enqueued state.
1345 	 */
1346 	while (trb != &ep->transfer_ring[ep->enq_ptr]) {
1347 		pcs = trb_read_cycle(trb);
1348 		memset(trb, 0, sizeof(*trb));
1349 		trb_write_cycle(trb, !pcs);
1350 		trb++;
1351 
1352 		if (trb_read_type(trb) == TRB_TYPE_LINK)
1353 			trb = ep->transfer_ring;
1354 	}
1355 
1356 	/* Requests will be re-queued at the start of the cancelled request. */
1357 	ep->enq_ptr = req->first_trb - ep->transfer_ring;
1358 	/*
1359 	 * Retrieve the correct cycle bit state from the first trb of
1360 	 * the cancelled request.
1361 	 */
1362 	ep->pcs = pcs_enq;
1363 	ep->ring_full = false;
1364 	list_for_each_entry_continue(req, &ep->queue, list) {
1365 		req->usb_req.status = -EINPROGRESS;
1366 		req->usb_req.actual = 0;
1367 
1368 		req->first_trb = NULL;
1369 		req->last_trb = NULL;
1370 		req->buf_queued = 0;
1371 		req->trbs_queued = 0;
1372 	}
1373 }
1374 
1375 /*
1376  * Determine if the given TRB is in the range [first trb, last trb] for the
1377  * given request.
1378  */
1379 static bool trb_in_request(struct tegra_xudc_ep *ep,
1380 			   struct tegra_xudc_request *req,
1381 			   struct tegra_xudc_trb *trb)
1382 {
1383 	dev_dbg(ep->xudc->dev, "%s: request %p -> %p; trb %p\n", __func__,
1384 		req->first_trb, req->last_trb, trb);
1385 
1386 	if (trb >= req->first_trb && (trb <= req->last_trb ||
1387 				      req->last_trb < req->first_trb))
1388 		return true;
1389 
1390 	if (trb < req->first_trb && trb <= req->last_trb &&
1391 	    req->last_trb < req->first_trb)
1392 		return true;
1393 
1394 	return false;
1395 }
1396 
1397 /*
1398  * Determine if the given TRB is in the range [EP enqueue pointer, first TRB)
1399  * for the given endpoint and request.
1400  */
1401 static bool trb_before_request(struct tegra_xudc_ep *ep,
1402 			       struct tegra_xudc_request *req,
1403 			       struct tegra_xudc_trb *trb)
1404 {
1405 	struct tegra_xudc_trb *enq_trb = &ep->transfer_ring[ep->enq_ptr];
1406 
1407 	dev_dbg(ep->xudc->dev, "%s: request %p -> %p; enq ptr: %p; trb %p\n",
1408 		__func__, req->first_trb, req->last_trb, enq_trb, trb);
1409 
1410 	if (trb < req->first_trb && (enq_trb <= trb ||
1411 				     req->first_trb < enq_trb))
1412 		return true;
1413 
1414 	if (trb > req->first_trb && req->first_trb < enq_trb && enq_trb <= trb)
1415 		return true;
1416 
1417 	return false;
1418 }
1419 
1420 static int
1421 __tegra_xudc_ep_dequeue(struct tegra_xudc_ep *ep,
1422 			struct tegra_xudc_request *req)
1423 {
1424 	struct tegra_xudc *xudc = ep->xudc;
1425 	struct tegra_xudc_request *r = NULL, *iter;
1426 	struct tegra_xudc_trb *deq_trb;
1427 	bool busy, kick_queue = false;
1428 	int ret = 0;
1429 
1430 	/* Make sure the request is actually queued to this endpoint. */
1431 	list_for_each_entry(iter, &ep->queue, list) {
1432 		if (iter != req)
1433 			continue;
1434 		r = iter;
1435 		break;
1436 	}
1437 
1438 	if (!r)
1439 		return -EINVAL;
1440 
1441 	/* Request hasn't been queued in the transfer ring yet. */
1442 	if (!req->trbs_queued) {
1443 		tegra_xudc_req_done(ep, req, -ECONNRESET);
1444 		return 0;
1445 	}
1446 
1447 	/* Halt DMA for this endpoint. */
1448 	if (ep_ctx_read_state(ep->context) == EP_STATE_RUNNING) {
1449 		ep_pause(xudc, ep->index);
1450 		ep_wait_for_inactive(xudc, ep->index);
1451 	}
1452 
1453 	deq_trb = trb_phys_to_virt(ep, ep_ctx_read_deq_ptr(ep->context));
1454 	/* Is the hardware processing the TRB at the dequeue pointer? */
1455 	busy = (trb_read_cycle(deq_trb) == ep_ctx_read_dcs(ep->context));
1456 
1457 	if (trb_in_request(ep, req, deq_trb) && busy) {
1458 		/*
1459 		 * Request has been partially completed or it hasn't
1460 		 * started processing yet.
1461 		 */
1462 		dma_addr_t deq_ptr;
1463 
1464 		squeeze_transfer_ring(ep, req);
1465 
1466 		req->usb_req.actual = ep_ctx_read_edtla(ep->context);
1467 		tegra_xudc_req_done(ep, req, -ECONNRESET);
1468 		kick_queue = true;
1469 
1470 		/* EDTLA is > 0: request has been partially completed */
1471 		if (req->usb_req.actual > 0) {
1472 			/*
1473 			 * Abort the pending transfer and update the dequeue
1474 			 * pointer
1475 			 */
1476 			ep_ctx_write_edtla(ep->context, 0);
1477 			ep_ctx_write_partial_td(ep->context, 0);
1478 			ep_ctx_write_data_offset(ep->context, 0);
1479 
1480 			deq_ptr = trb_virt_to_phys(ep,
1481 					&ep->transfer_ring[ep->enq_ptr]);
1482 
1483 			if (dma_mapping_error(xudc->dev, deq_ptr)) {
1484 				ret = -EINVAL;
1485 			} else {
1486 				ep_ctx_write_deq_ptr(ep->context, deq_ptr);
1487 				ep_ctx_write_dcs(ep->context, ep->pcs);
1488 				ep_reload(xudc, ep->index);
1489 			}
1490 		}
1491 	} else if (trb_before_request(ep, req, deq_trb) && busy) {
1492 		/* Request hasn't started processing yet. */
1493 		squeeze_transfer_ring(ep, req);
1494 
1495 		tegra_xudc_req_done(ep, req, -ECONNRESET);
1496 		kick_queue = true;
1497 	} else {
1498 		/*
1499 		 * Request has completed, but we haven't processed the
1500 		 * completion event yet.
1501 		 */
1502 		tegra_xudc_req_done(ep, req, -ECONNRESET);
1503 		ret = -EINVAL;
1504 	}
1505 
1506 	/* Resume the endpoint. */
1507 	ep_unpause(xudc, ep->index);
1508 
1509 	if (kick_queue)
1510 		tegra_xudc_ep_kick_queue(ep);
1511 
1512 	return ret;
1513 }
1514 
1515 static int
1516 tegra_xudc_ep_dequeue(struct usb_ep *usb_ep, struct usb_request *usb_req)
1517 {
1518 	struct tegra_xudc_request *req;
1519 	struct tegra_xudc_ep *ep;
1520 	struct tegra_xudc *xudc;
1521 	unsigned long flags;
1522 	int ret;
1523 
1524 	if (!usb_ep || !usb_req)
1525 		return -EINVAL;
1526 
1527 	ep = to_xudc_ep(usb_ep);
1528 	req = to_xudc_req(usb_req);
1529 	xudc = ep->xudc;
1530 
1531 	spin_lock_irqsave(&xudc->lock, flags);
1532 
1533 	if (xudc->powergated || !ep->desc) {
1534 		ret = -ESHUTDOWN;
1535 		goto unlock;
1536 	}
1537 
1538 	ret = __tegra_xudc_ep_dequeue(ep, req);
1539 unlock:
1540 	spin_unlock_irqrestore(&xudc->lock, flags);
1541 
1542 	return ret;
1543 }
1544 
1545 static int __tegra_xudc_ep_set_halt(struct tegra_xudc_ep *ep, bool halt)
1546 {
1547 	struct tegra_xudc *xudc = ep->xudc;
1548 
1549 	if (!ep->desc)
1550 		return -EINVAL;
1551 
1552 	if (usb_endpoint_xfer_isoc(ep->desc)) {
1553 		dev_err(xudc->dev, "can't halt isochronous EP\n");
1554 		return -ENOTSUPP;
1555 	}
1556 
1557 	if (!!(xudc_readl(xudc, EP_HALT) & BIT(ep->index)) == halt) {
1558 		dev_dbg(xudc->dev, "EP %u already %s\n", ep->index,
1559 			halt ? "halted" : "not halted");
1560 		return 0;
1561 	}
1562 
1563 	if (halt) {
1564 		ep_halt(xudc, ep->index);
1565 	} else {
1566 		ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1567 
1568 		ep_reload(xudc, ep->index);
1569 
1570 		ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1571 		ep_ctx_write_rsvd(ep->context, 0);
1572 		ep_ctx_write_partial_td(ep->context, 0);
1573 		ep_ctx_write_splitxstate(ep->context, 0);
1574 		ep_ctx_write_seq_num(ep->context, 0);
1575 
1576 		ep_reload(xudc, ep->index);
1577 		ep_unpause(xudc, ep->index);
1578 		ep_unhalt(xudc, ep->index);
1579 
1580 		tegra_xudc_ep_ring_doorbell(ep);
1581 	}
1582 
1583 	return 0;
1584 }
1585 
1586 static int tegra_xudc_ep_set_halt(struct usb_ep *usb_ep, int value)
1587 {
1588 	struct tegra_xudc_ep *ep;
1589 	struct tegra_xudc *xudc;
1590 	unsigned long flags;
1591 	int ret;
1592 
1593 	if (!usb_ep)
1594 		return -EINVAL;
1595 
1596 	ep = to_xudc_ep(usb_ep);
1597 	xudc = ep->xudc;
1598 
1599 	spin_lock_irqsave(&xudc->lock, flags);
1600 	if (xudc->powergated) {
1601 		ret = -ESHUTDOWN;
1602 		goto unlock;
1603 	}
1604 
1605 	if (value && usb_endpoint_dir_in(ep->desc) &&
1606 	    !list_empty(&ep->queue)) {
1607 		dev_err(xudc->dev, "can't halt EP with requests pending\n");
1608 		ret = -EAGAIN;
1609 		goto unlock;
1610 	}
1611 
1612 	ret = __tegra_xudc_ep_set_halt(ep, value);
1613 unlock:
1614 	spin_unlock_irqrestore(&xudc->lock, flags);
1615 
1616 	return ret;
1617 }
1618 
1619 static void tegra_xudc_ep_context_setup(struct tegra_xudc_ep *ep)
1620 {
1621 	const struct usb_endpoint_descriptor *desc = ep->desc;
1622 	const struct usb_ss_ep_comp_descriptor *comp_desc = ep->comp_desc;
1623 	struct tegra_xudc *xudc = ep->xudc;
1624 	u16 maxpacket, maxburst = 0, esit = 0;
1625 	u32 val;
1626 
1627 	maxpacket = usb_endpoint_maxp(desc);
1628 	if (xudc->gadget.speed == USB_SPEED_SUPER) {
1629 		if (!usb_endpoint_xfer_control(desc))
1630 			maxburst = comp_desc->bMaxBurst;
1631 
1632 		if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc))
1633 			esit = le16_to_cpu(comp_desc->wBytesPerInterval);
1634 	} else if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
1635 		   (usb_endpoint_xfer_int(desc) ||
1636 		    usb_endpoint_xfer_isoc(desc))) {
1637 		if (xudc->gadget.speed == USB_SPEED_HIGH) {
1638 			maxburst = usb_endpoint_maxp_mult(desc) - 1;
1639 			if (maxburst == 0x3) {
1640 				dev_warn(xudc->dev,
1641 					 "invalid endpoint maxburst\n");
1642 				maxburst = 0x2;
1643 			}
1644 		}
1645 		esit = maxpacket * (maxburst + 1);
1646 	}
1647 
1648 	memset(ep->context, 0, sizeof(*ep->context));
1649 
1650 	ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1651 	ep_ctx_write_interval(ep->context, desc->bInterval);
1652 	if (xudc->gadget.speed == USB_SPEED_SUPER) {
1653 		if (usb_endpoint_xfer_isoc(desc)) {
1654 			ep_ctx_write_mult(ep->context,
1655 					  comp_desc->bmAttributes & 0x3);
1656 		}
1657 
1658 		if (usb_endpoint_xfer_bulk(desc)) {
1659 			ep_ctx_write_max_pstreams(ep->context,
1660 						  comp_desc->bmAttributes &
1661 						  0x1f);
1662 			ep_ctx_write_lsa(ep->context, 1);
1663 		}
1664 	}
1665 
1666 	if (!usb_endpoint_xfer_control(desc) && usb_endpoint_dir_out(desc))
1667 		val = usb_endpoint_type(desc);
1668 	else
1669 		val = usb_endpoint_type(desc) + EP_TYPE_CONTROL;
1670 
1671 	ep_ctx_write_type(ep->context, val);
1672 	ep_ctx_write_cerr(ep->context, 0x3);
1673 	ep_ctx_write_max_packet_size(ep->context, maxpacket);
1674 	ep_ctx_write_max_burst_size(ep->context, maxburst);
1675 
1676 	ep_ctx_write_deq_ptr(ep->context, ep->transfer_ring_phys);
1677 	ep_ctx_write_dcs(ep->context, ep->pcs);
1678 
1679 	/* Select a reasonable average TRB length based on endpoint type. */
1680 	switch (usb_endpoint_type(desc)) {
1681 	case USB_ENDPOINT_XFER_CONTROL:
1682 		val = 8;
1683 		break;
1684 	case USB_ENDPOINT_XFER_INT:
1685 		val = 1024;
1686 		break;
1687 	case USB_ENDPOINT_XFER_BULK:
1688 	case USB_ENDPOINT_XFER_ISOC:
1689 	default:
1690 		val = 3072;
1691 		break;
1692 	}
1693 
1694 	ep_ctx_write_avg_trb_len(ep->context, val);
1695 	ep_ctx_write_max_esit_payload(ep->context, esit);
1696 
1697 	ep_ctx_write_cerrcnt(ep->context, 0x3);
1698 }
1699 
1700 static void setup_link_trb(struct tegra_xudc_ep *ep,
1701 			   struct tegra_xudc_trb *trb)
1702 {
1703 	trb_write_data_ptr(trb, ep->transfer_ring_phys);
1704 	trb_write_type(trb, TRB_TYPE_LINK);
1705 	trb_write_toggle_cycle(trb, 1);
1706 }
1707 
1708 static int __tegra_xudc_ep_disable(struct tegra_xudc_ep *ep)
1709 {
1710 	struct tegra_xudc *xudc = ep->xudc;
1711 
1712 	if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
1713 		dev_err(xudc->dev, "endpoint %u already disabled\n",
1714 			ep->index);
1715 		return -EINVAL;
1716 	}
1717 
1718 	ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1719 
1720 	ep_reload(xudc, ep->index);
1721 
1722 	tegra_xudc_ep_nuke(ep, -ESHUTDOWN);
1723 
1724 	xudc->nr_enabled_eps--;
1725 	if (usb_endpoint_xfer_isoc(ep->desc))
1726 		xudc->nr_isoch_eps--;
1727 
1728 	ep->desc = NULL;
1729 	ep->comp_desc = NULL;
1730 
1731 	memset(ep->context, 0, sizeof(*ep->context));
1732 
1733 	ep_unpause(xudc, ep->index);
1734 	ep_unhalt(xudc, ep->index);
1735 	if (xudc_readl(xudc, EP_STOPPED) & BIT(ep->index))
1736 		xudc_writel(xudc, BIT(ep->index), EP_STOPPED);
1737 
1738 	/*
1739 	 * If this is the last endpoint disabled in a de-configure request,
1740 	 * switch back to address state.
1741 	 */
1742 	if ((xudc->device_state == USB_STATE_CONFIGURED) &&
1743 	    (xudc->nr_enabled_eps == 1)) {
1744 		u32 val;
1745 
1746 		xudc->device_state = USB_STATE_ADDRESS;
1747 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1748 
1749 		val = xudc_readl(xudc, CTRL);
1750 		val &= ~CTRL_RUN;
1751 		xudc_writel(xudc, val, CTRL);
1752 	}
1753 
1754 	dev_info(xudc->dev, "ep %u disabled\n", ep->index);
1755 
1756 	return 0;
1757 }
1758 
1759 static int tegra_xudc_ep_disable(struct usb_ep *usb_ep)
1760 {
1761 	struct tegra_xudc_ep *ep;
1762 	struct tegra_xudc *xudc;
1763 	unsigned long flags;
1764 	int ret;
1765 
1766 	if (!usb_ep)
1767 		return -EINVAL;
1768 
1769 	ep = to_xudc_ep(usb_ep);
1770 	xudc = ep->xudc;
1771 
1772 	spin_lock_irqsave(&xudc->lock, flags);
1773 	if (xudc->powergated) {
1774 		ret = -ESHUTDOWN;
1775 		goto unlock;
1776 	}
1777 
1778 	ret = __tegra_xudc_ep_disable(ep);
1779 unlock:
1780 	spin_unlock_irqrestore(&xudc->lock, flags);
1781 
1782 	return ret;
1783 }
1784 
1785 static int __tegra_xudc_ep_enable(struct tegra_xudc_ep *ep,
1786 				  const struct usb_endpoint_descriptor *desc)
1787 {
1788 	struct tegra_xudc *xudc = ep->xudc;
1789 	unsigned int i;
1790 	u32 val;
1791 
1792 	if (xudc->gadget.speed == USB_SPEED_SUPER &&
1793 		!usb_endpoint_xfer_control(desc) && !ep->usb_ep.comp_desc)
1794 		return -EINVAL;
1795 
1796 	/* Disable the EP if it is not disabled */
1797 	if (ep_ctx_read_state(ep->context) != EP_STATE_DISABLED)
1798 		__tegra_xudc_ep_disable(ep);
1799 
1800 	ep->desc = desc;
1801 	ep->comp_desc = ep->usb_ep.comp_desc;
1802 
1803 	if (usb_endpoint_xfer_isoc(desc)) {
1804 		if (xudc->nr_isoch_eps > XUDC_MAX_ISOCH_EPS) {
1805 			dev_err(xudc->dev, "too many isochronous endpoints\n");
1806 			return -EBUSY;
1807 		}
1808 		xudc->nr_isoch_eps++;
1809 	}
1810 
1811 	memset(ep->transfer_ring, 0, XUDC_TRANSFER_RING_SIZE *
1812 	       sizeof(*ep->transfer_ring));
1813 	setup_link_trb(ep, &ep->transfer_ring[XUDC_TRANSFER_RING_SIZE - 1]);
1814 
1815 	ep->enq_ptr = 0;
1816 	ep->deq_ptr = 0;
1817 	ep->pcs = true;
1818 	ep->ring_full = false;
1819 	xudc->nr_enabled_eps++;
1820 
1821 	tegra_xudc_ep_context_setup(ep);
1822 
1823 	/*
1824 	 * No need to reload and un-halt EP0.  This will be done automatically
1825 	 * once a valid SETUP packet is received.
1826 	 */
1827 	if (usb_endpoint_xfer_control(desc))
1828 		goto out;
1829 
1830 	/*
1831 	 * Transition to configured state once the first non-control
1832 	 * endpoint is enabled.
1833 	 */
1834 	if (xudc->device_state == USB_STATE_ADDRESS) {
1835 		val = xudc_readl(xudc, CTRL);
1836 		val |= CTRL_RUN;
1837 		xudc_writel(xudc, val, CTRL);
1838 
1839 		xudc->device_state = USB_STATE_CONFIGURED;
1840 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1841 	}
1842 
1843 	if (usb_endpoint_xfer_isoc(desc)) {
1844 		/*
1845 		 * Pause all bulk endpoints when enabling an isoch endpoint
1846 		 * to ensure the isoch endpoint is allocated enough bandwidth.
1847 		 */
1848 		for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1849 			if (xudc->ep[i].desc &&
1850 			    usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1851 				ep_pause(xudc, i);
1852 		}
1853 	}
1854 
1855 	ep_reload(xudc, ep->index);
1856 	ep_unpause(xudc, ep->index);
1857 	ep_unhalt(xudc, ep->index);
1858 
1859 	if (usb_endpoint_xfer_isoc(desc)) {
1860 		for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1861 			if (xudc->ep[i].desc &&
1862 			    usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1863 				ep_unpause(xudc, i);
1864 		}
1865 	}
1866 
1867 out:
1868 	dev_info(xudc->dev, "EP %u (type: %s, dir: %s) enabled\n", ep->index,
1869 		 usb_ep_type_string(usb_endpoint_type(ep->desc)),
1870 		 usb_endpoint_dir_in(ep->desc) ? "in" : "out");
1871 
1872 	return 0;
1873 }
1874 
1875 static int tegra_xudc_ep_enable(struct usb_ep *usb_ep,
1876 				const struct usb_endpoint_descriptor *desc)
1877 {
1878 	struct tegra_xudc_ep *ep;
1879 	struct tegra_xudc *xudc;
1880 	unsigned long flags;
1881 	int ret;
1882 
1883 	if  (!usb_ep || !desc || (desc->bDescriptorType != USB_DT_ENDPOINT))
1884 		return -EINVAL;
1885 
1886 	ep = to_xudc_ep(usb_ep);
1887 	xudc = ep->xudc;
1888 
1889 	spin_lock_irqsave(&xudc->lock, flags);
1890 	if (xudc->powergated) {
1891 		ret = -ESHUTDOWN;
1892 		goto unlock;
1893 	}
1894 
1895 	ret = __tegra_xudc_ep_enable(ep, desc);
1896 unlock:
1897 	spin_unlock_irqrestore(&xudc->lock, flags);
1898 
1899 	return ret;
1900 }
1901 
1902 static struct usb_request *
1903 tegra_xudc_ep_alloc_request(struct usb_ep *usb_ep, gfp_t gfp)
1904 {
1905 	struct tegra_xudc_request *req;
1906 
1907 	req = kzalloc(sizeof(*req), gfp);
1908 	if (!req)
1909 		return NULL;
1910 
1911 	INIT_LIST_HEAD(&req->list);
1912 
1913 	return &req->usb_req;
1914 }
1915 
1916 static void tegra_xudc_ep_free_request(struct usb_ep *usb_ep,
1917 				       struct usb_request *usb_req)
1918 {
1919 	struct tegra_xudc_request *req = to_xudc_req(usb_req);
1920 
1921 	kfree(req);
1922 }
1923 
1924 static const struct usb_ep_ops tegra_xudc_ep_ops = {
1925 	.enable = tegra_xudc_ep_enable,
1926 	.disable = tegra_xudc_ep_disable,
1927 	.alloc_request = tegra_xudc_ep_alloc_request,
1928 	.free_request = tegra_xudc_ep_free_request,
1929 	.queue = tegra_xudc_ep_queue,
1930 	.dequeue = tegra_xudc_ep_dequeue,
1931 	.set_halt = tegra_xudc_ep_set_halt,
1932 };
1933 
1934 static int tegra_xudc_ep0_enable(struct usb_ep *usb_ep,
1935 				 const struct usb_endpoint_descriptor *desc)
1936 {
1937 	return -EBUSY;
1938 }
1939 
1940 static int tegra_xudc_ep0_disable(struct usb_ep *usb_ep)
1941 {
1942 	return -EBUSY;
1943 }
1944 
1945 static const struct usb_ep_ops tegra_xudc_ep0_ops = {
1946 	.enable = tegra_xudc_ep0_enable,
1947 	.disable = tegra_xudc_ep0_disable,
1948 	.alloc_request = tegra_xudc_ep_alloc_request,
1949 	.free_request = tegra_xudc_ep_free_request,
1950 	.queue = tegra_xudc_ep_queue,
1951 	.dequeue = tegra_xudc_ep_dequeue,
1952 	.set_halt = tegra_xudc_ep_set_halt,
1953 };
1954 
1955 static int tegra_xudc_gadget_get_frame(struct usb_gadget *gadget)
1956 {
1957 	struct tegra_xudc *xudc = to_xudc(gadget);
1958 	unsigned long flags;
1959 	int ret;
1960 
1961 	spin_lock_irqsave(&xudc->lock, flags);
1962 	if (xudc->powergated) {
1963 		ret = -ESHUTDOWN;
1964 		goto unlock;
1965 	}
1966 
1967 	ret = (xudc_readl(xudc, MFINDEX) & MFINDEX_FRAME_MASK) >>
1968 		MFINDEX_FRAME_SHIFT;
1969 unlock:
1970 	spin_unlock_irqrestore(&xudc->lock, flags);
1971 
1972 	return ret;
1973 }
1974 
1975 static void tegra_xudc_resume_device_state(struct tegra_xudc *xudc)
1976 {
1977 	unsigned int i;
1978 	u32 val;
1979 
1980 	ep_unpause_all(xudc);
1981 
1982 	/* Direct link to U0. */
1983 	val = xudc_readl(xudc, PORTSC);
1984 	if (((val & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT) != PORTSC_PLS_U0) {
1985 		val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
1986 		val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
1987 		xudc_writel(xudc, val, PORTSC);
1988 	}
1989 
1990 	if (xudc->device_state == USB_STATE_SUSPENDED) {
1991 		xudc->device_state = xudc->resume_state;
1992 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1993 		xudc->resume_state = 0;
1994 	}
1995 
1996 	/*
1997 	 * Doorbells may be dropped if they are sent too soon (< ~200ns)
1998 	 * after unpausing the endpoint.  Wait for 500ns just to be safe.
1999 	 */
2000 	ndelay(500);
2001 	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
2002 		tegra_xudc_ep_ring_doorbell(&xudc->ep[i]);
2003 }
2004 
2005 static int tegra_xudc_gadget_wakeup(struct usb_gadget *gadget)
2006 {
2007 	struct tegra_xudc *xudc = to_xudc(gadget);
2008 	unsigned long flags;
2009 	int ret = 0;
2010 	u32 val;
2011 
2012 	spin_lock_irqsave(&xudc->lock, flags);
2013 
2014 	if (xudc->powergated) {
2015 		ret = -ESHUTDOWN;
2016 		goto unlock;
2017 	}
2018 	val = xudc_readl(xudc, PORTPM);
2019 	dev_dbg(xudc->dev, "%s: PORTPM=%#x, speed=%x\n", __func__,
2020 			val, gadget->speed);
2021 
2022 	if (((xudc->gadget.speed <= USB_SPEED_HIGH) &&
2023 	     (val & PORTPM_RWE)) ||
2024 	    ((xudc->gadget.speed == USB_SPEED_SUPER) &&
2025 	     (val & PORTPM_FRWE))) {
2026 		tegra_xudc_resume_device_state(xudc);
2027 
2028 		/* Send Device Notification packet. */
2029 		if (xudc->gadget.speed == USB_SPEED_SUPER) {
2030 			val = DEVNOTIF_LO_TYPE(DEVNOTIF_LO_TYPE_FUNCTION_WAKE)
2031 					     | DEVNOTIF_LO_TRIG;
2032 			xudc_writel(xudc, 0, DEVNOTIF_HI);
2033 			xudc_writel(xudc, val, DEVNOTIF_LO);
2034 		}
2035 	}
2036 
2037 unlock:
2038 	dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2039 	spin_unlock_irqrestore(&xudc->lock, flags);
2040 
2041 	return ret;
2042 }
2043 
2044 static int tegra_xudc_gadget_pullup(struct usb_gadget *gadget, int is_on)
2045 {
2046 	struct tegra_xudc *xudc = to_xudc(gadget);
2047 	unsigned long flags;
2048 	u32 val;
2049 
2050 	pm_runtime_get_sync(xudc->dev);
2051 
2052 	spin_lock_irqsave(&xudc->lock, flags);
2053 
2054 	if (is_on != xudc->pullup) {
2055 		val = xudc_readl(xudc, CTRL);
2056 		if (is_on)
2057 			val |= CTRL_ENABLE;
2058 		else
2059 			val &= ~CTRL_ENABLE;
2060 		xudc_writel(xudc, val, CTRL);
2061 	}
2062 
2063 	xudc->pullup = is_on;
2064 	dev_dbg(xudc->dev, "%s: pullup:%d", __func__, is_on);
2065 
2066 	spin_unlock_irqrestore(&xudc->lock, flags);
2067 
2068 	pm_runtime_put(xudc->dev);
2069 
2070 	return 0;
2071 }
2072 
2073 static int tegra_xudc_gadget_start(struct usb_gadget *gadget,
2074 				   struct usb_gadget_driver *driver)
2075 {
2076 	struct tegra_xudc *xudc = to_xudc(gadget);
2077 	unsigned long flags;
2078 	u32 val;
2079 	int ret;
2080 	unsigned int i;
2081 
2082 	if (!driver)
2083 		return -EINVAL;
2084 
2085 	pm_runtime_get_sync(xudc->dev);
2086 
2087 	spin_lock_irqsave(&xudc->lock, flags);
2088 
2089 	if (xudc->driver) {
2090 		ret = -EBUSY;
2091 		goto unlock;
2092 	}
2093 
2094 	xudc->setup_state = WAIT_FOR_SETUP;
2095 	xudc->device_state = USB_STATE_DEFAULT;
2096 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2097 
2098 	ret = __tegra_xudc_ep_enable(&xudc->ep[0], &tegra_xudc_ep0_desc);
2099 	if (ret < 0)
2100 		goto unlock;
2101 
2102 	val = xudc_readl(xudc, CTRL);
2103 	val |= CTRL_IE | CTRL_LSE;
2104 	xudc_writel(xudc, val, CTRL);
2105 
2106 	val = xudc_readl(xudc, PORTHALT);
2107 	val |= PORTHALT_STCHG_INTR_EN;
2108 	xudc_writel(xudc, val, PORTHALT);
2109 
2110 	if (xudc->pullup) {
2111 		val = xudc_readl(xudc, CTRL);
2112 		val |= CTRL_ENABLE;
2113 		xudc_writel(xudc, val, CTRL);
2114 	}
2115 
2116 	for (i = 0; i < xudc->soc->num_phys; i++)
2117 		if (xudc->usbphy[i])
2118 			otg_set_peripheral(xudc->usbphy[i]->otg, gadget);
2119 
2120 	xudc->driver = driver;
2121 unlock:
2122 	dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2123 	spin_unlock_irqrestore(&xudc->lock, flags);
2124 
2125 	pm_runtime_put(xudc->dev);
2126 
2127 	return ret;
2128 }
2129 
2130 static int tegra_xudc_gadget_stop(struct usb_gadget *gadget)
2131 {
2132 	struct tegra_xudc *xudc = to_xudc(gadget);
2133 	unsigned long flags;
2134 	u32 val;
2135 	unsigned int i;
2136 
2137 	pm_runtime_get_sync(xudc->dev);
2138 
2139 	spin_lock_irqsave(&xudc->lock, flags);
2140 
2141 	for (i = 0; i < xudc->soc->num_phys; i++)
2142 		if (xudc->usbphy[i])
2143 			otg_set_peripheral(xudc->usbphy[i]->otg, NULL);
2144 
2145 	val = xudc_readl(xudc, CTRL);
2146 	val &= ~(CTRL_IE | CTRL_ENABLE);
2147 	xudc_writel(xudc, val, CTRL);
2148 
2149 	__tegra_xudc_ep_disable(&xudc->ep[0]);
2150 
2151 	xudc->driver = NULL;
2152 	dev_dbg(xudc->dev, "Gadget stopped");
2153 
2154 	spin_unlock_irqrestore(&xudc->lock, flags);
2155 
2156 	pm_runtime_put(xudc->dev);
2157 
2158 	return 0;
2159 }
2160 
2161 static int tegra_xudc_gadget_vbus_draw(struct usb_gadget *gadget,
2162 						unsigned int m_a)
2163 {
2164 	struct tegra_xudc *xudc = to_xudc(gadget);
2165 
2166 	dev_dbg(xudc->dev, "%s: %u mA\n", __func__, m_a);
2167 
2168 	if (xudc->curr_usbphy && xudc->curr_usbphy->chg_type == SDP_TYPE)
2169 		return usb_phy_set_power(xudc->curr_usbphy, m_a);
2170 
2171 	return 0;
2172 }
2173 
2174 static int tegra_xudc_set_selfpowered(struct usb_gadget *gadget, int is_on)
2175 {
2176 	struct tegra_xudc *xudc = to_xudc(gadget);
2177 
2178 	dev_dbg(xudc->dev, "%s: %d\n", __func__, is_on);
2179 	xudc->selfpowered = !!is_on;
2180 
2181 	return 0;
2182 }
2183 
2184 static const struct usb_gadget_ops tegra_xudc_gadget_ops = {
2185 	.get_frame = tegra_xudc_gadget_get_frame,
2186 	.wakeup = tegra_xudc_gadget_wakeup,
2187 	.pullup = tegra_xudc_gadget_pullup,
2188 	.udc_start = tegra_xudc_gadget_start,
2189 	.udc_stop = tegra_xudc_gadget_stop,
2190 	.vbus_draw = tegra_xudc_gadget_vbus_draw,
2191 	.set_selfpowered = tegra_xudc_set_selfpowered,
2192 };
2193 
2194 static void no_op_complete(struct usb_ep *ep, struct usb_request *req)
2195 {
2196 }
2197 
2198 static int
2199 tegra_xudc_ep0_queue_status(struct tegra_xudc *xudc,
2200 		void (*cmpl)(struct usb_ep *, struct usb_request *))
2201 {
2202 	xudc->ep0_req->usb_req.buf = NULL;
2203 	xudc->ep0_req->usb_req.dma = 0;
2204 	xudc->ep0_req->usb_req.length = 0;
2205 	xudc->ep0_req->usb_req.complete = cmpl;
2206 	xudc->ep0_req->usb_req.context = xudc;
2207 
2208 	return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2209 }
2210 
2211 static int
2212 tegra_xudc_ep0_queue_data(struct tegra_xudc *xudc, void *buf, size_t len,
2213 		void (*cmpl)(struct usb_ep *, struct usb_request *))
2214 {
2215 	xudc->ep0_req->usb_req.buf = buf;
2216 	xudc->ep0_req->usb_req.length = len;
2217 	xudc->ep0_req->usb_req.complete = cmpl;
2218 	xudc->ep0_req->usb_req.context = xudc;
2219 
2220 	return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2221 }
2222 
2223 static void tegra_xudc_ep0_req_done(struct tegra_xudc *xudc)
2224 {
2225 	switch (xudc->setup_state) {
2226 	case DATA_STAGE_XFER:
2227 		xudc->setup_state = STATUS_STAGE_RECV;
2228 		tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2229 		break;
2230 	case DATA_STAGE_RECV:
2231 		xudc->setup_state = STATUS_STAGE_XFER;
2232 		tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2233 		break;
2234 	default:
2235 		xudc->setup_state = WAIT_FOR_SETUP;
2236 		break;
2237 	}
2238 }
2239 
2240 static int tegra_xudc_ep0_delegate_req(struct tegra_xudc *xudc,
2241 				       struct usb_ctrlrequest *ctrl)
2242 {
2243 	int ret;
2244 
2245 	spin_unlock(&xudc->lock);
2246 	ret = xudc->driver->setup(&xudc->gadget, ctrl);
2247 	spin_lock(&xudc->lock);
2248 
2249 	return ret;
2250 }
2251 
2252 static void set_feature_complete(struct usb_ep *ep, struct usb_request *req)
2253 {
2254 	struct tegra_xudc *xudc = req->context;
2255 
2256 	if (xudc->test_mode_pattern) {
2257 		xudc_writel(xudc, xudc->test_mode_pattern, PORT_TM);
2258 		xudc->test_mode_pattern = 0;
2259 	}
2260 }
2261 
2262 static int tegra_xudc_ep0_set_feature(struct tegra_xudc *xudc,
2263 				      struct usb_ctrlrequest *ctrl)
2264 {
2265 	bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
2266 	u32 feature = le16_to_cpu(ctrl->wValue);
2267 	u32 index = le16_to_cpu(ctrl->wIndex);
2268 	u32 val, ep;
2269 	int ret;
2270 
2271 	if (le16_to_cpu(ctrl->wLength) != 0)
2272 		return -EINVAL;
2273 
2274 	switch (ctrl->bRequestType & USB_RECIP_MASK) {
2275 	case USB_RECIP_DEVICE:
2276 		switch (feature) {
2277 		case USB_DEVICE_REMOTE_WAKEUP:
2278 			if ((xudc->gadget.speed == USB_SPEED_SUPER) ||
2279 			    (xudc->device_state == USB_STATE_DEFAULT))
2280 				return -EINVAL;
2281 
2282 			val = xudc_readl(xudc, PORTPM);
2283 			if (set)
2284 				val |= PORTPM_RWE;
2285 			else
2286 				val &= ~PORTPM_RWE;
2287 
2288 			xudc_writel(xudc, val, PORTPM);
2289 			break;
2290 		case USB_DEVICE_U1_ENABLE:
2291 		case USB_DEVICE_U2_ENABLE:
2292 			if ((xudc->device_state != USB_STATE_CONFIGURED) ||
2293 			    (xudc->gadget.speed != USB_SPEED_SUPER))
2294 				return -EINVAL;
2295 
2296 			val = xudc_readl(xudc, PORTPM);
2297 			if ((feature == USB_DEVICE_U1_ENABLE) &&
2298 			     xudc->soc->u1_enable) {
2299 				if (set)
2300 					val |= PORTPM_U1E;
2301 				else
2302 					val &= ~PORTPM_U1E;
2303 			}
2304 
2305 			if ((feature == USB_DEVICE_U2_ENABLE) &&
2306 			     xudc->soc->u2_enable) {
2307 				if (set)
2308 					val |= PORTPM_U2E;
2309 				else
2310 					val &= ~PORTPM_U2E;
2311 			}
2312 
2313 			xudc_writel(xudc, val, PORTPM);
2314 			break;
2315 		case USB_DEVICE_TEST_MODE:
2316 			if (xudc->gadget.speed != USB_SPEED_HIGH)
2317 				return -EINVAL;
2318 
2319 			if (!set)
2320 				return -EINVAL;
2321 
2322 			xudc->test_mode_pattern = index >> 8;
2323 			break;
2324 		default:
2325 			return -EINVAL;
2326 		}
2327 
2328 		break;
2329 	case USB_RECIP_INTERFACE:
2330 		if (xudc->device_state != USB_STATE_CONFIGURED)
2331 			return -EINVAL;
2332 
2333 		switch (feature) {
2334 		case USB_INTRF_FUNC_SUSPEND:
2335 			if (set) {
2336 				val = xudc_readl(xudc, PORTPM);
2337 
2338 				if (index & USB_INTRF_FUNC_SUSPEND_RW)
2339 					val |= PORTPM_FRWE;
2340 				else
2341 					val &= ~PORTPM_FRWE;
2342 
2343 				xudc_writel(xudc, val, PORTPM);
2344 			}
2345 
2346 			return tegra_xudc_ep0_delegate_req(xudc, ctrl);
2347 		default:
2348 			return -EINVAL;
2349 		}
2350 
2351 		break;
2352 	case USB_RECIP_ENDPOINT:
2353 		ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2354 			((index & USB_DIR_IN) ? 1 : 0);
2355 
2356 		if ((xudc->device_state == USB_STATE_DEFAULT) ||
2357 		    ((xudc->device_state == USB_STATE_ADDRESS) &&
2358 		     (index != 0)))
2359 			return -EINVAL;
2360 
2361 		ret = __tegra_xudc_ep_set_halt(&xudc->ep[ep], set);
2362 		if (ret < 0)
2363 			return ret;
2364 		break;
2365 	default:
2366 		return -EINVAL;
2367 	}
2368 
2369 	return tegra_xudc_ep0_queue_status(xudc, set_feature_complete);
2370 }
2371 
2372 static int tegra_xudc_ep0_get_status(struct tegra_xudc *xudc,
2373 				     struct usb_ctrlrequest *ctrl)
2374 {
2375 	struct tegra_xudc_ep_context *ep_ctx;
2376 	u32 val, ep, index = le16_to_cpu(ctrl->wIndex);
2377 	u16 status = 0;
2378 
2379 	if (!(ctrl->bRequestType & USB_DIR_IN))
2380 		return -EINVAL;
2381 
2382 	if ((le16_to_cpu(ctrl->wValue) != 0) ||
2383 	    (le16_to_cpu(ctrl->wLength) != 2))
2384 		return -EINVAL;
2385 
2386 	switch (ctrl->bRequestType & USB_RECIP_MASK) {
2387 	case USB_RECIP_DEVICE:
2388 		val = xudc_readl(xudc, PORTPM);
2389 
2390 		if (xudc->selfpowered)
2391 			status |= BIT(USB_DEVICE_SELF_POWERED);
2392 
2393 		if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
2394 		    (val & PORTPM_RWE))
2395 			status |= BIT(USB_DEVICE_REMOTE_WAKEUP);
2396 
2397 		if (xudc->gadget.speed == USB_SPEED_SUPER) {
2398 			if (val & PORTPM_U1E)
2399 				status |= BIT(USB_DEV_STAT_U1_ENABLED);
2400 			if (val & PORTPM_U2E)
2401 				status |= BIT(USB_DEV_STAT_U2_ENABLED);
2402 		}
2403 		break;
2404 	case USB_RECIP_INTERFACE:
2405 		if (xudc->gadget.speed == USB_SPEED_SUPER) {
2406 			status |= USB_INTRF_STAT_FUNC_RW_CAP;
2407 			val = xudc_readl(xudc, PORTPM);
2408 			if (val & PORTPM_FRWE)
2409 				status |= USB_INTRF_STAT_FUNC_RW;
2410 		}
2411 		break;
2412 	case USB_RECIP_ENDPOINT:
2413 		ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2414 			((index & USB_DIR_IN) ? 1 : 0);
2415 		ep_ctx = &xudc->ep_context[ep];
2416 
2417 		if ((xudc->device_state != USB_STATE_CONFIGURED) &&
2418 		    ((xudc->device_state != USB_STATE_ADDRESS) || (ep != 0)))
2419 			return -EINVAL;
2420 
2421 		if (ep_ctx_read_state(ep_ctx) == EP_STATE_DISABLED)
2422 			return -EINVAL;
2423 
2424 		if (xudc_readl(xudc, EP_HALT) & BIT(ep))
2425 			status |= BIT(USB_ENDPOINT_HALT);
2426 		break;
2427 	default:
2428 		return -EINVAL;
2429 	}
2430 
2431 	xudc->status_buf = cpu_to_le16(status);
2432 	return tegra_xudc_ep0_queue_data(xudc, &xudc->status_buf,
2433 					 sizeof(xudc->status_buf),
2434 					 no_op_complete);
2435 }
2436 
2437 static void set_sel_complete(struct usb_ep *ep, struct usb_request *req)
2438 {
2439 	/* Nothing to do with SEL values */
2440 }
2441 
2442 static int tegra_xudc_ep0_set_sel(struct tegra_xudc *xudc,
2443 				  struct usb_ctrlrequest *ctrl)
2444 {
2445 	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2446 				     USB_TYPE_STANDARD))
2447 		return -EINVAL;
2448 
2449 	if (xudc->device_state == USB_STATE_DEFAULT)
2450 		return -EINVAL;
2451 
2452 	if ((le16_to_cpu(ctrl->wIndex) != 0) ||
2453 	    (le16_to_cpu(ctrl->wValue) != 0) ||
2454 	    (le16_to_cpu(ctrl->wLength) != 6))
2455 		return -EINVAL;
2456 
2457 	return tegra_xudc_ep0_queue_data(xudc, &xudc->sel_timing,
2458 					 sizeof(xudc->sel_timing),
2459 					 set_sel_complete);
2460 }
2461 
2462 static void set_isoch_delay_complete(struct usb_ep *ep, struct usb_request *req)
2463 {
2464 	/* Nothing to do with isoch delay */
2465 }
2466 
2467 static int tegra_xudc_ep0_set_isoch_delay(struct tegra_xudc *xudc,
2468 					  struct usb_ctrlrequest *ctrl)
2469 {
2470 	u32 delay = le16_to_cpu(ctrl->wValue);
2471 
2472 	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2473 				   USB_TYPE_STANDARD))
2474 		return -EINVAL;
2475 
2476 	if ((delay > 65535) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2477 	    (le16_to_cpu(ctrl->wLength) != 0))
2478 		return -EINVAL;
2479 
2480 	xudc->isoch_delay = delay;
2481 
2482 	return tegra_xudc_ep0_queue_status(xudc, set_isoch_delay_complete);
2483 }
2484 
2485 static void set_address_complete(struct usb_ep *ep, struct usb_request *req)
2486 {
2487 	struct tegra_xudc *xudc = req->context;
2488 
2489 	if ((xudc->device_state == USB_STATE_DEFAULT) &&
2490 	    (xudc->dev_addr != 0)) {
2491 		xudc->device_state = USB_STATE_ADDRESS;
2492 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2493 	} else if ((xudc->device_state == USB_STATE_ADDRESS) &&
2494 		   (xudc->dev_addr == 0)) {
2495 		xudc->device_state = USB_STATE_DEFAULT;
2496 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2497 	}
2498 }
2499 
2500 static int tegra_xudc_ep0_set_address(struct tegra_xudc *xudc,
2501 				      struct usb_ctrlrequest *ctrl)
2502 {
2503 	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2504 	u32 val, addr = le16_to_cpu(ctrl->wValue);
2505 
2506 	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2507 				     USB_TYPE_STANDARD))
2508 		return -EINVAL;
2509 
2510 	if ((addr > 127) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2511 	    (le16_to_cpu(ctrl->wLength) != 0))
2512 		return -EINVAL;
2513 
2514 	if (xudc->device_state == USB_STATE_CONFIGURED)
2515 		return -EINVAL;
2516 
2517 	dev_dbg(xudc->dev, "set address: %u\n", addr);
2518 
2519 	xudc->dev_addr = addr;
2520 	val = xudc_readl(xudc, CTRL);
2521 	val &= ~(CTRL_DEVADDR_MASK);
2522 	val |= CTRL_DEVADDR(addr);
2523 	xudc_writel(xudc, val, CTRL);
2524 
2525 	ep_ctx_write_devaddr(ep0->context, addr);
2526 
2527 	return tegra_xudc_ep0_queue_status(xudc, set_address_complete);
2528 }
2529 
2530 static int tegra_xudc_ep0_standard_req(struct tegra_xudc *xudc,
2531 				      struct usb_ctrlrequest *ctrl)
2532 {
2533 	int ret;
2534 
2535 	switch (ctrl->bRequest) {
2536 	case USB_REQ_GET_STATUS:
2537 		dev_dbg(xudc->dev, "USB_REQ_GET_STATUS\n");
2538 		ret = tegra_xudc_ep0_get_status(xudc, ctrl);
2539 		break;
2540 	case USB_REQ_SET_ADDRESS:
2541 		dev_dbg(xudc->dev, "USB_REQ_SET_ADDRESS\n");
2542 		ret = tegra_xudc_ep0_set_address(xudc, ctrl);
2543 		break;
2544 	case USB_REQ_SET_SEL:
2545 		dev_dbg(xudc->dev, "USB_REQ_SET_SEL\n");
2546 		ret = tegra_xudc_ep0_set_sel(xudc, ctrl);
2547 		break;
2548 	case USB_REQ_SET_ISOCH_DELAY:
2549 		dev_dbg(xudc->dev, "USB_REQ_SET_ISOCH_DELAY\n");
2550 		ret = tegra_xudc_ep0_set_isoch_delay(xudc, ctrl);
2551 		break;
2552 	case USB_REQ_CLEAR_FEATURE:
2553 	case USB_REQ_SET_FEATURE:
2554 		dev_dbg(xudc->dev, "USB_REQ_CLEAR/SET_FEATURE\n");
2555 		ret = tegra_xudc_ep0_set_feature(xudc, ctrl);
2556 		break;
2557 	case USB_REQ_SET_CONFIGURATION:
2558 		dev_dbg(xudc->dev, "USB_REQ_SET_CONFIGURATION\n");
2559 		/*
2560 		 * In theory we need to clear RUN bit before status stage of
2561 		 * deconfig request sent, but this seems to be causing problems.
2562 		 * Clear RUN once all endpoints are disabled instead.
2563 		 */
2564 		fallthrough;
2565 	default:
2566 		ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2567 		break;
2568 	}
2569 
2570 	return ret;
2571 }
2572 
2573 static void tegra_xudc_handle_ep0_setup_packet(struct tegra_xudc *xudc,
2574 					       struct usb_ctrlrequest *ctrl,
2575 					       u16 seq_num)
2576 {
2577 	int ret;
2578 
2579 	xudc->setup_seq_num = seq_num;
2580 
2581 	/* Ensure EP0 is unhalted. */
2582 	ep_unhalt(xudc, 0);
2583 
2584 	/*
2585 	 * On Tegra210, setup packets with sequence numbers 0xfffe or 0xffff
2586 	 * are invalid.  Halt EP0 until we get a valid packet.
2587 	 */
2588 	if (xudc->soc->invalid_seq_num &&
2589 	    (seq_num == 0xfffe || seq_num == 0xffff)) {
2590 		dev_warn(xudc->dev, "invalid sequence number detected\n");
2591 		ep_halt(xudc, 0);
2592 		return;
2593 	}
2594 
2595 	if (ctrl->wLength)
2596 		xudc->setup_state = (ctrl->bRequestType & USB_DIR_IN) ?
2597 			DATA_STAGE_XFER :  DATA_STAGE_RECV;
2598 	else
2599 		xudc->setup_state = STATUS_STAGE_XFER;
2600 
2601 	if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
2602 		ret = tegra_xudc_ep0_standard_req(xudc, ctrl);
2603 	else
2604 		ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2605 
2606 	if (ret < 0) {
2607 		dev_warn(xudc->dev, "setup request failed: %d\n", ret);
2608 		xudc->setup_state = WAIT_FOR_SETUP;
2609 		ep_halt(xudc, 0);
2610 	}
2611 }
2612 
2613 static void tegra_xudc_handle_ep0_event(struct tegra_xudc *xudc,
2614 					struct tegra_xudc_trb *event)
2615 {
2616 	struct usb_ctrlrequest *ctrl = (struct usb_ctrlrequest *)event;
2617 	u16 seq_num = trb_read_seq_num(event);
2618 
2619 	if (xudc->setup_state != WAIT_FOR_SETUP) {
2620 		/*
2621 		 * The controller is in the process of handling another
2622 		 * setup request.  Queue subsequent requests and handle
2623 		 * the last one once the controller reports a sequence
2624 		 * number error.
2625 		 */
2626 		memcpy(&xudc->setup_packet.ctrl_req, ctrl, sizeof(*ctrl));
2627 		xudc->setup_packet.seq_num = seq_num;
2628 		xudc->queued_setup_packet = true;
2629 	} else {
2630 		tegra_xudc_handle_ep0_setup_packet(xudc, ctrl, seq_num);
2631 	}
2632 }
2633 
2634 static struct tegra_xudc_request *
2635 trb_to_request(struct tegra_xudc_ep *ep, struct tegra_xudc_trb *trb)
2636 {
2637 	struct tegra_xudc_request *req;
2638 
2639 	list_for_each_entry(req, &ep->queue, list) {
2640 		if (!req->trbs_queued)
2641 			break;
2642 
2643 		if (trb_in_request(ep, req, trb))
2644 			return req;
2645 	}
2646 
2647 	return NULL;
2648 }
2649 
2650 static void tegra_xudc_handle_transfer_completion(struct tegra_xudc *xudc,
2651 						  struct tegra_xudc_ep *ep,
2652 						  struct tegra_xudc_trb *event)
2653 {
2654 	struct tegra_xudc_request *req;
2655 	struct tegra_xudc_trb *trb;
2656 	bool short_packet;
2657 
2658 	short_packet = (trb_read_cmpl_code(event) ==
2659 			TRB_CMPL_CODE_SHORT_PACKET);
2660 
2661 	trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2662 	req = trb_to_request(ep, trb);
2663 
2664 	/*
2665 	 * TDs are complete on short packet or when the completed TRB is the
2666 	 * last TRB in the TD (the CHAIN bit is unset).
2667 	 */
2668 	if (req && (short_packet || (!trb_read_chain(trb) &&
2669 		(req->trbs_needed == req->trbs_queued)))) {
2670 		struct tegra_xudc_trb *last = req->last_trb;
2671 		unsigned int residual;
2672 
2673 		residual = trb_read_transfer_len(event);
2674 		req->usb_req.actual = req->usb_req.length - residual;
2675 
2676 		dev_dbg(xudc->dev, "bytes transferred %u / %u\n",
2677 			req->usb_req.actual, req->usb_req.length);
2678 
2679 		tegra_xudc_req_done(ep, req, 0);
2680 
2681 		if (ep->desc && usb_endpoint_xfer_control(ep->desc))
2682 			tegra_xudc_ep0_req_done(xudc);
2683 
2684 		/*
2685 		 * Advance the dequeue pointer past the end of the current TD
2686 		 * on short packet completion.
2687 		 */
2688 		if (short_packet) {
2689 			ep->deq_ptr = (last - ep->transfer_ring) + 1;
2690 			if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2691 				ep->deq_ptr = 0;
2692 		}
2693 	} else if (!req) {
2694 		dev_warn(xudc->dev, "transfer event on dequeued request\n");
2695 	}
2696 
2697 	if (ep->desc)
2698 		tegra_xudc_ep_kick_queue(ep);
2699 }
2700 
2701 static void tegra_xudc_handle_transfer_event(struct tegra_xudc *xudc,
2702 					     struct tegra_xudc_trb *event)
2703 {
2704 	unsigned int ep_index = trb_read_endpoint_id(event);
2705 	struct tegra_xudc_ep *ep = &xudc->ep[ep_index];
2706 	struct tegra_xudc_trb *trb;
2707 	u16 comp_code;
2708 
2709 	if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
2710 		dev_warn(xudc->dev, "transfer event on disabled EP %u\n",
2711 			 ep_index);
2712 		return;
2713 	}
2714 
2715 	/* Update transfer ring dequeue pointer. */
2716 	trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2717 	comp_code = trb_read_cmpl_code(event);
2718 	if (comp_code != TRB_CMPL_CODE_BABBLE_DETECTED_ERR) {
2719 		ep->deq_ptr = (trb - ep->transfer_ring) + 1;
2720 
2721 		if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2722 			ep->deq_ptr = 0;
2723 		ep->ring_full = false;
2724 	}
2725 
2726 	switch (comp_code) {
2727 	case TRB_CMPL_CODE_SUCCESS:
2728 	case TRB_CMPL_CODE_SHORT_PACKET:
2729 		tegra_xudc_handle_transfer_completion(xudc, ep, event);
2730 		break;
2731 	case TRB_CMPL_CODE_HOST_REJECTED:
2732 		dev_info(xudc->dev, "stream rejected on EP %u\n", ep_index);
2733 
2734 		ep->stream_rejected = true;
2735 		break;
2736 	case TRB_CMPL_CODE_PRIME_PIPE_RECEIVED:
2737 		dev_info(xudc->dev, "prime pipe received on EP %u\n", ep_index);
2738 
2739 		if (ep->stream_rejected) {
2740 			ep->stream_rejected = false;
2741 			/*
2742 			 * An EP is stopped when a stream is rejected.  Wait
2743 			 * for the EP to report that it is stopped and then
2744 			 * un-stop it.
2745 			 */
2746 			ep_wait_for_stopped(xudc, ep_index);
2747 		}
2748 		tegra_xudc_ep_ring_doorbell(ep);
2749 		break;
2750 	case TRB_CMPL_CODE_BABBLE_DETECTED_ERR:
2751 		/*
2752 		 * Wait for the EP to be stopped so the controller stops
2753 		 * processing doorbells.
2754 		 */
2755 		ep_wait_for_stopped(xudc, ep_index);
2756 		ep->enq_ptr = ep->deq_ptr;
2757 		tegra_xudc_ep_nuke(ep, -EIO);
2758 		fallthrough;
2759 	case TRB_CMPL_CODE_STREAM_NUMP_ERROR:
2760 	case TRB_CMPL_CODE_CTRL_DIR_ERR:
2761 	case TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR:
2762 	case TRB_CMPL_CODE_RING_UNDERRUN:
2763 	case TRB_CMPL_CODE_RING_OVERRUN:
2764 	case TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN:
2765 	case TRB_CMPL_CODE_USB_TRANS_ERR:
2766 	case TRB_CMPL_CODE_TRB_ERR:
2767 		dev_err(xudc->dev, "completion error %#x on EP %u\n",
2768 			comp_code, ep_index);
2769 
2770 		ep_halt(xudc, ep_index);
2771 		break;
2772 	case TRB_CMPL_CODE_CTRL_SEQNUM_ERR:
2773 		dev_info(xudc->dev, "sequence number error\n");
2774 
2775 		/*
2776 		 * Kill any queued control request and skip to the last
2777 		 * setup packet we received.
2778 		 */
2779 		tegra_xudc_ep_nuke(ep, -EINVAL);
2780 		xudc->setup_state = WAIT_FOR_SETUP;
2781 		if (!xudc->queued_setup_packet)
2782 			break;
2783 
2784 		tegra_xudc_handle_ep0_setup_packet(xudc,
2785 						   &xudc->setup_packet.ctrl_req,
2786 						   xudc->setup_packet.seq_num);
2787 		xudc->queued_setup_packet = false;
2788 		break;
2789 	case TRB_CMPL_CODE_STOPPED:
2790 		dev_dbg(xudc->dev, "stop completion code on EP %u\n",
2791 			ep_index);
2792 
2793 		/* Disconnected. */
2794 		tegra_xudc_ep_nuke(ep, -ECONNREFUSED);
2795 		break;
2796 	default:
2797 		dev_dbg(xudc->dev, "completion event %#x on EP %u\n",
2798 			comp_code, ep_index);
2799 		break;
2800 	}
2801 }
2802 
2803 static void tegra_xudc_reset(struct tegra_xudc *xudc)
2804 {
2805 	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2806 	dma_addr_t deq_ptr;
2807 	unsigned int i;
2808 
2809 	xudc->setup_state = WAIT_FOR_SETUP;
2810 	xudc->device_state = USB_STATE_DEFAULT;
2811 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2812 
2813 	ep_unpause_all(xudc);
2814 
2815 	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
2816 		tegra_xudc_ep_nuke(&xudc->ep[i], -ESHUTDOWN);
2817 
2818 	/*
2819 	 * Reset sequence number and dequeue pointer to flush the transfer
2820 	 * ring.
2821 	 */
2822 	ep0->deq_ptr = ep0->enq_ptr;
2823 	ep0->ring_full = false;
2824 
2825 	xudc->setup_seq_num = 0;
2826 	xudc->queued_setup_packet = false;
2827 
2828 	ep_ctx_write_rsvd(ep0->context, 0);
2829 	ep_ctx_write_partial_td(ep0->context, 0);
2830 	ep_ctx_write_splitxstate(ep0->context, 0);
2831 	ep_ctx_write_seq_num(ep0->context, 0);
2832 
2833 	deq_ptr = trb_virt_to_phys(ep0, &ep0->transfer_ring[ep0->deq_ptr]);
2834 
2835 	if (!dma_mapping_error(xudc->dev, deq_ptr)) {
2836 		ep_ctx_write_deq_ptr(ep0->context, deq_ptr);
2837 		ep_ctx_write_dcs(ep0->context, ep0->pcs);
2838 	}
2839 
2840 	ep_unhalt_all(xudc);
2841 	ep_reload(xudc, 0);
2842 	ep_unpause(xudc, 0);
2843 }
2844 
2845 static void tegra_xudc_port_connect(struct tegra_xudc *xudc)
2846 {
2847 	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2848 	u16 maxpacket;
2849 	u32 val;
2850 
2851 	val = (xudc_readl(xudc, PORTSC) & PORTSC_PS_MASK) >> PORTSC_PS_SHIFT;
2852 	switch (val) {
2853 	case PORTSC_PS_LS:
2854 		xudc->gadget.speed = USB_SPEED_LOW;
2855 		break;
2856 	case PORTSC_PS_FS:
2857 		xudc->gadget.speed = USB_SPEED_FULL;
2858 		break;
2859 	case PORTSC_PS_HS:
2860 		xudc->gadget.speed = USB_SPEED_HIGH;
2861 		break;
2862 	case PORTSC_PS_SS:
2863 		xudc->gadget.speed = USB_SPEED_SUPER;
2864 		break;
2865 	default:
2866 		xudc->gadget.speed = USB_SPEED_UNKNOWN;
2867 		break;
2868 	}
2869 
2870 	xudc->device_state = USB_STATE_DEFAULT;
2871 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2872 
2873 	xudc->setup_state = WAIT_FOR_SETUP;
2874 
2875 	if (xudc->gadget.speed == USB_SPEED_SUPER)
2876 		maxpacket = 512;
2877 	else
2878 		maxpacket = 64;
2879 
2880 	ep_ctx_write_max_packet_size(ep0->context, maxpacket);
2881 	tegra_xudc_ep0_desc.wMaxPacketSize = cpu_to_le16(maxpacket);
2882 	usb_ep_set_maxpacket_limit(&ep0->usb_ep, maxpacket);
2883 
2884 	if (!xudc->soc->u1_enable) {
2885 		val = xudc_readl(xudc, PORTPM);
2886 		val &= ~(PORTPM_U1TIMEOUT_MASK);
2887 		xudc_writel(xudc, val, PORTPM);
2888 	}
2889 
2890 	if (!xudc->soc->u2_enable) {
2891 		val = xudc_readl(xudc, PORTPM);
2892 		val &= ~(PORTPM_U2TIMEOUT_MASK);
2893 		xudc_writel(xudc, val, PORTPM);
2894 	}
2895 
2896 	if (xudc->gadget.speed <= USB_SPEED_HIGH) {
2897 		val = xudc_readl(xudc, PORTPM);
2898 		val &= ~(PORTPM_L1S_MASK);
2899 		if (xudc->soc->lpm_enable)
2900 			val |= PORTPM_L1S(PORTPM_L1S_ACCEPT);
2901 		else
2902 			val |= PORTPM_L1S(PORTPM_L1S_NYET);
2903 		xudc_writel(xudc, val, PORTPM);
2904 	}
2905 
2906 	val = xudc_readl(xudc, ST);
2907 	if (val & ST_RC)
2908 		xudc_writel(xudc, ST_RC, ST);
2909 }
2910 
2911 static void tegra_xudc_port_disconnect(struct tegra_xudc *xudc)
2912 {
2913 	tegra_xudc_reset(xudc);
2914 
2915 	if (xudc->driver && xudc->driver->disconnect) {
2916 		spin_unlock(&xudc->lock);
2917 		xudc->driver->disconnect(&xudc->gadget);
2918 		spin_lock(&xudc->lock);
2919 	}
2920 
2921 	xudc->device_state = USB_STATE_NOTATTACHED;
2922 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2923 
2924 	complete(&xudc->disconnect_complete);
2925 }
2926 
2927 static void tegra_xudc_port_reset(struct tegra_xudc *xudc)
2928 {
2929 	tegra_xudc_reset(xudc);
2930 
2931 	if (xudc->driver) {
2932 		spin_unlock(&xudc->lock);
2933 		usb_gadget_udc_reset(&xudc->gadget, xudc->driver);
2934 		spin_lock(&xudc->lock);
2935 	}
2936 
2937 	tegra_xudc_port_connect(xudc);
2938 }
2939 
2940 static void tegra_xudc_port_suspend(struct tegra_xudc *xudc)
2941 {
2942 	dev_dbg(xudc->dev, "port suspend\n");
2943 
2944 	xudc->resume_state = xudc->device_state;
2945 	xudc->device_state = USB_STATE_SUSPENDED;
2946 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2947 
2948 	if (xudc->driver->suspend) {
2949 		spin_unlock(&xudc->lock);
2950 		xudc->driver->suspend(&xudc->gadget);
2951 		spin_lock(&xudc->lock);
2952 	}
2953 }
2954 
2955 static void tegra_xudc_port_resume(struct tegra_xudc *xudc)
2956 {
2957 	dev_dbg(xudc->dev, "port resume\n");
2958 
2959 	tegra_xudc_resume_device_state(xudc);
2960 
2961 	if (xudc->driver->resume) {
2962 		spin_unlock(&xudc->lock);
2963 		xudc->driver->resume(&xudc->gadget);
2964 		spin_lock(&xudc->lock);
2965 	}
2966 }
2967 
2968 static inline void clear_port_change(struct tegra_xudc *xudc, u32 flag)
2969 {
2970 	u32 val;
2971 
2972 	val = xudc_readl(xudc, PORTSC);
2973 	val &= ~PORTSC_CHANGE_MASK;
2974 	val |= flag;
2975 	xudc_writel(xudc, val, PORTSC);
2976 }
2977 
2978 static void __tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
2979 {
2980 	u32 portsc, porthalt;
2981 
2982 	porthalt = xudc_readl(xudc, PORTHALT);
2983 	if ((porthalt & PORTHALT_STCHG_REQ) &&
2984 	    (porthalt & PORTHALT_HALT_LTSSM)) {
2985 		dev_dbg(xudc->dev, "STCHG_REQ, PORTHALT = %#x\n", porthalt);
2986 		porthalt &= ~PORTHALT_HALT_LTSSM;
2987 		xudc_writel(xudc, porthalt, PORTHALT);
2988 	}
2989 
2990 	portsc = xudc_readl(xudc, PORTSC);
2991 	if ((portsc & PORTSC_PRC) && (portsc & PORTSC_PR)) {
2992 		dev_dbg(xudc->dev, "PRC, PR, PORTSC = %#x\n", portsc);
2993 		clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
2994 #define TOGGLE_VBUS_WAIT_MS 100
2995 		if (xudc->soc->port_reset_quirk) {
2996 			schedule_delayed_work(&xudc->port_reset_war_work,
2997 				msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
2998 			xudc->wait_for_sec_prc = 1;
2999 		}
3000 	}
3001 
3002 	if ((portsc & PORTSC_PRC) && !(portsc & PORTSC_PR)) {
3003 		dev_dbg(xudc->dev, "PRC, Not PR, PORTSC = %#x\n", portsc);
3004 		clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
3005 		tegra_xudc_port_reset(xudc);
3006 		cancel_delayed_work(&xudc->port_reset_war_work);
3007 		xudc->wait_for_sec_prc = 0;
3008 	}
3009 
3010 	portsc = xudc_readl(xudc, PORTSC);
3011 	if (portsc & PORTSC_WRC) {
3012 		dev_dbg(xudc->dev, "WRC, PORTSC = %#x\n", portsc);
3013 		clear_port_change(xudc, PORTSC_WRC | PORTSC_PED);
3014 		if (!(xudc_readl(xudc, PORTSC) & PORTSC_WPR))
3015 			tegra_xudc_port_reset(xudc);
3016 	}
3017 
3018 	portsc = xudc_readl(xudc, PORTSC);
3019 	if (portsc & PORTSC_CSC) {
3020 		dev_dbg(xudc->dev, "CSC, PORTSC = %#x\n", portsc);
3021 		clear_port_change(xudc, PORTSC_CSC);
3022 
3023 		if (portsc & PORTSC_CCS)
3024 			tegra_xudc_port_connect(xudc);
3025 		else
3026 			tegra_xudc_port_disconnect(xudc);
3027 
3028 		if (xudc->wait_csc) {
3029 			cancel_delayed_work(&xudc->plc_reset_work);
3030 			xudc->wait_csc = false;
3031 		}
3032 	}
3033 
3034 	portsc = xudc_readl(xudc, PORTSC);
3035 	if (portsc & PORTSC_PLC) {
3036 		u32 pls = (portsc & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT;
3037 
3038 		dev_dbg(xudc->dev, "PLC, PORTSC = %#x\n", portsc);
3039 		clear_port_change(xudc, PORTSC_PLC);
3040 		switch (pls) {
3041 		case PORTSC_PLS_U3:
3042 			tegra_xudc_port_suspend(xudc);
3043 			break;
3044 		case PORTSC_PLS_U0:
3045 			if (xudc->gadget.speed < USB_SPEED_SUPER)
3046 				tegra_xudc_port_resume(xudc);
3047 			break;
3048 		case PORTSC_PLS_RESUME:
3049 			if (xudc->gadget.speed == USB_SPEED_SUPER)
3050 				tegra_xudc_port_resume(xudc);
3051 			break;
3052 		case PORTSC_PLS_INACTIVE:
3053 			schedule_delayed_work(&xudc->plc_reset_work,
3054 					msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
3055 			xudc->wait_csc = true;
3056 			break;
3057 		default:
3058 			break;
3059 		}
3060 	}
3061 
3062 	if (portsc & PORTSC_CEC) {
3063 		dev_warn(xudc->dev, "CEC, PORTSC = %#x\n", portsc);
3064 		clear_port_change(xudc, PORTSC_CEC);
3065 	}
3066 
3067 	dev_dbg(xudc->dev, "PORTSC = %#x\n", xudc_readl(xudc, PORTSC));
3068 }
3069 
3070 static void tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
3071 {
3072 	while ((xudc_readl(xudc, PORTSC) & PORTSC_CHANGE_MASK) ||
3073 	       (xudc_readl(xudc, PORTHALT) & PORTHALT_STCHG_REQ))
3074 		__tegra_xudc_handle_port_status(xudc);
3075 }
3076 
3077 static void tegra_xudc_handle_event(struct tegra_xudc *xudc,
3078 				    struct tegra_xudc_trb *event)
3079 {
3080 	u32 type = trb_read_type(event);
3081 
3082 	dump_trb(xudc, "EVENT", event);
3083 
3084 	switch (type) {
3085 	case TRB_TYPE_PORT_STATUS_CHANGE_EVENT:
3086 		tegra_xudc_handle_port_status(xudc);
3087 		break;
3088 	case TRB_TYPE_TRANSFER_EVENT:
3089 		tegra_xudc_handle_transfer_event(xudc, event);
3090 		break;
3091 	case TRB_TYPE_SETUP_PACKET_EVENT:
3092 		tegra_xudc_handle_ep0_event(xudc, event);
3093 		break;
3094 	default:
3095 		dev_info(xudc->dev, "Unrecognized TRB type = %#x\n", type);
3096 		break;
3097 	}
3098 }
3099 
3100 static void tegra_xudc_process_event_ring(struct tegra_xudc *xudc)
3101 {
3102 	struct tegra_xudc_trb *event;
3103 	dma_addr_t erdp;
3104 
3105 	while (true) {
3106 		event = xudc->event_ring[xudc->event_ring_index] +
3107 			xudc->event_ring_deq_ptr;
3108 
3109 		if (trb_read_cycle(event) != xudc->ccs)
3110 			break;
3111 
3112 		tegra_xudc_handle_event(xudc, event);
3113 
3114 		xudc->event_ring_deq_ptr++;
3115 		if (xudc->event_ring_deq_ptr == XUDC_EVENT_RING_SIZE) {
3116 			xudc->event_ring_deq_ptr = 0;
3117 			xudc->event_ring_index++;
3118 		}
3119 
3120 		if (xudc->event_ring_index == XUDC_NR_EVENT_RINGS) {
3121 			xudc->event_ring_index = 0;
3122 			xudc->ccs = !xudc->ccs;
3123 		}
3124 	}
3125 
3126 	erdp = xudc->event_ring_phys[xudc->event_ring_index] +
3127 		xudc->event_ring_deq_ptr * sizeof(*event);
3128 
3129 	xudc_writel(xudc, upper_32_bits(erdp), ERDPHI);
3130 	xudc_writel(xudc, lower_32_bits(erdp) | ERDPLO_EHB, ERDPLO);
3131 }
3132 
3133 static irqreturn_t tegra_xudc_irq(int irq, void *data)
3134 {
3135 	struct tegra_xudc *xudc = data;
3136 	unsigned long flags;
3137 	u32 val;
3138 
3139 	val = xudc_readl(xudc, ST);
3140 	if (!(val & ST_IP))
3141 		return IRQ_NONE;
3142 	xudc_writel(xudc, ST_IP, ST);
3143 
3144 	spin_lock_irqsave(&xudc->lock, flags);
3145 	tegra_xudc_process_event_ring(xudc);
3146 	spin_unlock_irqrestore(&xudc->lock, flags);
3147 
3148 	return IRQ_HANDLED;
3149 }
3150 
3151 static int tegra_xudc_alloc_ep(struct tegra_xudc *xudc, unsigned int index)
3152 {
3153 	struct tegra_xudc_ep *ep = &xudc->ep[index];
3154 
3155 	ep->xudc = xudc;
3156 	ep->index = index;
3157 	ep->context = &xudc->ep_context[index];
3158 	INIT_LIST_HEAD(&ep->queue);
3159 
3160 	/*
3161 	 * EP1 would be the input endpoint corresponding to EP0, but since
3162 	 * EP0 is bi-directional, EP1 is unused.
3163 	 */
3164 	if (index == 1)
3165 		return 0;
3166 
3167 	ep->transfer_ring = dma_pool_alloc(xudc->transfer_ring_pool,
3168 					   GFP_KERNEL,
3169 					   &ep->transfer_ring_phys);
3170 	if (!ep->transfer_ring)
3171 		return -ENOMEM;
3172 
3173 	if (index) {
3174 		snprintf(ep->name, sizeof(ep->name), "ep%u%s", index / 2,
3175 			 (index % 2 == 0) ? "out" : "in");
3176 		ep->usb_ep.name = ep->name;
3177 		usb_ep_set_maxpacket_limit(&ep->usb_ep, 1024);
3178 		ep->usb_ep.max_streams = 16;
3179 		ep->usb_ep.ops = &tegra_xudc_ep_ops;
3180 		ep->usb_ep.caps.type_bulk = true;
3181 		ep->usb_ep.caps.type_int = true;
3182 		if (index & 1)
3183 			ep->usb_ep.caps.dir_in = true;
3184 		else
3185 			ep->usb_ep.caps.dir_out = true;
3186 		list_add_tail(&ep->usb_ep.ep_list, &xudc->gadget.ep_list);
3187 	} else {
3188 		strscpy(ep->name, "ep0", 3);
3189 		ep->usb_ep.name = ep->name;
3190 		usb_ep_set_maxpacket_limit(&ep->usb_ep, 512);
3191 		ep->usb_ep.ops = &tegra_xudc_ep0_ops;
3192 		ep->usb_ep.caps.type_control = true;
3193 		ep->usb_ep.caps.dir_in = true;
3194 		ep->usb_ep.caps.dir_out = true;
3195 	}
3196 
3197 	return 0;
3198 }
3199 
3200 static void tegra_xudc_free_ep(struct tegra_xudc *xudc, unsigned int index)
3201 {
3202 	struct tegra_xudc_ep *ep = &xudc->ep[index];
3203 
3204 	/*
3205 	 * EP1 would be the input endpoint corresponding to EP0, but since
3206 	 * EP0 is bi-directional, EP1 is unused.
3207 	 */
3208 	if (index == 1)
3209 		return;
3210 
3211 	dma_pool_free(xudc->transfer_ring_pool, ep->transfer_ring,
3212 		      ep->transfer_ring_phys);
3213 }
3214 
3215 static int tegra_xudc_alloc_eps(struct tegra_xudc *xudc)
3216 {
3217 	struct usb_request *req;
3218 	unsigned int i;
3219 	int err;
3220 
3221 	xudc->ep_context =
3222 		dma_alloc_coherent(xudc->dev, XUDC_NR_EPS *
3223 				    sizeof(*xudc->ep_context),
3224 				    &xudc->ep_context_phys, GFP_KERNEL);
3225 	if (!xudc->ep_context)
3226 		return -ENOMEM;
3227 
3228 	xudc->transfer_ring_pool =
3229 		dmam_pool_create(dev_name(xudc->dev), xudc->dev,
3230 				 XUDC_TRANSFER_RING_SIZE *
3231 				 sizeof(struct tegra_xudc_trb),
3232 				 sizeof(struct tegra_xudc_trb), 0);
3233 	if (!xudc->transfer_ring_pool) {
3234 		err = -ENOMEM;
3235 		goto free_ep_context;
3236 	}
3237 
3238 	INIT_LIST_HEAD(&xudc->gadget.ep_list);
3239 	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
3240 		err = tegra_xudc_alloc_ep(xudc, i);
3241 		if (err < 0)
3242 			goto free_eps;
3243 	}
3244 
3245 	req = tegra_xudc_ep_alloc_request(&xudc->ep[0].usb_ep, GFP_KERNEL);
3246 	if (!req) {
3247 		err = -ENOMEM;
3248 		goto free_eps;
3249 	}
3250 	xudc->ep0_req = to_xudc_req(req);
3251 
3252 	return 0;
3253 
3254 free_eps:
3255 	for (; i > 0; i--)
3256 		tegra_xudc_free_ep(xudc, i - 1);
3257 free_ep_context:
3258 	dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3259 			  xudc->ep_context, xudc->ep_context_phys);
3260 	return err;
3261 }
3262 
3263 static void tegra_xudc_init_eps(struct tegra_xudc *xudc)
3264 {
3265 	xudc_writel(xudc, lower_32_bits(xudc->ep_context_phys), ECPLO);
3266 	xudc_writel(xudc, upper_32_bits(xudc->ep_context_phys), ECPHI);
3267 }
3268 
3269 static void tegra_xudc_free_eps(struct tegra_xudc *xudc)
3270 {
3271 	unsigned int i;
3272 
3273 	tegra_xudc_ep_free_request(&xudc->ep[0].usb_ep,
3274 				   &xudc->ep0_req->usb_req);
3275 
3276 	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
3277 		tegra_xudc_free_ep(xudc, i);
3278 
3279 	dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3280 			  xudc->ep_context, xudc->ep_context_phys);
3281 }
3282 
3283 static int tegra_xudc_alloc_event_ring(struct tegra_xudc *xudc)
3284 {
3285 	unsigned int i;
3286 
3287 	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3288 		xudc->event_ring[i] =
3289 			dma_alloc_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3290 					   sizeof(*xudc->event_ring[i]),
3291 					   &xudc->event_ring_phys[i],
3292 					   GFP_KERNEL);
3293 		if (!xudc->event_ring[i])
3294 			goto free_dma;
3295 	}
3296 
3297 	return 0;
3298 
3299 free_dma:
3300 	for (; i > 0; i--) {
3301 		dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3302 				  sizeof(*xudc->event_ring[i - 1]),
3303 				  xudc->event_ring[i - 1],
3304 				  xudc->event_ring_phys[i - 1]);
3305 	}
3306 	return -ENOMEM;
3307 }
3308 
3309 static void tegra_xudc_init_event_ring(struct tegra_xudc *xudc)
3310 {
3311 	unsigned int i;
3312 	u32 val;
3313 
3314 	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3315 		memset(xudc->event_ring[i], 0, XUDC_EVENT_RING_SIZE *
3316 		       sizeof(*xudc->event_ring[i]));
3317 
3318 		val = xudc_readl(xudc, ERSTSZ);
3319 		val &= ~(ERSTSZ_ERSTXSZ_MASK << ERSTSZ_ERSTXSZ_SHIFT(i));
3320 		val |= XUDC_EVENT_RING_SIZE << ERSTSZ_ERSTXSZ_SHIFT(i);
3321 		xudc_writel(xudc, val, ERSTSZ);
3322 
3323 		xudc_writel(xudc, lower_32_bits(xudc->event_ring_phys[i]),
3324 			    ERSTXBALO(i));
3325 		xudc_writel(xudc, upper_32_bits(xudc->event_ring_phys[i]),
3326 			    ERSTXBAHI(i));
3327 	}
3328 
3329 	val = lower_32_bits(xudc->event_ring_phys[0]);
3330 	xudc_writel(xudc, val, ERDPLO);
3331 	val |= EREPLO_ECS;
3332 	xudc_writel(xudc, val, EREPLO);
3333 
3334 	val = upper_32_bits(xudc->event_ring_phys[0]);
3335 	xudc_writel(xudc, val, ERDPHI);
3336 	xudc_writel(xudc, val, EREPHI);
3337 
3338 	xudc->ccs = true;
3339 	xudc->event_ring_index = 0;
3340 	xudc->event_ring_deq_ptr = 0;
3341 }
3342 
3343 static void tegra_xudc_free_event_ring(struct tegra_xudc *xudc)
3344 {
3345 	unsigned int i;
3346 
3347 	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3348 		dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3349 				  sizeof(*xudc->event_ring[i]),
3350 				  xudc->event_ring[i],
3351 				  xudc->event_ring_phys[i]);
3352 	}
3353 }
3354 
3355 static void tegra_xudc_fpci_ipfs_init(struct tegra_xudc *xudc)
3356 {
3357 	u32 val;
3358 
3359 	if (xudc->soc->has_ipfs) {
3360 		val = ipfs_readl(xudc, XUSB_DEV_CONFIGURATION_0);
3361 		val |= XUSB_DEV_CONFIGURATION_0_EN_FPCI;
3362 		ipfs_writel(xudc, val, XUSB_DEV_CONFIGURATION_0);
3363 		usleep_range(10, 15);
3364 	}
3365 
3366 	/* Enable bus master */
3367 	val = XUSB_DEV_CFG_1_IO_SPACE_EN | XUSB_DEV_CFG_1_MEMORY_SPACE_EN |
3368 		XUSB_DEV_CFG_1_BUS_MASTER_EN;
3369 	fpci_writel(xudc, val, XUSB_DEV_CFG_1);
3370 
3371 	/* Program BAR0 space */
3372 	val = fpci_readl(xudc, XUSB_DEV_CFG_4);
3373 	val &= ~(XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3374 	val |= xudc->phys_base & (XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3375 
3376 	fpci_writel(xudc, val, XUSB_DEV_CFG_4);
3377 	fpci_writel(xudc, upper_32_bits(xudc->phys_base), XUSB_DEV_CFG_5);
3378 
3379 	usleep_range(100, 200);
3380 
3381 	if (xudc->soc->has_ipfs) {
3382 		/* Enable interrupt assertion */
3383 		val = ipfs_readl(xudc, XUSB_DEV_INTR_MASK_0);
3384 		val |= XUSB_DEV_INTR_MASK_0_IP_INT_MASK;
3385 		ipfs_writel(xudc, val, XUSB_DEV_INTR_MASK_0);
3386 	}
3387 }
3388 
3389 static void tegra_xudc_device_params_init(struct tegra_xudc *xudc)
3390 {
3391 	u32 val, imod;
3392 
3393 	if (xudc->soc->has_ipfs) {
3394 		val = xudc_readl(xudc, BLCG);
3395 		val |= BLCG_ALL;
3396 		val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE |
3397 				BLCG_COREPLL_PWRDN);
3398 		val |= BLCG_IOPLL_0_PWRDN;
3399 		val |= BLCG_IOPLL_1_PWRDN;
3400 		val |= BLCG_IOPLL_2_PWRDN;
3401 
3402 		xudc_writel(xudc, val, BLCG);
3403 	}
3404 
3405 	if (xudc->soc->port_speed_quirk)
3406 		tegra_xudc_limit_port_speed(xudc);
3407 
3408 	/* Set a reasonable U3 exit timer value. */
3409 	val = xudc_readl(xudc, SSPX_CORE_PADCTL4);
3410 	val &= ~(SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK);
3411 	val |= SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(0x5dc0);
3412 	xudc_writel(xudc, val, SSPX_CORE_PADCTL4);
3413 
3414 	/* Default ping LFPS tBurst is too large. */
3415 	val = xudc_readl(xudc, SSPX_CORE_CNT0);
3416 	val &= ~(SSPX_CORE_CNT0_PING_TBURST_MASK);
3417 	val |= SSPX_CORE_CNT0_PING_TBURST(0xa);
3418 	xudc_writel(xudc, val, SSPX_CORE_CNT0);
3419 
3420 	/* Default tPortConfiguration timeout is too small. */
3421 	val = xudc_readl(xudc, SSPX_CORE_CNT30);
3422 	val &= ~(SSPX_CORE_CNT30_LMPITP_TIMER_MASK);
3423 	val |= SSPX_CORE_CNT30_LMPITP_TIMER(0x978);
3424 	xudc_writel(xudc, val, SSPX_CORE_CNT30);
3425 
3426 	if (xudc->soc->lpm_enable) {
3427 		/* Set L1 resume duration to 95 us. */
3428 		val = xudc_readl(xudc, HSFSPI_COUNT13);
3429 		val &= ~(HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK);
3430 		val |= HSFSPI_COUNT13_U2_RESUME_K_DURATION(0x2c88);
3431 		xudc_writel(xudc, val, HSFSPI_COUNT13);
3432 	}
3433 
3434 	/*
3435 	 * Compliance suite appears to be violating polling LFPS tBurst max
3436 	 * of 1.4us.  Send 1.45us instead.
3437 	 */
3438 	val = xudc_readl(xudc, SSPX_CORE_CNT32);
3439 	val &= ~(SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK);
3440 	val |= SSPX_CORE_CNT32_POLL_TBURST_MAX(0xb0);
3441 	xudc_writel(xudc, val, SSPX_CORE_CNT32);
3442 
3443 	/* Direct HS/FS port instance to RxDetect. */
3444 	val = xudc_readl(xudc, CFG_DEV_FE);
3445 	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3446 	val |= CFG_DEV_FE_PORTREGSEL(CFG_DEV_FE_PORTREGSEL_HSFS_PI);
3447 	xudc_writel(xudc, val, CFG_DEV_FE);
3448 
3449 	val = xudc_readl(xudc, PORTSC);
3450 	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3451 	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3452 	xudc_writel(xudc, val, PORTSC);
3453 
3454 	/* Direct SS port instance to RxDetect. */
3455 	val = xudc_readl(xudc, CFG_DEV_FE);
3456 	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3457 	val |= CFG_DEV_FE_PORTREGSEL_SS_PI & CFG_DEV_FE_PORTREGSEL_MASK;
3458 	xudc_writel(xudc, val, CFG_DEV_FE);
3459 
3460 	val = xudc_readl(xudc, PORTSC);
3461 	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3462 	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3463 	xudc_writel(xudc, val, PORTSC);
3464 
3465 	/* Restore port instance. */
3466 	val = xudc_readl(xudc, CFG_DEV_FE);
3467 	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3468 	xudc_writel(xudc, val, CFG_DEV_FE);
3469 
3470 	/*
3471 	 * Enable INFINITE_SS_RETRY to prevent device from entering
3472 	 * Disabled.Error when attached to buggy SuperSpeed hubs.
3473 	 */
3474 	val = xudc_readl(xudc, CFG_DEV_FE);
3475 	val |= CFG_DEV_FE_INFINITE_SS_RETRY;
3476 	xudc_writel(xudc, val, CFG_DEV_FE);
3477 
3478 	/* Set interrupt moderation. */
3479 	imod = XUDC_INTERRUPT_MODERATION_US * 4;
3480 	val = xudc_readl(xudc, RT_IMOD);
3481 	val &= ~((RT_IMOD_IMODI_MASK) | (RT_IMOD_IMODC_MASK));
3482 	val |= (RT_IMOD_IMODI(imod) | RT_IMOD_IMODC(imod));
3483 	xudc_writel(xudc, val, RT_IMOD);
3484 
3485 	/* increase SSPI transaction timeout from 32us to 512us */
3486 	val = xudc_readl(xudc, CFG_DEV_SSPI_XFER);
3487 	val &= ~(CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK);
3488 	val |= CFG_DEV_SSPI_XFER_ACKTIMEOUT(0xf000);
3489 	xudc_writel(xudc, val, CFG_DEV_SSPI_XFER);
3490 }
3491 
3492 static int tegra_xudc_phy_get(struct tegra_xudc *xudc)
3493 {
3494 	int err = 0, usb3_companion_port;
3495 	unsigned int i, j;
3496 
3497 	xudc->utmi_phy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3498 					   sizeof(*xudc->utmi_phy), GFP_KERNEL);
3499 	if (!xudc->utmi_phy)
3500 		return -ENOMEM;
3501 
3502 	xudc->usb3_phy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3503 					   sizeof(*xudc->usb3_phy), GFP_KERNEL);
3504 	if (!xudc->usb3_phy)
3505 		return -ENOMEM;
3506 
3507 	xudc->usbphy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3508 					   sizeof(*xudc->usbphy), GFP_KERNEL);
3509 	if (!xudc->usbphy)
3510 		return -ENOMEM;
3511 
3512 	xudc->vbus_nb.notifier_call = tegra_xudc_vbus_notify;
3513 
3514 	for (i = 0; i < xudc->soc->num_phys; i++) {
3515 		char phy_name[] = "usb.-.";
3516 
3517 		/* Get USB2 phy */
3518 		snprintf(phy_name, sizeof(phy_name), "usb2-%d", i);
3519 		xudc->utmi_phy[i] = devm_phy_optional_get(xudc->dev, phy_name);
3520 		if (IS_ERR(xudc->utmi_phy[i])) {
3521 			err = PTR_ERR(xudc->utmi_phy[i]);
3522 			dev_err_probe(xudc->dev, err,
3523 				"failed to get PHY for phy-name usb2-%d\n", i);
3524 			goto clean_up;
3525 		} else if (xudc->utmi_phy[i]) {
3526 			/* Get usb-phy, if utmi phy is available */
3527 			xudc->usbphy[i] = devm_usb_get_phy_by_node(xudc->dev,
3528 						xudc->utmi_phy[i]->dev.of_node,
3529 						NULL);
3530 			if (IS_ERR(xudc->usbphy[i])) {
3531 				err = PTR_ERR(xudc->usbphy[i]);
3532 				dev_err_probe(xudc->dev, err,
3533 					      "failed to get usbphy-%d\n", i);
3534 				goto clean_up;
3535 			}
3536 		} else if (!xudc->utmi_phy[i]) {
3537 			/* if utmi phy is not available, ignore USB3 phy get */
3538 			continue;
3539 		}
3540 
3541 		/* Get USB3 phy */
3542 		usb3_companion_port = tegra_xusb_padctl_get_usb3_companion(xudc->padctl, i);
3543 		if (usb3_companion_port < 0)
3544 			continue;
3545 
3546 		for (j = 0; j < xudc->soc->num_phys; j++) {
3547 			snprintf(phy_name, sizeof(phy_name), "usb3-%d", j);
3548 			xudc->usb3_phy[i] = devm_phy_optional_get(xudc->dev, phy_name);
3549 			if (IS_ERR(xudc->usb3_phy[i])) {
3550 				err = PTR_ERR(xudc->usb3_phy[i]);
3551 				dev_err_probe(xudc->dev, err,
3552 					"failed to get PHY for phy-name usb3-%d\n", j);
3553 				goto clean_up;
3554 			} else if (xudc->usb3_phy[i]) {
3555 				int usb2_port =
3556 					tegra_xusb_padctl_get_port_number(xudc->utmi_phy[i]);
3557 				int usb3_port =
3558 					tegra_xusb_padctl_get_port_number(xudc->usb3_phy[i]);
3559 				if (usb3_port == usb3_companion_port) {
3560 					dev_dbg(xudc->dev, "USB2 port %d is paired with USB3 port %d for device mode port %d\n",
3561 					 usb2_port, usb3_port, i);
3562 					break;
3563 				}
3564 			}
3565 		}
3566 	}
3567 
3568 	return err;
3569 
3570 clean_up:
3571 	for (i = 0; i < xudc->soc->num_phys; i++) {
3572 		xudc->usb3_phy[i] = NULL;
3573 		xudc->utmi_phy[i] = NULL;
3574 		xudc->usbphy[i] = NULL;
3575 	}
3576 
3577 	return err;
3578 }
3579 
3580 static void tegra_xudc_phy_exit(struct tegra_xudc *xudc)
3581 {
3582 	unsigned int i;
3583 
3584 	for (i = 0; i < xudc->soc->num_phys; i++) {
3585 		phy_exit(xudc->usb3_phy[i]);
3586 		phy_exit(xudc->utmi_phy[i]);
3587 	}
3588 }
3589 
3590 static int tegra_xudc_phy_init(struct tegra_xudc *xudc)
3591 {
3592 	int err;
3593 	unsigned int i;
3594 
3595 	for (i = 0; i < xudc->soc->num_phys; i++) {
3596 		err = phy_init(xudc->utmi_phy[i]);
3597 		if (err < 0) {
3598 			dev_err(xudc->dev, "UTMI PHY #%u initialization failed: %d\n", i, err);
3599 			goto exit_phy;
3600 		}
3601 
3602 		err = phy_init(xudc->usb3_phy[i]);
3603 		if (err < 0) {
3604 			dev_err(xudc->dev, "USB3 PHY #%u initialization failed: %d\n", i, err);
3605 			goto exit_phy;
3606 		}
3607 	}
3608 	return 0;
3609 
3610 exit_phy:
3611 	tegra_xudc_phy_exit(xudc);
3612 	return err;
3613 }
3614 
3615 static const char * const tegra210_xudc_supply_names[] = {
3616 	"hvdd-usb",
3617 	"avddio-usb",
3618 };
3619 
3620 static const char * const tegra210_xudc_clock_names[] = {
3621 	"dev",
3622 	"ss",
3623 	"ss_src",
3624 	"hs_src",
3625 	"fs_src",
3626 };
3627 
3628 static const char * const tegra186_xudc_clock_names[] = {
3629 	"dev",
3630 	"ss",
3631 	"ss_src",
3632 	"fs_src",
3633 };
3634 
3635 static struct tegra_xudc_soc tegra210_xudc_soc_data = {
3636 	.supply_names = tegra210_xudc_supply_names,
3637 	.num_supplies = ARRAY_SIZE(tegra210_xudc_supply_names),
3638 	.clock_names = tegra210_xudc_clock_names,
3639 	.num_clks = ARRAY_SIZE(tegra210_xudc_clock_names),
3640 	.num_phys = 4,
3641 	.u1_enable = false,
3642 	.u2_enable = true,
3643 	.lpm_enable = false,
3644 	.invalid_seq_num = true,
3645 	.pls_quirk = true,
3646 	.port_reset_quirk = true,
3647 	.port_speed_quirk = false,
3648 	.has_ipfs = true,
3649 };
3650 
3651 static struct tegra_xudc_soc tegra186_xudc_soc_data = {
3652 	.clock_names = tegra186_xudc_clock_names,
3653 	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3654 	.num_phys = 4,
3655 	.u1_enable = true,
3656 	.u2_enable = true,
3657 	.lpm_enable = false,
3658 	.invalid_seq_num = false,
3659 	.pls_quirk = false,
3660 	.port_reset_quirk = false,
3661 	.port_speed_quirk = false,
3662 	.has_ipfs = false,
3663 };
3664 
3665 static struct tegra_xudc_soc tegra194_xudc_soc_data = {
3666 	.clock_names = tegra186_xudc_clock_names,
3667 	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3668 	.num_phys = 4,
3669 	.u1_enable = true,
3670 	.u2_enable = true,
3671 	.lpm_enable = true,
3672 	.invalid_seq_num = false,
3673 	.pls_quirk = false,
3674 	.port_reset_quirk = false,
3675 	.port_speed_quirk = true,
3676 	.has_ipfs = false,
3677 };
3678 
3679 static struct tegra_xudc_soc tegra234_xudc_soc_data = {
3680 	.clock_names = tegra186_xudc_clock_names,
3681 	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3682 	.num_phys = 4,
3683 	.u1_enable = true,
3684 	.u2_enable = true,
3685 	.lpm_enable = true,
3686 	.invalid_seq_num = false,
3687 	.pls_quirk = false,
3688 	.port_reset_quirk = false,
3689 	.has_ipfs = false,
3690 };
3691 
3692 static const struct of_device_id tegra_xudc_of_match[] = {
3693 	{
3694 		.compatible = "nvidia,tegra210-xudc",
3695 		.data = &tegra210_xudc_soc_data
3696 	},
3697 	{
3698 		.compatible = "nvidia,tegra186-xudc",
3699 		.data = &tegra186_xudc_soc_data
3700 	},
3701 	{
3702 		.compatible = "nvidia,tegra194-xudc",
3703 		.data = &tegra194_xudc_soc_data
3704 	},
3705 	{
3706 		.compatible = "nvidia,tegra234-xudc",
3707 		.data = &tegra234_xudc_soc_data
3708 	},
3709 	{ }
3710 };
3711 MODULE_DEVICE_TABLE(of, tegra_xudc_of_match);
3712 
3713 static void tegra_xudc_powerdomain_remove(struct tegra_xudc *xudc)
3714 {
3715 	if (xudc->genpd_dl_ss)
3716 		device_link_del(xudc->genpd_dl_ss);
3717 	if (xudc->genpd_dl_device)
3718 		device_link_del(xudc->genpd_dl_device);
3719 	if (xudc->genpd_dev_ss)
3720 		dev_pm_domain_detach(xudc->genpd_dev_ss, true);
3721 	if (xudc->genpd_dev_device)
3722 		dev_pm_domain_detach(xudc->genpd_dev_device, true);
3723 }
3724 
3725 static int tegra_xudc_powerdomain_init(struct tegra_xudc *xudc)
3726 {
3727 	struct device *dev = xudc->dev;
3728 	int err;
3729 
3730 	xudc->genpd_dev_device = dev_pm_domain_attach_by_name(dev, "dev");
3731 	if (IS_ERR(xudc->genpd_dev_device)) {
3732 		err = PTR_ERR(xudc->genpd_dev_device);
3733 		dev_err(dev, "failed to get device power domain: %d\n", err);
3734 		return err;
3735 	}
3736 
3737 	xudc->genpd_dev_ss = dev_pm_domain_attach_by_name(dev, "ss");
3738 	if (IS_ERR(xudc->genpd_dev_ss)) {
3739 		err = PTR_ERR(xudc->genpd_dev_ss);
3740 		dev_err(dev, "failed to get SuperSpeed power domain: %d\n", err);
3741 		return err;
3742 	}
3743 
3744 	xudc->genpd_dl_device = device_link_add(dev, xudc->genpd_dev_device,
3745 						DL_FLAG_PM_RUNTIME |
3746 						DL_FLAG_STATELESS);
3747 	if (!xudc->genpd_dl_device) {
3748 		dev_err(dev, "failed to add USB device link\n");
3749 		return -ENODEV;
3750 	}
3751 
3752 	xudc->genpd_dl_ss = device_link_add(dev, xudc->genpd_dev_ss,
3753 					    DL_FLAG_PM_RUNTIME |
3754 					    DL_FLAG_STATELESS);
3755 	if (!xudc->genpd_dl_ss) {
3756 		dev_err(dev, "failed to add SuperSpeed device link\n");
3757 		return -ENODEV;
3758 	}
3759 
3760 	return 0;
3761 }
3762 
3763 static int tegra_xudc_probe(struct platform_device *pdev)
3764 {
3765 	struct tegra_xudc *xudc;
3766 	struct resource *res;
3767 	unsigned int i;
3768 	int err;
3769 
3770 	xudc = devm_kzalloc(&pdev->dev, sizeof(*xudc), GFP_KERNEL);
3771 	if (!xudc)
3772 		return -ENOMEM;
3773 
3774 	xudc->dev = &pdev->dev;
3775 	platform_set_drvdata(pdev, xudc);
3776 
3777 	xudc->soc = of_device_get_match_data(&pdev->dev);
3778 	if (!xudc->soc)
3779 		return -ENODEV;
3780 
3781 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
3782 	xudc->base = devm_ioremap_resource(&pdev->dev, res);
3783 	if (IS_ERR(xudc->base))
3784 		return PTR_ERR(xudc->base);
3785 	xudc->phys_base = res->start;
3786 
3787 	xudc->fpci = devm_platform_ioremap_resource_byname(pdev, "fpci");
3788 	if (IS_ERR(xudc->fpci))
3789 		return PTR_ERR(xudc->fpci);
3790 
3791 	if (xudc->soc->has_ipfs) {
3792 		xudc->ipfs = devm_platform_ioremap_resource_byname(pdev, "ipfs");
3793 		if (IS_ERR(xudc->ipfs))
3794 			return PTR_ERR(xudc->ipfs);
3795 	}
3796 
3797 	xudc->irq = platform_get_irq(pdev, 0);
3798 	if (xudc->irq < 0)
3799 		return xudc->irq;
3800 
3801 	err = devm_request_irq(&pdev->dev, xudc->irq, tegra_xudc_irq, 0,
3802 			       dev_name(&pdev->dev), xudc);
3803 	if (err < 0) {
3804 		dev_err(xudc->dev, "failed to claim IRQ#%u: %d\n", xudc->irq,
3805 			err);
3806 		return err;
3807 	}
3808 
3809 	xudc->clks = devm_kcalloc(&pdev->dev, xudc->soc->num_clks, sizeof(*xudc->clks),
3810 				  GFP_KERNEL);
3811 	if (!xudc->clks)
3812 		return -ENOMEM;
3813 
3814 	for (i = 0; i < xudc->soc->num_clks; i++)
3815 		xudc->clks[i].id = xudc->soc->clock_names[i];
3816 
3817 	err = devm_clk_bulk_get(&pdev->dev, xudc->soc->num_clks, xudc->clks);
3818 	if (err) {
3819 		dev_err_probe(xudc->dev, err, "failed to request clocks\n");
3820 		return err;
3821 	}
3822 
3823 	xudc->supplies = devm_kcalloc(&pdev->dev, xudc->soc->num_supplies,
3824 				      sizeof(*xudc->supplies), GFP_KERNEL);
3825 	if (!xudc->supplies)
3826 		return -ENOMEM;
3827 
3828 	for (i = 0; i < xudc->soc->num_supplies; i++)
3829 		xudc->supplies[i].supply = xudc->soc->supply_names[i];
3830 
3831 	err = devm_regulator_bulk_get(&pdev->dev, xudc->soc->num_supplies,
3832 				      xudc->supplies);
3833 	if (err) {
3834 		dev_err_probe(xudc->dev, err, "failed to request regulators\n");
3835 		return err;
3836 	}
3837 
3838 	xudc->padctl = tegra_xusb_padctl_get(&pdev->dev);
3839 	if (IS_ERR(xudc->padctl))
3840 		return PTR_ERR(xudc->padctl);
3841 
3842 	err = regulator_bulk_enable(xudc->soc->num_supplies, xudc->supplies);
3843 	if (err) {
3844 		dev_err(xudc->dev, "failed to enable regulators: %d\n", err);
3845 		goto put_padctl;
3846 	}
3847 
3848 	err = tegra_xudc_phy_get(xudc);
3849 	if (err)
3850 		goto disable_regulator;
3851 
3852 	err = tegra_xudc_powerdomain_init(xudc);
3853 	if (err)
3854 		goto put_powerdomains;
3855 
3856 	err = tegra_xudc_phy_init(xudc);
3857 	if (err)
3858 		goto put_powerdomains;
3859 
3860 	err = tegra_xudc_alloc_event_ring(xudc);
3861 	if (err)
3862 		goto disable_phy;
3863 
3864 	err = tegra_xudc_alloc_eps(xudc);
3865 	if (err)
3866 		goto free_event_ring;
3867 
3868 	spin_lock_init(&xudc->lock);
3869 
3870 	init_completion(&xudc->disconnect_complete);
3871 
3872 	INIT_WORK(&xudc->usb_role_sw_work, tegra_xudc_usb_role_sw_work);
3873 
3874 	INIT_DELAYED_WORK(&xudc->plc_reset_work, tegra_xudc_plc_reset_work);
3875 
3876 	INIT_DELAYED_WORK(&xudc->port_reset_war_work,
3877 				tegra_xudc_port_reset_war_work);
3878 
3879 	pm_runtime_enable(&pdev->dev);
3880 
3881 	xudc->gadget.ops = &tegra_xudc_gadget_ops;
3882 	xudc->gadget.ep0 = &xudc->ep[0].usb_ep;
3883 	xudc->gadget.name = "tegra-xudc";
3884 	xudc->gadget.max_speed = USB_SPEED_SUPER;
3885 
3886 	err = usb_add_gadget_udc(&pdev->dev, &xudc->gadget);
3887 	if (err) {
3888 		dev_err(&pdev->dev, "failed to add USB gadget: %d\n", err);
3889 		goto free_eps;
3890 	}
3891 
3892 	for (i = 0; i < xudc->soc->num_phys; i++) {
3893 		if (!xudc->usbphy[i])
3894 			continue;
3895 
3896 		usb_register_notifier(xudc->usbphy[i], &xudc->vbus_nb);
3897 		tegra_xudc_update_data_role(xudc, xudc->usbphy[i]);
3898 	}
3899 
3900 	return 0;
3901 
3902 free_eps:
3903 	pm_runtime_disable(&pdev->dev);
3904 	tegra_xudc_free_eps(xudc);
3905 free_event_ring:
3906 	tegra_xudc_free_event_ring(xudc);
3907 disable_phy:
3908 	tegra_xudc_phy_exit(xudc);
3909 put_powerdomains:
3910 	tegra_xudc_powerdomain_remove(xudc);
3911 disable_regulator:
3912 	regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3913 put_padctl:
3914 	tegra_xusb_padctl_put(xudc->padctl);
3915 
3916 	return err;
3917 }
3918 
3919 static void tegra_xudc_remove(struct platform_device *pdev)
3920 {
3921 	struct tegra_xudc *xudc = platform_get_drvdata(pdev);
3922 	unsigned int i;
3923 
3924 	pm_runtime_get_sync(xudc->dev);
3925 
3926 	cancel_delayed_work_sync(&xudc->plc_reset_work);
3927 	cancel_work_sync(&xudc->usb_role_sw_work);
3928 
3929 	usb_del_gadget_udc(&xudc->gadget);
3930 
3931 	tegra_xudc_free_eps(xudc);
3932 	tegra_xudc_free_event_ring(xudc);
3933 
3934 	tegra_xudc_powerdomain_remove(xudc);
3935 
3936 	regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3937 
3938 	for (i = 0; i < xudc->soc->num_phys; i++) {
3939 		phy_power_off(xudc->utmi_phy[i]);
3940 		phy_power_off(xudc->usb3_phy[i]);
3941 	}
3942 
3943 	tegra_xudc_phy_exit(xudc);
3944 
3945 	pm_runtime_disable(xudc->dev);
3946 	pm_runtime_put(xudc->dev);
3947 
3948 	tegra_xusb_padctl_put(xudc->padctl);
3949 }
3950 
3951 static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc)
3952 {
3953 	unsigned long flags;
3954 
3955 	dev_dbg(xudc->dev, "entering ELPG\n");
3956 
3957 	spin_lock_irqsave(&xudc->lock, flags);
3958 
3959 	xudc->powergated = true;
3960 	xudc->saved_regs.ctrl = xudc_readl(xudc, CTRL);
3961 	xudc->saved_regs.portpm = xudc_readl(xudc, PORTPM);
3962 	xudc_writel(xudc, 0, CTRL);
3963 
3964 	spin_unlock_irqrestore(&xudc->lock, flags);
3965 
3966 	clk_bulk_disable_unprepare(xudc->soc->num_clks, xudc->clks);
3967 
3968 	regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3969 
3970 	dev_dbg(xudc->dev, "entering ELPG done\n");
3971 	return 0;
3972 }
3973 
3974 static int __maybe_unused tegra_xudc_unpowergate(struct tegra_xudc *xudc)
3975 {
3976 	unsigned long flags;
3977 	int err;
3978 
3979 	dev_dbg(xudc->dev, "exiting ELPG\n");
3980 
3981 	err = regulator_bulk_enable(xudc->soc->num_supplies,
3982 			xudc->supplies);
3983 	if (err < 0)
3984 		return err;
3985 
3986 	err = clk_bulk_prepare_enable(xudc->soc->num_clks, xudc->clks);
3987 	if (err < 0)
3988 		return err;
3989 
3990 	tegra_xudc_fpci_ipfs_init(xudc);
3991 
3992 	tegra_xudc_device_params_init(xudc);
3993 
3994 	tegra_xudc_init_event_ring(xudc);
3995 
3996 	tegra_xudc_init_eps(xudc);
3997 
3998 	xudc_writel(xudc, xudc->saved_regs.portpm, PORTPM);
3999 	xudc_writel(xudc, xudc->saved_regs.ctrl, CTRL);
4000 
4001 	spin_lock_irqsave(&xudc->lock, flags);
4002 	xudc->powergated = false;
4003 	spin_unlock_irqrestore(&xudc->lock, flags);
4004 
4005 	dev_dbg(xudc->dev, "exiting ELPG done\n");
4006 	return 0;
4007 }
4008 
4009 static int __maybe_unused tegra_xudc_suspend(struct device *dev)
4010 {
4011 	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4012 	unsigned long flags;
4013 
4014 	spin_lock_irqsave(&xudc->lock, flags);
4015 	xudc->suspended = true;
4016 	spin_unlock_irqrestore(&xudc->lock, flags);
4017 
4018 	flush_work(&xudc->usb_role_sw_work);
4019 
4020 	if (!pm_runtime_status_suspended(dev)) {
4021 		/* Forcibly disconnect before powergating. */
4022 		tegra_xudc_device_mode_off(xudc);
4023 		tegra_xudc_powergate(xudc);
4024 	}
4025 
4026 	pm_runtime_disable(dev);
4027 
4028 	return 0;
4029 }
4030 
4031 static int __maybe_unused tegra_xudc_resume(struct device *dev)
4032 {
4033 	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4034 	unsigned long flags;
4035 	int err;
4036 
4037 	err = tegra_xudc_unpowergate(xudc);
4038 	if (err < 0)
4039 		return err;
4040 
4041 	spin_lock_irqsave(&xudc->lock, flags);
4042 	xudc->suspended = false;
4043 	spin_unlock_irqrestore(&xudc->lock, flags);
4044 
4045 	schedule_work(&xudc->usb_role_sw_work);
4046 
4047 	pm_runtime_enable(dev);
4048 
4049 	return 0;
4050 }
4051 
4052 static int __maybe_unused tegra_xudc_runtime_suspend(struct device *dev)
4053 {
4054 	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4055 
4056 	return tegra_xudc_powergate(xudc);
4057 }
4058 
4059 static int __maybe_unused tegra_xudc_runtime_resume(struct device *dev)
4060 {
4061 	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4062 
4063 	return tegra_xudc_unpowergate(xudc);
4064 }
4065 
4066 static const struct dev_pm_ops tegra_xudc_pm_ops = {
4067 	SET_SYSTEM_SLEEP_PM_OPS(tegra_xudc_suspend, tegra_xudc_resume)
4068 	SET_RUNTIME_PM_OPS(tegra_xudc_runtime_suspend,
4069 			   tegra_xudc_runtime_resume, NULL)
4070 };
4071 
4072 static struct platform_driver tegra_xudc_driver = {
4073 	.probe = tegra_xudc_probe,
4074 	.remove_new = tegra_xudc_remove,
4075 	.driver = {
4076 		.name = "tegra-xudc",
4077 		.pm = &tegra_xudc_pm_ops,
4078 		.of_match_table = tegra_xudc_of_match,
4079 	},
4080 };
4081 module_platform_driver(tegra_xudc_driver);
4082 
4083 MODULE_DESCRIPTION("NVIDIA Tegra XUSB Device Controller");
4084 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
4085 MODULE_AUTHOR("Hui Fu <hfu@nvidia.com>");
4086 MODULE_AUTHOR("Nagarjuna Kristam <nkristam@nvidia.com>");
4087 MODULE_LICENSE("GPL v2");
4088