1 // SPDX-License-Identifier: GPL-2.0+
2 /*
3  * NVIDIA Tegra XUSB device mode controller
4  *
5  * Copyright (c) 2013-2019, NVIDIA CORPORATION.  All rights reserved.
6  * Copyright (c) 2015, Google Inc.
7  */
8 
9 #include <linux/clk.h>
10 #include <linux/completion.h>
11 #include <linux/delay.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/dmapool.h>
14 #include <linux/interrupt.h>
15 #include <linux/iopoll.h>
16 #include <linux/kernel.h>
17 #include <linux/module.h>
18 #include <linux/of.h>
19 #include <linux/of_device.h>
20 #include <linux/phy/phy.h>
21 #include <linux/phy/tegra/xusb.h>
22 #include <linux/pm_domain.h>
23 #include <linux/platform_device.h>
24 #include <linux/pm_runtime.h>
25 #include <linux/regulator/consumer.h>
26 #include <linux/reset.h>
27 #include <linux/usb/ch9.h>
28 #include <linux/usb/gadget.h>
29 #include <linux/usb/otg.h>
30 #include <linux/usb/role.h>
31 #include <linux/usb/phy.h>
32 #include <linux/workqueue.h>
33 
34 /* XUSB_DEV registers */
35 #define SPARAM 0x000
36 #define  SPARAM_ERSTMAX_MASK GENMASK(20, 16)
37 #define  SPARAM_ERSTMAX(x) (((x) << 16) & SPARAM_ERSTMAX_MASK)
38 #define DB 0x004
39 #define  DB_TARGET_MASK GENMASK(15, 8)
40 #define  DB_TARGET(x) (((x) << 8) & DB_TARGET_MASK)
41 #define  DB_STREAMID_MASK GENMASK(31, 16)
42 #define  DB_STREAMID(x) (((x) << 16) & DB_STREAMID_MASK)
43 #define ERSTSZ 0x008
44 #define  ERSTSZ_ERSTXSZ_SHIFT(x) ((x) * 16)
45 #define  ERSTSZ_ERSTXSZ_MASK GENMASK(15, 0)
46 #define ERSTXBALO(x) (0x010 + 8 * (x))
47 #define ERSTXBAHI(x) (0x014 + 8 * (x))
48 #define ERDPLO 0x020
49 #define  ERDPLO_EHB BIT(3)
50 #define ERDPHI 0x024
51 #define EREPLO 0x028
52 #define  EREPLO_ECS BIT(0)
53 #define  EREPLO_SEGI BIT(1)
54 #define EREPHI 0x02c
55 #define CTRL 0x030
56 #define  CTRL_RUN BIT(0)
57 #define  CTRL_LSE BIT(1)
58 #define  CTRL_IE BIT(4)
59 #define  CTRL_SMI_EVT BIT(5)
60 #define  CTRL_SMI_DSE BIT(6)
61 #define  CTRL_EWE BIT(7)
62 #define  CTRL_DEVADDR_MASK GENMASK(30, 24)
63 #define  CTRL_DEVADDR(x) (((x) << 24) & CTRL_DEVADDR_MASK)
64 #define  CTRL_ENABLE BIT(31)
65 #define ST 0x034
66 #define  ST_RC BIT(0)
67 #define  ST_IP BIT(4)
68 #define RT_IMOD	0x038
69 #define  RT_IMOD_IMODI_MASK GENMASK(15, 0)
70 #define  RT_IMOD_IMODI(x) ((x) & RT_IMOD_IMODI_MASK)
71 #define  RT_IMOD_IMODC_MASK GENMASK(31, 16)
72 #define  RT_IMOD_IMODC(x) (((x) << 16) & RT_IMOD_IMODC_MASK)
73 #define PORTSC 0x03c
74 #define  PORTSC_CCS BIT(0)
75 #define  PORTSC_PED BIT(1)
76 #define  PORTSC_PR BIT(4)
77 #define  PORTSC_PLS_SHIFT 5
78 #define  PORTSC_PLS_MASK GENMASK(8, 5)
79 #define  PORTSC_PLS_U0 0x0
80 #define  PORTSC_PLS_U2 0x2
81 #define  PORTSC_PLS_U3 0x3
82 #define  PORTSC_PLS_DISABLED 0x4
83 #define  PORTSC_PLS_RXDETECT 0x5
84 #define  PORTSC_PLS_INACTIVE 0x6
85 #define  PORTSC_PLS_RESUME 0xf
86 #define  PORTSC_PLS(x) (((x) << PORTSC_PLS_SHIFT) & PORTSC_PLS_MASK)
87 #define  PORTSC_PS_SHIFT 10
88 #define  PORTSC_PS_MASK GENMASK(13, 10)
89 #define  PORTSC_PS_UNDEFINED 0x0
90 #define  PORTSC_PS_FS 0x1
91 #define  PORTSC_PS_LS 0x2
92 #define  PORTSC_PS_HS 0x3
93 #define  PORTSC_PS_SS 0x4
94 #define  PORTSC_LWS BIT(16)
95 #define  PORTSC_CSC BIT(17)
96 #define  PORTSC_WRC BIT(19)
97 #define  PORTSC_PRC BIT(21)
98 #define  PORTSC_PLC BIT(22)
99 #define  PORTSC_CEC BIT(23)
100 #define  PORTSC_WPR BIT(30)
101 #define  PORTSC_CHANGE_MASK (PORTSC_CSC | PORTSC_WRC | PORTSC_PRC | \
102 			     PORTSC_PLC | PORTSC_CEC)
103 #define ECPLO 0x040
104 #define ECPHI 0x044
105 #define MFINDEX 0x048
106 #define  MFINDEX_FRAME_SHIFT 3
107 #define  MFINDEX_FRAME_MASK GENMASK(13, 3)
108 #define PORTPM 0x04c
109 #define  PORTPM_L1S_MASK GENMASK(1, 0)
110 #define  PORTPM_L1S_DROP 0x0
111 #define  PORTPM_L1S_ACCEPT 0x1
112 #define  PORTPM_L1S_NYET 0x2
113 #define  PORTPM_L1S_STALL 0x3
114 #define  PORTPM_L1S(x) ((x) & PORTPM_L1S_MASK)
115 #define  PORTPM_RWE BIT(3)
116 #define  PORTPM_U2TIMEOUT_MASK GENMASK(15, 8)
117 #define  PORTPM_U1TIMEOUT_MASK GENMASK(23, 16)
118 #define  PORTPM_FLA BIT(24)
119 #define  PORTPM_VBA BIT(25)
120 #define  PORTPM_WOC BIT(26)
121 #define  PORTPM_WOD BIT(27)
122 #define  PORTPM_U1E BIT(28)
123 #define  PORTPM_U2E BIT(29)
124 #define  PORTPM_FRWE BIT(30)
125 #define  PORTPM_PNG_CYA BIT(31)
126 #define EP_HALT 0x050
127 #define EP_PAUSE 0x054
128 #define EP_RELOAD 0x058
129 #define EP_STCHG 0x05c
130 #define DEVNOTIF_LO 0x064
131 #define  DEVNOTIF_LO_TRIG BIT(0)
132 #define  DEVNOTIF_LO_TYPE_MASK GENMASK(7, 4)
133 #define  DEVNOTIF_LO_TYPE(x) (((x) << 4)  & DEVNOTIF_LO_TYPE_MASK)
134 #define  DEVNOTIF_LO_TYPE_FUNCTION_WAKE 0x1
135 #define DEVNOTIF_HI 0x068
136 #define PORTHALT 0x06c
137 #define  PORTHALT_HALT_LTSSM BIT(0)
138 #define  PORTHALT_HALT_REJECT BIT(1)
139 #define  PORTHALT_STCHG_REQ BIT(20)
140 #define  PORTHALT_STCHG_INTR_EN BIT(24)
141 #define PORT_TM	0x070
142 #define EP_THREAD_ACTIVE 0x074
143 #define EP_STOPPED 0x078
144 #define HSFSPI_COUNT0 0x100
145 #define HSFSPI_COUNT13 0x134
146 #define  HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK GENMASK(29, 0)
147 #define  HSFSPI_COUNT13_U2_RESUME_K_DURATION(x) ((x) & \
148 				HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK)
149 #define BLCG 0x840
150 #define SSPX_CORE_CNT0 0x610
151 #define  SSPX_CORE_CNT0_PING_TBURST_MASK GENMASK(7, 0)
152 #define  SSPX_CORE_CNT0_PING_TBURST(x) ((x) & SSPX_CORE_CNT0_PING_TBURST_MASK)
153 #define SSPX_CORE_CNT30 0x688
154 #define  SSPX_CORE_CNT30_LMPITP_TIMER_MASK GENMASK(19, 0)
155 #define  SSPX_CORE_CNT30_LMPITP_TIMER(x) ((x) & \
156 					SSPX_CORE_CNT30_LMPITP_TIMER_MASK)
157 #define SSPX_CORE_CNT32 0x690
158 #define  SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK GENMASK(7, 0)
159 #define  SSPX_CORE_CNT32_POLL_TBURST_MAX(x) ((x) & \
160 					SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK)
161 #define SSPX_CORE_CNT56 0x6fc
162 #define  SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK GENMASK(19, 0)
163 #define  SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(x) ((x) & \
164 				SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK)
165 #define SSPX_CORE_CNT57 0x700
166 #define  SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK GENMASK(19, 0)
167 #define  SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(x) ((x) & \
168 				SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK)
169 #define SSPX_CORE_CNT65 0x720
170 #define  SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK GENMASK(19, 0)
171 #define  SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(x) ((x) & \
172 				SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK)
173 #define SSPX_CORE_CNT66 0x724
174 #define  SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK GENMASK(19, 0)
175 #define  SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(x) ((x) & \
176 				SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK)
177 #define SSPX_CORE_CNT67 0x728
178 #define  SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK GENMASK(19, 0)
179 #define  SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(x) ((x) & \
180 				SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK)
181 #define SSPX_CORE_CNT72 0x73c
182 #define  SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK GENMASK(19, 0)
183 #define  SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(x) ((x) & \
184 				SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK)
185 #define SSPX_CORE_PADCTL4 0x750
186 #define  SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK GENMASK(19, 0)
187 #define  SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(x) ((x) & \
188 				SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK)
189 #define  BLCG_DFPCI BIT(0)
190 #define  BLCG_UFPCI BIT(1)
191 #define  BLCG_FE BIT(2)
192 #define  BLCG_COREPLL_PWRDN BIT(8)
193 #define  BLCG_IOPLL_0_PWRDN BIT(9)
194 #define  BLCG_IOPLL_1_PWRDN BIT(10)
195 #define  BLCG_IOPLL_2_PWRDN BIT(11)
196 #define  BLCG_ALL 0x1ff
197 #define CFG_DEV_SSPI_XFER 0x858
198 #define  CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK GENMASK(31, 0)
199 #define  CFG_DEV_SSPI_XFER_ACKTIMEOUT(x) ((x) & \
200 					CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK)
201 #define CFG_DEV_FE 0x85c
202 #define  CFG_DEV_FE_PORTREGSEL_MASK GENMASK(1, 0)
203 #define  CFG_DEV_FE_PORTREGSEL_SS_PI 1
204 #define  CFG_DEV_FE_PORTREGSEL_HSFS_PI 2
205 #define  CFG_DEV_FE_PORTREGSEL(x) ((x) & CFG_DEV_FE_PORTREGSEL_MASK)
206 #define  CFG_DEV_FE_INFINITE_SS_RETRY BIT(29)
207 
208 /* FPCI registers */
209 #define XUSB_DEV_CFG_1 0x004
210 #define  XUSB_DEV_CFG_1_IO_SPACE_EN BIT(0)
211 #define  XUSB_DEV_CFG_1_MEMORY_SPACE_EN BIT(1)
212 #define  XUSB_DEV_CFG_1_BUS_MASTER_EN BIT(2)
213 #define XUSB_DEV_CFG_4 0x010
214 #define  XUSB_DEV_CFG_4_BASE_ADDR_MASK GENMASK(31, 15)
215 #define XUSB_DEV_CFG_5 0x014
216 
217 /* IPFS registers */
218 #define XUSB_DEV_CONFIGURATION_0 0x180
219 #define  XUSB_DEV_CONFIGURATION_0_EN_FPCI BIT(0)
220 #define XUSB_DEV_INTR_MASK_0 0x188
221 #define  XUSB_DEV_INTR_MASK_0_IP_INT_MASK BIT(16)
222 
223 struct tegra_xudc_ep_context {
224 	__le32 info0;
225 	__le32 info1;
226 	__le32 deq_lo;
227 	__le32 deq_hi;
228 	__le32 tx_info;
229 	__le32 rsvd[11];
230 };
231 
232 #define EP_STATE_DISABLED 0
233 #define EP_STATE_RUNNING 1
234 #define EP_STATE_HALTED 2
235 #define EP_STATE_STOPPED 3
236 #define EP_STATE_ERROR 4
237 
238 #define EP_TYPE_INVALID 0
239 #define EP_TYPE_ISOCH_OUT 1
240 #define EP_TYPE_BULK_OUT 2
241 #define EP_TYPE_INTERRUPT_OUT 3
242 #define EP_TYPE_CONTROL 4
243 #define EP_TYPE_ISCOH_IN 5
244 #define EP_TYPE_BULK_IN 6
245 #define EP_TYPE_INTERRUPT_IN 7
246 
247 #define BUILD_EP_CONTEXT_RW(name, member, shift, mask)			\
248 static inline u32 ep_ctx_read_##name(struct tegra_xudc_ep_context *ctx)	\
249 {									\
250 	return (le32_to_cpu(ctx->member) >> (shift)) & (mask);		\
251 }									\
252 static inline void							\
253 ep_ctx_write_##name(struct tegra_xudc_ep_context *ctx, u32 val)		\
254 {									\
255 	u32 tmp;							\
256 									\
257 	tmp = le32_to_cpu(ctx->member) & ~((mask) << (shift));		\
258 	tmp |= (val & (mask)) << (shift);				\
259 	ctx->member = cpu_to_le32(tmp);					\
260 }
261 
262 BUILD_EP_CONTEXT_RW(state, info0, 0, 0x7)
263 BUILD_EP_CONTEXT_RW(mult, info0, 8, 0x3)
264 BUILD_EP_CONTEXT_RW(max_pstreams, info0, 10, 0x1f)
265 BUILD_EP_CONTEXT_RW(lsa, info0, 15, 0x1)
266 BUILD_EP_CONTEXT_RW(interval, info0, 16, 0xff)
267 BUILD_EP_CONTEXT_RW(cerr, info1, 1, 0x3)
268 BUILD_EP_CONTEXT_RW(type, info1, 3, 0x7)
269 BUILD_EP_CONTEXT_RW(hid, info1, 7, 0x1)
270 BUILD_EP_CONTEXT_RW(max_burst_size, info1, 8, 0xff)
271 BUILD_EP_CONTEXT_RW(max_packet_size, info1, 16, 0xffff)
272 BUILD_EP_CONTEXT_RW(dcs, deq_lo, 0, 0x1)
273 BUILD_EP_CONTEXT_RW(deq_lo, deq_lo, 4, 0xfffffff)
274 BUILD_EP_CONTEXT_RW(deq_hi, deq_hi, 0, 0xffffffff)
275 BUILD_EP_CONTEXT_RW(avg_trb_len, tx_info, 0, 0xffff)
276 BUILD_EP_CONTEXT_RW(max_esit_payload, tx_info, 16, 0xffff)
277 BUILD_EP_CONTEXT_RW(edtla, rsvd[0], 0, 0xffffff)
278 BUILD_EP_CONTEXT_RW(seq_num, rsvd[0], 24, 0xff)
279 BUILD_EP_CONTEXT_RW(partial_td, rsvd[0], 25, 0x1)
280 BUILD_EP_CONTEXT_RW(cerrcnt, rsvd[1], 18, 0x3)
281 BUILD_EP_CONTEXT_RW(data_offset, rsvd[2], 0, 0x1ffff)
282 BUILD_EP_CONTEXT_RW(numtrbs, rsvd[2], 22, 0x1f)
283 BUILD_EP_CONTEXT_RW(devaddr, rsvd[6], 0, 0x7f)
284 
285 static inline u64 ep_ctx_read_deq_ptr(struct tegra_xudc_ep_context *ctx)
286 {
287 	return ((u64)ep_ctx_read_deq_hi(ctx) << 32) |
288 		(ep_ctx_read_deq_lo(ctx) << 4);
289 }
290 
291 static inline void
292 ep_ctx_write_deq_ptr(struct tegra_xudc_ep_context *ctx, u64 addr)
293 {
294 	ep_ctx_write_deq_lo(ctx, lower_32_bits(addr) >> 4);
295 	ep_ctx_write_deq_hi(ctx, upper_32_bits(addr));
296 }
297 
298 struct tegra_xudc_trb {
299 	__le32 data_lo;
300 	__le32 data_hi;
301 	__le32 status;
302 	__le32 control;
303 };
304 
305 #define TRB_TYPE_RSVD 0
306 #define TRB_TYPE_NORMAL 1
307 #define TRB_TYPE_SETUP_STAGE 2
308 #define TRB_TYPE_DATA_STAGE 3
309 #define TRB_TYPE_STATUS_STAGE 4
310 #define TRB_TYPE_ISOCH 5
311 #define TRB_TYPE_LINK 6
312 #define TRB_TYPE_TRANSFER_EVENT 32
313 #define TRB_TYPE_PORT_STATUS_CHANGE_EVENT 34
314 #define TRB_TYPE_STREAM 48
315 #define TRB_TYPE_SETUP_PACKET_EVENT 63
316 
317 #define TRB_CMPL_CODE_INVALID 0
318 #define TRB_CMPL_CODE_SUCCESS 1
319 #define TRB_CMPL_CODE_DATA_BUFFER_ERR 2
320 #define TRB_CMPL_CODE_BABBLE_DETECTED_ERR 3
321 #define TRB_CMPL_CODE_USB_TRANS_ERR 4
322 #define TRB_CMPL_CODE_TRB_ERR 5
323 #define TRB_CMPL_CODE_STALL 6
324 #define TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR 10
325 #define TRB_CMPL_CODE_SHORT_PACKET 13
326 #define TRB_CMPL_CODE_RING_UNDERRUN 14
327 #define TRB_CMPL_CODE_RING_OVERRUN 15
328 #define TRB_CMPL_CODE_EVENT_RING_FULL_ERR 21
329 #define TRB_CMPL_CODE_STOPPED 26
330 #define TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN 31
331 #define TRB_CMPL_CODE_STREAM_NUMP_ERROR 219
332 #define TRB_CMPL_CODE_PRIME_PIPE_RECEIVED 220
333 #define TRB_CMPL_CODE_HOST_REJECTED 221
334 #define TRB_CMPL_CODE_CTRL_DIR_ERR 222
335 #define TRB_CMPL_CODE_CTRL_SEQNUM_ERR 223
336 
337 #define BUILD_TRB_RW(name, member, shift, mask)				\
338 static inline u32 trb_read_##name(struct tegra_xudc_trb *trb)		\
339 {									\
340 	return (le32_to_cpu(trb->member) >> (shift)) & (mask);		\
341 }									\
342 static inline void							\
343 trb_write_##name(struct tegra_xudc_trb *trb, u32 val)			\
344 {									\
345 	u32 tmp;							\
346 									\
347 	tmp = le32_to_cpu(trb->member) & ~((mask) << (shift));		\
348 	tmp |= (val & (mask)) << (shift);				\
349 	trb->member = cpu_to_le32(tmp);					\
350 }
351 
352 BUILD_TRB_RW(data_lo, data_lo, 0, 0xffffffff)
353 BUILD_TRB_RW(data_hi, data_hi, 0, 0xffffffff)
354 BUILD_TRB_RW(seq_num, status, 0, 0xffff)
355 BUILD_TRB_RW(transfer_len, status, 0, 0xffffff)
356 BUILD_TRB_RW(td_size, status, 17, 0x1f)
357 BUILD_TRB_RW(cmpl_code, status, 24, 0xff)
358 BUILD_TRB_RW(cycle, control, 0, 0x1)
359 BUILD_TRB_RW(toggle_cycle, control, 1, 0x1)
360 BUILD_TRB_RW(isp, control, 2, 0x1)
361 BUILD_TRB_RW(chain, control, 4, 0x1)
362 BUILD_TRB_RW(ioc, control, 5, 0x1)
363 BUILD_TRB_RW(type, control, 10, 0x3f)
364 BUILD_TRB_RW(stream_id, control, 16, 0xffff)
365 BUILD_TRB_RW(endpoint_id, control, 16, 0x1f)
366 BUILD_TRB_RW(tlbpc, control, 16, 0xf)
367 BUILD_TRB_RW(data_stage_dir, control, 16, 0x1)
368 BUILD_TRB_RW(frame_id, control, 20, 0x7ff)
369 BUILD_TRB_RW(sia, control, 31, 0x1)
370 
371 static inline u64 trb_read_data_ptr(struct tegra_xudc_trb *trb)
372 {
373 	return ((u64)trb_read_data_hi(trb) << 32) |
374 		trb_read_data_lo(trb);
375 }
376 
377 static inline void trb_write_data_ptr(struct tegra_xudc_trb *trb, u64 addr)
378 {
379 	trb_write_data_lo(trb, lower_32_bits(addr));
380 	trb_write_data_hi(trb, upper_32_bits(addr));
381 }
382 
383 struct tegra_xudc_request {
384 	struct usb_request usb_req;
385 
386 	size_t buf_queued;
387 	unsigned int trbs_queued;
388 	unsigned int trbs_needed;
389 	bool need_zlp;
390 
391 	struct tegra_xudc_trb *first_trb;
392 	struct tegra_xudc_trb *last_trb;
393 
394 	struct list_head list;
395 };
396 
397 struct tegra_xudc_ep {
398 	struct tegra_xudc *xudc;
399 	struct usb_ep usb_ep;
400 	unsigned int index;
401 	char name[8];
402 
403 	struct tegra_xudc_ep_context *context;
404 
405 #define XUDC_TRANSFER_RING_SIZE 64
406 	struct tegra_xudc_trb *transfer_ring;
407 	dma_addr_t transfer_ring_phys;
408 
409 	unsigned int enq_ptr;
410 	unsigned int deq_ptr;
411 	bool pcs;
412 	bool ring_full;
413 	bool stream_rejected;
414 
415 	struct list_head queue;
416 	const struct usb_endpoint_descriptor *desc;
417 	const struct usb_ss_ep_comp_descriptor *comp_desc;
418 };
419 
420 struct tegra_xudc_sel_timing {
421 	__u8 u1sel;
422 	__u8 u1pel;
423 	__le16 u2sel;
424 	__le16 u2pel;
425 };
426 
427 enum tegra_xudc_setup_state {
428 	WAIT_FOR_SETUP,
429 	DATA_STAGE_XFER,
430 	DATA_STAGE_RECV,
431 	STATUS_STAGE_XFER,
432 	STATUS_STAGE_RECV,
433 };
434 
435 struct tegra_xudc_setup_packet {
436 	struct usb_ctrlrequest ctrl_req;
437 	unsigned int seq_num;
438 };
439 
440 struct tegra_xudc_save_regs {
441 	u32 ctrl;
442 	u32 portpm;
443 };
444 
445 struct tegra_xudc {
446 	struct device *dev;
447 	const struct tegra_xudc_soc *soc;
448 	struct tegra_xusb_padctl *padctl;
449 
450 	spinlock_t lock;
451 
452 	struct usb_gadget gadget;
453 	struct usb_gadget_driver *driver;
454 
455 #define XUDC_NR_EVENT_RINGS 2
456 #define XUDC_EVENT_RING_SIZE 4096
457 	struct tegra_xudc_trb *event_ring[XUDC_NR_EVENT_RINGS];
458 	dma_addr_t event_ring_phys[XUDC_NR_EVENT_RINGS];
459 	unsigned int event_ring_index;
460 	unsigned int event_ring_deq_ptr;
461 	bool ccs;
462 
463 #define XUDC_NR_EPS 32
464 	struct tegra_xudc_ep ep[XUDC_NR_EPS];
465 	struct tegra_xudc_ep_context *ep_context;
466 	dma_addr_t ep_context_phys;
467 
468 	struct device *genpd_dev_device;
469 	struct device *genpd_dev_ss;
470 	struct device_link *genpd_dl_device;
471 	struct device_link *genpd_dl_ss;
472 
473 	struct dma_pool *transfer_ring_pool;
474 
475 	bool queued_setup_packet;
476 	struct tegra_xudc_setup_packet setup_packet;
477 	enum tegra_xudc_setup_state setup_state;
478 	u16 setup_seq_num;
479 
480 	u16 dev_addr;
481 	u16 isoch_delay;
482 	struct tegra_xudc_sel_timing sel_timing;
483 	u8 test_mode_pattern;
484 	u16 status_buf;
485 	struct tegra_xudc_request *ep0_req;
486 
487 	bool pullup;
488 
489 	unsigned int nr_enabled_eps;
490 	unsigned int nr_isoch_eps;
491 
492 	unsigned int device_state;
493 	unsigned int resume_state;
494 
495 	int irq;
496 
497 	void __iomem *base;
498 	resource_size_t phys_base;
499 	void __iomem *ipfs;
500 	void __iomem *fpci;
501 
502 	struct regulator_bulk_data *supplies;
503 
504 	struct clk_bulk_data *clks;
505 
506 	bool device_mode;
507 	struct work_struct usb_role_sw_work;
508 
509 	struct phy **usb3_phy;
510 	struct phy *curr_usb3_phy;
511 	struct phy **utmi_phy;
512 	struct phy *curr_utmi_phy;
513 
514 	struct tegra_xudc_save_regs saved_regs;
515 	bool suspended;
516 	bool powergated;
517 
518 	struct usb_phy **usbphy;
519 	struct usb_phy *curr_usbphy;
520 	struct notifier_block vbus_nb;
521 
522 	struct completion disconnect_complete;
523 
524 	bool selfpowered;
525 
526 #define TOGGLE_VBUS_WAIT_MS 100
527 	struct delayed_work plc_reset_work;
528 	bool wait_csc;
529 
530 	struct delayed_work port_reset_war_work;
531 	bool wait_for_sec_prc;
532 };
533 
534 #define XUDC_TRB_MAX_BUFFER_SIZE 65536
535 #define XUDC_MAX_ISOCH_EPS 4
536 #define XUDC_INTERRUPT_MODERATION_US 0
537 
538 static struct usb_endpoint_descriptor tegra_xudc_ep0_desc = {
539 	.bLength = USB_DT_ENDPOINT_SIZE,
540 	.bDescriptorType = USB_DT_ENDPOINT,
541 	.bEndpointAddress = 0,
542 	.bmAttributes = USB_ENDPOINT_XFER_CONTROL,
543 	.wMaxPacketSize = cpu_to_le16(64),
544 };
545 
546 struct tegra_xudc_soc {
547 	const char * const *supply_names;
548 	unsigned int num_supplies;
549 	const char * const *clock_names;
550 	unsigned int num_clks;
551 	unsigned int num_phys;
552 	bool u1_enable;
553 	bool u2_enable;
554 	bool lpm_enable;
555 	bool invalid_seq_num;
556 	bool pls_quirk;
557 	bool port_reset_quirk;
558 	bool port_speed_quirk;
559 	bool has_ipfs;
560 };
561 
562 static inline u32 fpci_readl(struct tegra_xudc *xudc, unsigned int offset)
563 {
564 	return readl(xudc->fpci + offset);
565 }
566 
567 static inline void fpci_writel(struct tegra_xudc *xudc, u32 val,
568 			       unsigned int offset)
569 {
570 	writel(val, xudc->fpci + offset);
571 }
572 
573 static inline u32 ipfs_readl(struct tegra_xudc *xudc, unsigned int offset)
574 {
575 	return readl(xudc->ipfs + offset);
576 }
577 
578 static inline void ipfs_writel(struct tegra_xudc *xudc, u32 val,
579 			       unsigned int offset)
580 {
581 	writel(val, xudc->ipfs + offset);
582 }
583 
584 static inline u32 xudc_readl(struct tegra_xudc *xudc, unsigned int offset)
585 {
586 	return readl(xudc->base + offset);
587 }
588 
589 static inline void xudc_writel(struct tegra_xudc *xudc, u32 val,
590 			       unsigned int offset)
591 {
592 	writel(val, xudc->base + offset);
593 }
594 
595 static inline int xudc_readl_poll(struct tegra_xudc *xudc,
596 				  unsigned int offset, u32 mask, u32 val)
597 {
598 	u32 regval;
599 
600 	return readl_poll_timeout_atomic(xudc->base + offset, regval,
601 					 (regval & mask) == val, 1, 100);
602 }
603 
604 static inline struct tegra_xudc *to_xudc(struct usb_gadget *gadget)
605 {
606 	return container_of(gadget, struct tegra_xudc, gadget);
607 }
608 
609 static inline struct tegra_xudc_ep *to_xudc_ep(struct usb_ep *ep)
610 {
611 	return container_of(ep, struct tegra_xudc_ep, usb_ep);
612 }
613 
614 static inline struct tegra_xudc_request *to_xudc_req(struct usb_request *req)
615 {
616 	return container_of(req, struct tegra_xudc_request, usb_req);
617 }
618 
619 static inline void dump_trb(struct tegra_xudc *xudc, const char *type,
620 			    struct tegra_xudc_trb *trb)
621 {
622 	dev_dbg(xudc->dev,
623 		"%s: %p, lo = %#x, hi = %#x, status = %#x, control = %#x\n",
624 		type, trb, trb->data_lo, trb->data_hi, trb->status,
625 		trb->control);
626 }
627 
628 static void tegra_xudc_limit_port_speed(struct tegra_xudc *xudc)
629 {
630 	u32 val;
631 
632 	/* limit port speed to gen 1 */
633 	val = xudc_readl(xudc, SSPX_CORE_CNT56);
634 	val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
635 	val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x260);
636 	xudc_writel(xudc, val, SSPX_CORE_CNT56);
637 
638 	val = xudc_readl(xudc, SSPX_CORE_CNT57);
639 	val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
640 	val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x6D6);
641 	xudc_writel(xudc, val, SSPX_CORE_CNT57);
642 
643 	val = xudc_readl(xudc, SSPX_CORE_CNT65);
644 	val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
645 	val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0x4B0);
646 	xudc_writel(xudc, val, SSPX_CORE_CNT66);
647 
648 	val = xudc_readl(xudc, SSPX_CORE_CNT66);
649 	val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
650 	val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x4B0);
651 	xudc_writel(xudc, val, SSPX_CORE_CNT66);
652 
653 	val = xudc_readl(xudc, SSPX_CORE_CNT67);
654 	val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
655 	val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x4B0);
656 	xudc_writel(xudc, val, SSPX_CORE_CNT67);
657 
658 	val = xudc_readl(xudc, SSPX_CORE_CNT72);
659 	val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
660 	val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x10);
661 	xudc_writel(xudc, val, SSPX_CORE_CNT72);
662 }
663 
664 static void tegra_xudc_restore_port_speed(struct tegra_xudc *xudc)
665 {
666 	u32 val;
667 
668 	/* restore port speed to gen2 */
669 	val = xudc_readl(xudc, SSPX_CORE_CNT56);
670 	val &= ~(SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX_MASK);
671 	val |= SSPX_CORE_CNT56_SCD_BIT0_TRPT_MAX(0x438);
672 	xudc_writel(xudc, val, SSPX_CORE_CNT56);
673 
674 	val = xudc_readl(xudc, SSPX_CORE_CNT57);
675 	val &= ~(SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX_MASK);
676 	val |= SSPX_CORE_CNT57_SCD_BIT1_TRPT_MAX(0x528);
677 	xudc_writel(xudc, val, SSPX_CORE_CNT57);
678 
679 	val = xudc_readl(xudc, SSPX_CORE_CNT65);
680 	val &= ~(SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID_MASK);
681 	val |= SSPX_CORE_CNT65_TX_SCD_END_TRPT_MID(0xE10);
682 	xudc_writel(xudc, val, SSPX_CORE_CNT66);
683 
684 	val = xudc_readl(xudc, SSPX_CORE_CNT66);
685 	val &= ~(SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID_MASK);
686 	val |= SSPX_CORE_CNT66_TX_SCD_BIT0_TRPT_MID(0x348);
687 	xudc_writel(xudc, val, SSPX_CORE_CNT66);
688 
689 	val = xudc_readl(xudc, SSPX_CORE_CNT67);
690 	val &= ~(SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID_MASK);
691 	val |= SSPX_CORE_CNT67_TX_SCD_BIT1_TRPT_MID(0x5a0);
692 	xudc_writel(xudc, val, SSPX_CORE_CNT67);
693 
694 	val = xudc_readl(xudc, SSPX_CORE_CNT72);
695 	val &= ~(SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT_MASK);
696 	val |= SSPX_CORE_CNT72_SCD_LFPS_TIMEOUT(0x1c21);
697 	xudc_writel(xudc, val, SSPX_CORE_CNT72);
698 }
699 
700 static void tegra_xudc_device_mode_on(struct tegra_xudc *xudc)
701 {
702 	int err;
703 
704 	pm_runtime_get_sync(xudc->dev);
705 
706 	err = phy_power_on(xudc->curr_utmi_phy);
707 	if (err < 0)
708 		dev_err(xudc->dev, "UTMI power on failed: %d\n", err);
709 
710 	err = phy_power_on(xudc->curr_usb3_phy);
711 	if (err < 0)
712 		dev_err(xudc->dev, "USB3 PHY power on failed: %d\n", err);
713 
714 	dev_dbg(xudc->dev, "device mode on\n");
715 
716 	phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
717 			 USB_ROLE_DEVICE);
718 }
719 
720 static void tegra_xudc_device_mode_off(struct tegra_xudc *xudc)
721 {
722 	bool connected = false;
723 	u32 pls, val;
724 	int err;
725 
726 	dev_dbg(xudc->dev, "device mode off\n");
727 
728 	connected = !!(xudc_readl(xudc, PORTSC) & PORTSC_CCS);
729 
730 	reinit_completion(&xudc->disconnect_complete);
731 
732 	if (xudc->soc->port_speed_quirk)
733 		tegra_xudc_restore_port_speed(xudc);
734 
735 	phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG, USB_ROLE_NONE);
736 
737 	pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
738 		PORTSC_PLS_SHIFT;
739 
740 	/* Direct link to U0 if disconnected in RESUME or U2. */
741 	if (xudc->soc->pls_quirk && xudc->gadget.speed == USB_SPEED_SUPER &&
742 	    (pls == PORTSC_PLS_RESUME || pls == PORTSC_PLS_U2)) {
743 		val = xudc_readl(xudc, PORTPM);
744 		val |= PORTPM_FRWE;
745 		xudc_writel(xudc, val, PORTPM);
746 
747 		val = xudc_readl(xudc, PORTSC);
748 		val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
749 		val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
750 		xudc_writel(xudc, val, PORTSC);
751 	}
752 
753 	/* Wait for disconnect event. */
754 	if (connected)
755 		wait_for_completion(&xudc->disconnect_complete);
756 
757 	/* Make sure interrupt handler has completed before powergating. */
758 	synchronize_irq(xudc->irq);
759 
760 	err = phy_power_off(xudc->curr_utmi_phy);
761 	if (err < 0)
762 		dev_err(xudc->dev, "UTMI PHY power off failed: %d\n", err);
763 
764 	err = phy_power_off(xudc->curr_usb3_phy);
765 	if (err < 0)
766 		dev_err(xudc->dev, "USB3 PHY power off failed: %d\n", err);
767 
768 	pm_runtime_put(xudc->dev);
769 }
770 
771 static void tegra_xudc_usb_role_sw_work(struct work_struct *work)
772 {
773 	struct tegra_xudc *xudc = container_of(work, struct tegra_xudc,
774 					       usb_role_sw_work);
775 
776 	if (xudc->device_mode)
777 		tegra_xudc_device_mode_on(xudc);
778 	else
779 		tegra_xudc_device_mode_off(xudc);
780 }
781 
782 static int tegra_xudc_get_phy_index(struct tegra_xudc *xudc,
783 					      struct usb_phy *usbphy)
784 {
785 	unsigned int i;
786 
787 	for (i = 0; i < xudc->soc->num_phys; i++) {
788 		if (xudc->usbphy[i] && usbphy == xudc->usbphy[i])
789 			return i;
790 	}
791 
792 	dev_info(xudc->dev, "phy index could not be found for shared USB PHY");
793 	return -1;
794 }
795 
796 static int tegra_xudc_vbus_notify(struct notifier_block *nb,
797 					 unsigned long action, void *data)
798 {
799 	struct tegra_xudc *xudc = container_of(nb, struct tegra_xudc,
800 					       vbus_nb);
801 	struct usb_phy *usbphy = (struct usb_phy *)data;
802 	int phy_index;
803 
804 	dev_dbg(xudc->dev, "%s(): event is %d\n", __func__, usbphy->last_event);
805 
806 	if ((xudc->device_mode && usbphy->last_event == USB_EVENT_VBUS) ||
807 	    (!xudc->device_mode && usbphy->last_event != USB_EVENT_VBUS)) {
808 		dev_dbg(xudc->dev, "Same role(%d) received. Ignore",
809 			xudc->device_mode);
810 		return NOTIFY_OK;
811 	}
812 
813 	xudc->device_mode = (usbphy->last_event == USB_EVENT_VBUS) ? true :
814 								     false;
815 
816 	phy_index = tegra_xudc_get_phy_index(xudc, usbphy);
817 	dev_dbg(xudc->dev, "%s(): current phy index is %d\n", __func__,
818 		phy_index);
819 
820 	if (!xudc->suspended && phy_index != -1) {
821 		xudc->curr_utmi_phy = xudc->utmi_phy[phy_index];
822 		xudc->curr_usb3_phy = xudc->usb3_phy[phy_index];
823 		xudc->curr_usbphy = usbphy;
824 		schedule_work(&xudc->usb_role_sw_work);
825 	}
826 
827 	return NOTIFY_OK;
828 }
829 
830 static void tegra_xudc_plc_reset_work(struct work_struct *work)
831 {
832 	struct delayed_work *dwork = to_delayed_work(work);
833 	struct tegra_xudc *xudc = container_of(dwork, struct tegra_xudc,
834 					       plc_reset_work);
835 	unsigned long flags;
836 
837 	spin_lock_irqsave(&xudc->lock, flags);
838 
839 	if (xudc->wait_csc) {
840 		u32 pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
841 			PORTSC_PLS_SHIFT;
842 
843 		if (pls == PORTSC_PLS_INACTIVE) {
844 			dev_info(xudc->dev, "PLS = Inactive. Toggle VBUS\n");
845 			phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
846 					 USB_ROLE_NONE);
847 			phy_set_mode_ext(xudc->curr_utmi_phy, PHY_MODE_USB_OTG,
848 					 USB_ROLE_DEVICE);
849 
850 			xudc->wait_csc = false;
851 		}
852 	}
853 
854 	spin_unlock_irqrestore(&xudc->lock, flags);
855 }
856 
857 static void tegra_xudc_port_reset_war_work(struct work_struct *work)
858 {
859 	struct delayed_work *dwork = to_delayed_work(work);
860 	struct tegra_xudc *xudc =
861 		container_of(dwork, struct tegra_xudc, port_reset_war_work);
862 	unsigned long flags;
863 	u32 pls;
864 	int ret;
865 
866 	spin_lock_irqsave(&xudc->lock, flags);
867 
868 	if (xudc->device_mode && xudc->wait_for_sec_prc) {
869 		pls = (xudc_readl(xudc, PORTSC) & PORTSC_PLS_MASK) >>
870 			PORTSC_PLS_SHIFT;
871 		dev_dbg(xudc->dev, "pls = %x\n", pls);
872 
873 		if (pls == PORTSC_PLS_DISABLED) {
874 			dev_dbg(xudc->dev, "toggle vbus\n");
875 			/* PRC doesn't complete in 100ms, toggle the vbus */
876 			ret = tegra_phy_xusb_utmi_port_reset(
877 				xudc->curr_utmi_phy);
878 			if (ret == 1)
879 				xudc->wait_for_sec_prc = 0;
880 		}
881 	}
882 
883 	spin_unlock_irqrestore(&xudc->lock, flags);
884 }
885 
886 static dma_addr_t trb_virt_to_phys(struct tegra_xudc_ep *ep,
887 				   struct tegra_xudc_trb *trb)
888 {
889 	unsigned int index;
890 
891 	index = trb - ep->transfer_ring;
892 
893 	if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
894 		return 0;
895 
896 	return (ep->transfer_ring_phys + index * sizeof(*trb));
897 }
898 
899 static struct tegra_xudc_trb *trb_phys_to_virt(struct tegra_xudc_ep *ep,
900 					       dma_addr_t addr)
901 {
902 	struct tegra_xudc_trb *trb;
903 	unsigned int index;
904 
905 	index = (addr - ep->transfer_ring_phys) / sizeof(*trb);
906 
907 	if (WARN_ON(index >= XUDC_TRANSFER_RING_SIZE))
908 		return NULL;
909 
910 	trb = &ep->transfer_ring[index];
911 
912 	return trb;
913 }
914 
915 static void ep_reload(struct tegra_xudc *xudc, unsigned int ep)
916 {
917 	xudc_writel(xudc, BIT(ep), EP_RELOAD);
918 	xudc_readl_poll(xudc, EP_RELOAD, BIT(ep), 0);
919 }
920 
921 static void ep_pause(struct tegra_xudc *xudc, unsigned int ep)
922 {
923 	u32 val;
924 
925 	val = xudc_readl(xudc, EP_PAUSE);
926 	if (val & BIT(ep))
927 		return;
928 	val |= BIT(ep);
929 
930 	xudc_writel(xudc, val, EP_PAUSE);
931 
932 	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
933 
934 	xudc_writel(xudc, BIT(ep), EP_STCHG);
935 }
936 
937 static void ep_unpause(struct tegra_xudc *xudc, unsigned int ep)
938 {
939 	u32 val;
940 
941 	val = xudc_readl(xudc, EP_PAUSE);
942 	if (!(val & BIT(ep)))
943 		return;
944 	val &= ~BIT(ep);
945 
946 	xudc_writel(xudc, val, EP_PAUSE);
947 
948 	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
949 
950 	xudc_writel(xudc, BIT(ep), EP_STCHG);
951 }
952 
953 static void ep_unpause_all(struct tegra_xudc *xudc)
954 {
955 	u32 val;
956 
957 	val = xudc_readl(xudc, EP_PAUSE);
958 
959 	xudc_writel(xudc, 0, EP_PAUSE);
960 
961 	xudc_readl_poll(xudc, EP_STCHG, val, val);
962 
963 	xudc_writel(xudc, val, EP_STCHG);
964 }
965 
966 static void ep_halt(struct tegra_xudc *xudc, unsigned int ep)
967 {
968 	u32 val;
969 
970 	val = xudc_readl(xudc, EP_HALT);
971 	if (val & BIT(ep))
972 		return;
973 	val |= BIT(ep);
974 	xudc_writel(xudc, val, EP_HALT);
975 
976 	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
977 
978 	xudc_writel(xudc, BIT(ep), EP_STCHG);
979 }
980 
981 static void ep_unhalt(struct tegra_xudc *xudc, unsigned int ep)
982 {
983 	u32 val;
984 
985 	val = xudc_readl(xudc, EP_HALT);
986 	if (!(val & BIT(ep)))
987 		return;
988 	val &= ~BIT(ep);
989 	xudc_writel(xudc, val, EP_HALT);
990 
991 	xudc_readl_poll(xudc, EP_STCHG, BIT(ep), BIT(ep));
992 
993 	xudc_writel(xudc, BIT(ep), EP_STCHG);
994 }
995 
996 static void ep_unhalt_all(struct tegra_xudc *xudc)
997 {
998 	u32 val;
999 
1000 	val = xudc_readl(xudc, EP_HALT);
1001 	if (!val)
1002 		return;
1003 	xudc_writel(xudc, 0, EP_HALT);
1004 
1005 	xudc_readl_poll(xudc, EP_STCHG, val, val);
1006 
1007 	xudc_writel(xudc, val, EP_STCHG);
1008 }
1009 
1010 static void ep_wait_for_stopped(struct tegra_xudc *xudc, unsigned int ep)
1011 {
1012 	xudc_readl_poll(xudc, EP_STOPPED, BIT(ep), BIT(ep));
1013 	xudc_writel(xudc, BIT(ep), EP_STOPPED);
1014 }
1015 
1016 static void ep_wait_for_inactive(struct tegra_xudc *xudc, unsigned int ep)
1017 {
1018 	xudc_readl_poll(xudc, EP_THREAD_ACTIVE, BIT(ep), 0);
1019 }
1020 
1021 static void tegra_xudc_req_done(struct tegra_xudc_ep *ep,
1022 				struct tegra_xudc_request *req, int status)
1023 {
1024 	struct tegra_xudc *xudc = ep->xudc;
1025 
1026 	dev_dbg(xudc->dev, "completing request %p on EP %u with status %d\n",
1027 		 req, ep->index, status);
1028 
1029 	if (likely(req->usb_req.status == -EINPROGRESS))
1030 		req->usb_req.status = status;
1031 
1032 	list_del_init(&req->list);
1033 
1034 	if (usb_endpoint_xfer_control(ep->desc)) {
1035 		usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
1036 					 (xudc->setup_state ==
1037 					  DATA_STAGE_XFER));
1038 	} else {
1039 		usb_gadget_unmap_request(&xudc->gadget, &req->usb_req,
1040 					 usb_endpoint_dir_in(ep->desc));
1041 	}
1042 
1043 	spin_unlock(&xudc->lock);
1044 	usb_gadget_giveback_request(&ep->usb_ep, &req->usb_req);
1045 	spin_lock(&xudc->lock);
1046 }
1047 
1048 static void tegra_xudc_ep_nuke(struct tegra_xudc_ep *ep, int status)
1049 {
1050 	struct tegra_xudc_request *req;
1051 
1052 	while (!list_empty(&ep->queue)) {
1053 		req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1054 				       list);
1055 		tegra_xudc_req_done(ep, req, status);
1056 	}
1057 }
1058 
1059 static unsigned int ep_available_trbs(struct tegra_xudc_ep *ep)
1060 {
1061 	if (ep->ring_full)
1062 		return 0;
1063 
1064 	if (ep->deq_ptr > ep->enq_ptr)
1065 		return ep->deq_ptr - ep->enq_ptr - 1;
1066 
1067 	return XUDC_TRANSFER_RING_SIZE - (ep->enq_ptr - ep->deq_ptr) - 2;
1068 }
1069 
1070 static void tegra_xudc_queue_one_trb(struct tegra_xudc_ep *ep,
1071 				     struct tegra_xudc_request *req,
1072 				     struct tegra_xudc_trb *trb,
1073 				     bool ioc)
1074 {
1075 	struct tegra_xudc *xudc = ep->xudc;
1076 	dma_addr_t buf_addr;
1077 	size_t len;
1078 
1079 	len = min_t(size_t, XUDC_TRB_MAX_BUFFER_SIZE, req->usb_req.length -
1080 		    req->buf_queued);
1081 	if (len > 0)
1082 		buf_addr = req->usb_req.dma + req->buf_queued;
1083 	else
1084 		buf_addr = 0;
1085 
1086 	trb_write_data_ptr(trb, buf_addr);
1087 
1088 	trb_write_transfer_len(trb, len);
1089 	trb_write_td_size(trb, req->trbs_needed - req->trbs_queued - 1);
1090 
1091 	if (req->trbs_queued == req->trbs_needed - 1 ||
1092 		(req->need_zlp && req->trbs_queued == req->trbs_needed - 2))
1093 		trb_write_chain(trb, 0);
1094 	else
1095 		trb_write_chain(trb, 1);
1096 
1097 	trb_write_ioc(trb, ioc);
1098 
1099 	if (usb_endpoint_dir_out(ep->desc) ||
1100 	    (usb_endpoint_xfer_control(ep->desc) &&
1101 	     (xudc->setup_state == DATA_STAGE_RECV)))
1102 		trb_write_isp(trb, 1);
1103 	else
1104 		trb_write_isp(trb, 0);
1105 
1106 	if (usb_endpoint_xfer_control(ep->desc)) {
1107 		if (xudc->setup_state == DATA_STAGE_XFER ||
1108 		    xudc->setup_state == DATA_STAGE_RECV)
1109 			trb_write_type(trb, TRB_TYPE_DATA_STAGE);
1110 		else
1111 			trb_write_type(trb, TRB_TYPE_STATUS_STAGE);
1112 
1113 		if (xudc->setup_state == DATA_STAGE_XFER ||
1114 		    xudc->setup_state == STATUS_STAGE_XFER)
1115 			trb_write_data_stage_dir(trb, 1);
1116 		else
1117 			trb_write_data_stage_dir(trb, 0);
1118 	} else if (usb_endpoint_xfer_isoc(ep->desc)) {
1119 		trb_write_type(trb, TRB_TYPE_ISOCH);
1120 		trb_write_sia(trb, 1);
1121 		trb_write_frame_id(trb, 0);
1122 		trb_write_tlbpc(trb, 0);
1123 	} else if (usb_ss_max_streams(ep->comp_desc)) {
1124 		trb_write_type(trb, TRB_TYPE_STREAM);
1125 		trb_write_stream_id(trb, req->usb_req.stream_id);
1126 	} else {
1127 		trb_write_type(trb, TRB_TYPE_NORMAL);
1128 		trb_write_stream_id(trb, 0);
1129 	}
1130 
1131 	trb_write_cycle(trb, ep->pcs);
1132 
1133 	req->trbs_queued++;
1134 	req->buf_queued += len;
1135 
1136 	dump_trb(xudc, "TRANSFER", trb);
1137 }
1138 
1139 static unsigned int tegra_xudc_queue_trbs(struct tegra_xudc_ep *ep,
1140 					  struct tegra_xudc_request *req)
1141 {
1142 	unsigned int i, count, available;
1143 	bool wait_td = false;
1144 
1145 	available = ep_available_trbs(ep);
1146 	count = req->trbs_needed - req->trbs_queued;
1147 	if (available < count) {
1148 		count = available;
1149 		ep->ring_full = true;
1150 	}
1151 
1152 	/*
1153 	 * To generate zero-length packet on USB bus, SW needs schedule a
1154 	 * standalone zero-length TD. According to HW's behavior, SW needs
1155 	 * to schedule TDs in different ways for different endpoint types.
1156 	 *
1157 	 * For control endpoint:
1158 	 * - Data stage TD (IOC = 1, CH = 0)
1159 	 * - Ring doorbell and wait transfer event
1160 	 * - Data stage TD for ZLP (IOC = 1, CH = 0)
1161 	 * - Ring doorbell
1162 	 *
1163 	 * For bulk and interrupt endpoints:
1164 	 * - Normal transfer TD (IOC = 0, CH = 0)
1165 	 * - Normal transfer TD for ZLP (IOC = 1, CH = 0)
1166 	 * - Ring doorbell
1167 	 */
1168 
1169 	if (req->need_zlp && usb_endpoint_xfer_control(ep->desc) && count > 1)
1170 		wait_td = true;
1171 
1172 	if (!req->first_trb)
1173 		req->first_trb = &ep->transfer_ring[ep->enq_ptr];
1174 
1175 	for (i = 0; i < count; i++) {
1176 		struct tegra_xudc_trb *trb = &ep->transfer_ring[ep->enq_ptr];
1177 		bool ioc = false;
1178 
1179 		if ((i == count - 1) || (wait_td && i == count - 2))
1180 			ioc = true;
1181 
1182 		tegra_xudc_queue_one_trb(ep, req, trb, ioc);
1183 		req->last_trb = trb;
1184 
1185 		ep->enq_ptr++;
1186 		if (ep->enq_ptr == XUDC_TRANSFER_RING_SIZE - 1) {
1187 			trb = &ep->transfer_ring[ep->enq_ptr];
1188 			trb_write_cycle(trb, ep->pcs);
1189 			ep->pcs = !ep->pcs;
1190 			ep->enq_ptr = 0;
1191 		}
1192 
1193 		if (ioc)
1194 			break;
1195 	}
1196 
1197 	return count;
1198 }
1199 
1200 static void tegra_xudc_ep_ring_doorbell(struct tegra_xudc_ep *ep)
1201 {
1202 	struct tegra_xudc *xudc = ep->xudc;
1203 	u32 val;
1204 
1205 	if (list_empty(&ep->queue))
1206 		return;
1207 
1208 	val = DB_TARGET(ep->index);
1209 	if (usb_endpoint_xfer_control(ep->desc)) {
1210 		val |= DB_STREAMID(xudc->setup_seq_num);
1211 	} else if (usb_ss_max_streams(ep->comp_desc) > 0) {
1212 		struct tegra_xudc_request *req;
1213 
1214 		/* Don't ring doorbell if the stream has been rejected. */
1215 		if (ep->stream_rejected)
1216 			return;
1217 
1218 		req = list_first_entry(&ep->queue, struct tegra_xudc_request,
1219 				       list);
1220 		val |= DB_STREAMID(req->usb_req.stream_id);
1221 	}
1222 
1223 	dev_dbg(xudc->dev, "ring doorbell: %#x\n", val);
1224 	xudc_writel(xudc, val, DB);
1225 }
1226 
1227 static void tegra_xudc_ep_kick_queue(struct tegra_xudc_ep *ep)
1228 {
1229 	struct tegra_xudc_request *req;
1230 	bool trbs_queued = false;
1231 
1232 	list_for_each_entry(req, &ep->queue, list) {
1233 		if (ep->ring_full)
1234 			break;
1235 
1236 		if (tegra_xudc_queue_trbs(ep, req) > 0)
1237 			trbs_queued = true;
1238 	}
1239 
1240 	if (trbs_queued)
1241 		tegra_xudc_ep_ring_doorbell(ep);
1242 }
1243 
1244 static int
1245 __tegra_xudc_ep_queue(struct tegra_xudc_ep *ep, struct tegra_xudc_request *req)
1246 {
1247 	struct tegra_xudc *xudc = ep->xudc;
1248 	int err;
1249 
1250 	if (usb_endpoint_xfer_control(ep->desc) && !list_empty(&ep->queue)) {
1251 		dev_err(xudc->dev, "control EP has pending transfers\n");
1252 		return -EINVAL;
1253 	}
1254 
1255 	if (usb_endpoint_xfer_control(ep->desc)) {
1256 		err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1257 					     (xudc->setup_state ==
1258 					      DATA_STAGE_XFER));
1259 	} else {
1260 		err = usb_gadget_map_request(&xudc->gadget, &req->usb_req,
1261 					     usb_endpoint_dir_in(ep->desc));
1262 	}
1263 
1264 	if (err < 0) {
1265 		dev_err(xudc->dev, "failed to map request: %d\n", err);
1266 		return err;
1267 	}
1268 
1269 	req->first_trb = NULL;
1270 	req->last_trb = NULL;
1271 	req->buf_queued = 0;
1272 	req->trbs_queued = 0;
1273 	req->need_zlp = false;
1274 	req->trbs_needed = DIV_ROUND_UP(req->usb_req.length,
1275 					XUDC_TRB_MAX_BUFFER_SIZE);
1276 	if (req->usb_req.length == 0)
1277 		req->trbs_needed++;
1278 
1279 	if (!usb_endpoint_xfer_isoc(ep->desc) &&
1280 	    req->usb_req.zero && req->usb_req.length &&
1281 	    ((req->usb_req.length % ep->usb_ep.maxpacket) == 0)) {
1282 		req->trbs_needed++;
1283 		req->need_zlp = true;
1284 	}
1285 
1286 	req->usb_req.status = -EINPROGRESS;
1287 	req->usb_req.actual = 0;
1288 
1289 	list_add_tail(&req->list, &ep->queue);
1290 
1291 	tegra_xudc_ep_kick_queue(ep);
1292 
1293 	return 0;
1294 }
1295 
1296 static int
1297 tegra_xudc_ep_queue(struct usb_ep *usb_ep, struct usb_request *usb_req,
1298 		    gfp_t gfp)
1299 {
1300 	struct tegra_xudc_request *req;
1301 	struct tegra_xudc_ep *ep;
1302 	struct tegra_xudc *xudc;
1303 	unsigned long flags;
1304 	int ret;
1305 
1306 	if (!usb_ep || !usb_req)
1307 		return -EINVAL;
1308 
1309 	ep = to_xudc_ep(usb_ep);
1310 	req = to_xudc_req(usb_req);
1311 	xudc = ep->xudc;
1312 
1313 	spin_lock_irqsave(&xudc->lock, flags);
1314 	if (xudc->powergated || !ep->desc) {
1315 		ret = -ESHUTDOWN;
1316 		goto unlock;
1317 	}
1318 
1319 	ret = __tegra_xudc_ep_queue(ep, req);
1320 unlock:
1321 	spin_unlock_irqrestore(&xudc->lock, flags);
1322 
1323 	return ret;
1324 }
1325 
1326 static void squeeze_transfer_ring(struct tegra_xudc_ep *ep,
1327 				  struct tegra_xudc_request *req)
1328 {
1329 	struct tegra_xudc_trb *trb = req->first_trb;
1330 	bool pcs_enq = trb_read_cycle(trb);
1331 	bool pcs;
1332 
1333 	/*
1334 	 * Clear out all the TRBs part of or after the cancelled request,
1335 	 * and must correct trb cycle bit to the last un-enqueued state.
1336 	 */
1337 	while (trb != &ep->transfer_ring[ep->enq_ptr]) {
1338 		pcs = trb_read_cycle(trb);
1339 		memset(trb, 0, sizeof(*trb));
1340 		trb_write_cycle(trb, !pcs);
1341 		trb++;
1342 
1343 		if (trb_read_type(trb) == TRB_TYPE_LINK)
1344 			trb = ep->transfer_ring;
1345 	}
1346 
1347 	/* Requests will be re-queued at the start of the cancelled request. */
1348 	ep->enq_ptr = req->first_trb - ep->transfer_ring;
1349 	/*
1350 	 * Retrieve the correct cycle bit state from the first trb of
1351 	 * the cancelled request.
1352 	 */
1353 	ep->pcs = pcs_enq;
1354 	ep->ring_full = false;
1355 	list_for_each_entry_continue(req, &ep->queue, list) {
1356 		req->usb_req.status = -EINPROGRESS;
1357 		req->usb_req.actual = 0;
1358 
1359 		req->first_trb = NULL;
1360 		req->last_trb = NULL;
1361 		req->buf_queued = 0;
1362 		req->trbs_queued = 0;
1363 	}
1364 }
1365 
1366 /*
1367  * Determine if the given TRB is in the range [first trb, last trb] for the
1368  * given request.
1369  */
1370 static bool trb_in_request(struct tegra_xudc_ep *ep,
1371 			   struct tegra_xudc_request *req,
1372 			   struct tegra_xudc_trb *trb)
1373 {
1374 	dev_dbg(ep->xudc->dev, "%s: request %p -> %p; trb %p\n", __func__,
1375 		req->first_trb, req->last_trb, trb);
1376 
1377 	if (trb >= req->first_trb && (trb <= req->last_trb ||
1378 				      req->last_trb < req->first_trb))
1379 		return true;
1380 
1381 	if (trb < req->first_trb && trb <= req->last_trb &&
1382 	    req->last_trb < req->first_trb)
1383 		return true;
1384 
1385 	return false;
1386 }
1387 
1388 /*
1389  * Determine if the given TRB is in the range [EP enqueue pointer, first TRB)
1390  * for the given endpoint and request.
1391  */
1392 static bool trb_before_request(struct tegra_xudc_ep *ep,
1393 			       struct tegra_xudc_request *req,
1394 			       struct tegra_xudc_trb *trb)
1395 {
1396 	struct tegra_xudc_trb *enq_trb = &ep->transfer_ring[ep->enq_ptr];
1397 
1398 	dev_dbg(ep->xudc->dev, "%s: request %p -> %p; enq ptr: %p; trb %p\n",
1399 		__func__, req->first_trb, req->last_trb, enq_trb, trb);
1400 
1401 	if (trb < req->first_trb && (enq_trb <= trb ||
1402 				     req->first_trb < enq_trb))
1403 		return true;
1404 
1405 	if (trb > req->first_trb && req->first_trb < enq_trb && enq_trb <= trb)
1406 		return true;
1407 
1408 	return false;
1409 }
1410 
1411 static int
1412 __tegra_xudc_ep_dequeue(struct tegra_xudc_ep *ep,
1413 			struct tegra_xudc_request *req)
1414 {
1415 	struct tegra_xudc *xudc = ep->xudc;
1416 	struct tegra_xudc_request *r;
1417 	struct tegra_xudc_trb *deq_trb;
1418 	bool busy, kick_queue = false;
1419 	int ret = 0;
1420 
1421 	/* Make sure the request is actually queued to this endpoint. */
1422 	list_for_each_entry(r, &ep->queue, list) {
1423 		if (r == req)
1424 			break;
1425 	}
1426 
1427 	if (r != req)
1428 		return -EINVAL;
1429 
1430 	/* Request hasn't been queued in the transfer ring yet. */
1431 	if (!req->trbs_queued) {
1432 		tegra_xudc_req_done(ep, req, -ECONNRESET);
1433 		return 0;
1434 	}
1435 
1436 	/* Halt DMA for this endpiont. */
1437 	if (ep_ctx_read_state(ep->context) == EP_STATE_RUNNING) {
1438 		ep_pause(xudc, ep->index);
1439 		ep_wait_for_inactive(xudc, ep->index);
1440 	}
1441 
1442 	deq_trb = trb_phys_to_virt(ep, ep_ctx_read_deq_ptr(ep->context));
1443 	/* Is the hardware processing the TRB at the dequeue pointer? */
1444 	busy = (trb_read_cycle(deq_trb) == ep_ctx_read_dcs(ep->context));
1445 
1446 	if (trb_in_request(ep, req, deq_trb) && busy) {
1447 		/*
1448 		 * Request has been partially completed or it hasn't
1449 		 * started processing yet.
1450 		 */
1451 		dma_addr_t deq_ptr;
1452 
1453 		squeeze_transfer_ring(ep, req);
1454 
1455 		req->usb_req.actual = ep_ctx_read_edtla(ep->context);
1456 		tegra_xudc_req_done(ep, req, -ECONNRESET);
1457 		kick_queue = true;
1458 
1459 		/* EDTLA is > 0: request has been partially completed */
1460 		if (req->usb_req.actual > 0) {
1461 			/*
1462 			 * Abort the pending transfer and update the dequeue
1463 			 * pointer
1464 			 */
1465 			ep_ctx_write_edtla(ep->context, 0);
1466 			ep_ctx_write_partial_td(ep->context, 0);
1467 			ep_ctx_write_data_offset(ep->context, 0);
1468 
1469 			deq_ptr = trb_virt_to_phys(ep,
1470 					&ep->transfer_ring[ep->enq_ptr]);
1471 
1472 			if (dma_mapping_error(xudc->dev, deq_ptr)) {
1473 				ret = -EINVAL;
1474 			} else {
1475 				ep_ctx_write_deq_ptr(ep->context, deq_ptr);
1476 				ep_ctx_write_dcs(ep->context, ep->pcs);
1477 				ep_reload(xudc, ep->index);
1478 			}
1479 		}
1480 	} else if (trb_before_request(ep, req, deq_trb) && busy) {
1481 		/* Request hasn't started processing yet. */
1482 		squeeze_transfer_ring(ep, req);
1483 
1484 		tegra_xudc_req_done(ep, req, -ECONNRESET);
1485 		kick_queue = true;
1486 	} else {
1487 		/*
1488 		 * Request has completed, but we haven't processed the
1489 		 * completion event yet.
1490 		 */
1491 		tegra_xudc_req_done(ep, req, -ECONNRESET);
1492 		ret = -EINVAL;
1493 	}
1494 
1495 	/* Resume the endpoint. */
1496 	ep_unpause(xudc, ep->index);
1497 
1498 	if (kick_queue)
1499 		tegra_xudc_ep_kick_queue(ep);
1500 
1501 	return ret;
1502 }
1503 
1504 static int
1505 tegra_xudc_ep_dequeue(struct usb_ep *usb_ep, struct usb_request *usb_req)
1506 {
1507 	struct tegra_xudc_request *req;
1508 	struct tegra_xudc_ep *ep;
1509 	struct tegra_xudc *xudc;
1510 	unsigned long flags;
1511 	int ret;
1512 
1513 	if (!usb_ep || !usb_req)
1514 		return -EINVAL;
1515 
1516 	ep = to_xudc_ep(usb_ep);
1517 	req = to_xudc_req(usb_req);
1518 	xudc = ep->xudc;
1519 
1520 	spin_lock_irqsave(&xudc->lock, flags);
1521 
1522 	if (xudc->powergated || !ep->desc) {
1523 		ret = -ESHUTDOWN;
1524 		goto unlock;
1525 	}
1526 
1527 	ret = __tegra_xudc_ep_dequeue(ep, req);
1528 unlock:
1529 	spin_unlock_irqrestore(&xudc->lock, flags);
1530 
1531 	return ret;
1532 }
1533 
1534 static int __tegra_xudc_ep_set_halt(struct tegra_xudc_ep *ep, bool halt)
1535 {
1536 	struct tegra_xudc *xudc = ep->xudc;
1537 
1538 	if (!ep->desc)
1539 		return -EINVAL;
1540 
1541 	if (usb_endpoint_xfer_isoc(ep->desc)) {
1542 		dev_err(xudc->dev, "can't halt isochronous EP\n");
1543 		return -ENOTSUPP;
1544 	}
1545 
1546 	if (!!(xudc_readl(xudc, EP_HALT) & BIT(ep->index)) == halt) {
1547 		dev_dbg(xudc->dev, "EP %u already %s\n", ep->index,
1548 			halt ? "halted" : "not halted");
1549 		return 0;
1550 	}
1551 
1552 	if (halt) {
1553 		ep_halt(xudc, ep->index);
1554 	} else {
1555 		ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1556 
1557 		ep_reload(xudc, ep->index);
1558 
1559 		ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1560 		ep_ctx_write_seq_num(ep->context, 0);
1561 
1562 		ep_reload(xudc, ep->index);
1563 		ep_unpause(xudc, ep->index);
1564 		ep_unhalt(xudc, ep->index);
1565 
1566 		tegra_xudc_ep_ring_doorbell(ep);
1567 	}
1568 
1569 	return 0;
1570 }
1571 
1572 static int tegra_xudc_ep_set_halt(struct usb_ep *usb_ep, int value)
1573 {
1574 	struct tegra_xudc_ep *ep;
1575 	struct tegra_xudc *xudc;
1576 	unsigned long flags;
1577 	int ret;
1578 
1579 	if (!usb_ep)
1580 		return -EINVAL;
1581 
1582 	ep = to_xudc_ep(usb_ep);
1583 	xudc = ep->xudc;
1584 
1585 	spin_lock_irqsave(&xudc->lock, flags);
1586 	if (xudc->powergated) {
1587 		ret = -ESHUTDOWN;
1588 		goto unlock;
1589 	}
1590 
1591 	if (value && usb_endpoint_dir_in(ep->desc) &&
1592 	    !list_empty(&ep->queue)) {
1593 		dev_err(xudc->dev, "can't halt EP with requests pending\n");
1594 		ret = -EAGAIN;
1595 		goto unlock;
1596 	}
1597 
1598 	ret = __tegra_xudc_ep_set_halt(ep, value);
1599 unlock:
1600 	spin_unlock_irqrestore(&xudc->lock, flags);
1601 
1602 	return ret;
1603 }
1604 
1605 static void tegra_xudc_ep_context_setup(struct tegra_xudc_ep *ep)
1606 {
1607 	const struct usb_endpoint_descriptor *desc = ep->desc;
1608 	const struct usb_ss_ep_comp_descriptor *comp_desc = ep->comp_desc;
1609 	struct tegra_xudc *xudc = ep->xudc;
1610 	u16 maxpacket, maxburst = 0, esit = 0;
1611 	u32 val;
1612 
1613 	maxpacket = usb_endpoint_maxp(desc) & 0x7ff;
1614 	if (xudc->gadget.speed == USB_SPEED_SUPER) {
1615 		if (!usb_endpoint_xfer_control(desc))
1616 			maxburst = comp_desc->bMaxBurst;
1617 
1618 		if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc))
1619 			esit = le16_to_cpu(comp_desc->wBytesPerInterval);
1620 	} else if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
1621 		   (usb_endpoint_xfer_int(desc) ||
1622 		    usb_endpoint_xfer_isoc(desc))) {
1623 		if (xudc->gadget.speed == USB_SPEED_HIGH) {
1624 			maxburst = (usb_endpoint_maxp(desc) >> 11) & 0x3;
1625 			if (maxburst == 0x3) {
1626 				dev_warn(xudc->dev,
1627 					 "invalid endpoint maxburst\n");
1628 				maxburst = 0x2;
1629 			}
1630 		}
1631 		esit = maxpacket * (maxburst + 1);
1632 	}
1633 
1634 	memset(ep->context, 0, sizeof(*ep->context));
1635 
1636 	ep_ctx_write_state(ep->context, EP_STATE_RUNNING);
1637 	ep_ctx_write_interval(ep->context, desc->bInterval);
1638 	if (xudc->gadget.speed == USB_SPEED_SUPER) {
1639 		if (usb_endpoint_xfer_isoc(desc)) {
1640 			ep_ctx_write_mult(ep->context,
1641 					  comp_desc->bmAttributes & 0x3);
1642 		}
1643 
1644 		if (usb_endpoint_xfer_bulk(desc)) {
1645 			ep_ctx_write_max_pstreams(ep->context,
1646 						  comp_desc->bmAttributes &
1647 						  0x1f);
1648 			ep_ctx_write_lsa(ep->context, 1);
1649 		}
1650 	}
1651 
1652 	if (!usb_endpoint_xfer_control(desc) && usb_endpoint_dir_out(desc))
1653 		val = usb_endpoint_type(desc);
1654 	else
1655 		val = usb_endpoint_type(desc) + EP_TYPE_CONTROL;
1656 
1657 	ep_ctx_write_type(ep->context, val);
1658 	ep_ctx_write_cerr(ep->context, 0x3);
1659 	ep_ctx_write_max_packet_size(ep->context, maxpacket);
1660 	ep_ctx_write_max_burst_size(ep->context, maxburst);
1661 
1662 	ep_ctx_write_deq_ptr(ep->context, ep->transfer_ring_phys);
1663 	ep_ctx_write_dcs(ep->context, ep->pcs);
1664 
1665 	/* Select a reasonable average TRB length based on endpoint type. */
1666 	switch (usb_endpoint_type(desc)) {
1667 	case USB_ENDPOINT_XFER_CONTROL:
1668 		val = 8;
1669 		break;
1670 	case USB_ENDPOINT_XFER_INT:
1671 		val = 1024;
1672 		break;
1673 	case USB_ENDPOINT_XFER_BULK:
1674 	case USB_ENDPOINT_XFER_ISOC:
1675 	default:
1676 		val = 3072;
1677 		break;
1678 	}
1679 
1680 	ep_ctx_write_avg_trb_len(ep->context, val);
1681 	ep_ctx_write_max_esit_payload(ep->context, esit);
1682 
1683 	ep_ctx_write_cerrcnt(ep->context, 0x3);
1684 }
1685 
1686 static void setup_link_trb(struct tegra_xudc_ep *ep,
1687 			   struct tegra_xudc_trb *trb)
1688 {
1689 	trb_write_data_ptr(trb, ep->transfer_ring_phys);
1690 	trb_write_type(trb, TRB_TYPE_LINK);
1691 	trb_write_toggle_cycle(trb, 1);
1692 }
1693 
1694 static int __tegra_xudc_ep_disable(struct tegra_xudc_ep *ep)
1695 {
1696 	struct tegra_xudc *xudc = ep->xudc;
1697 
1698 	if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
1699 		dev_err(xudc->dev, "endpoint %u already disabled\n",
1700 			ep->index);
1701 		return -EINVAL;
1702 	}
1703 
1704 	ep_ctx_write_state(ep->context, EP_STATE_DISABLED);
1705 
1706 	ep_reload(xudc, ep->index);
1707 
1708 	tegra_xudc_ep_nuke(ep, -ESHUTDOWN);
1709 
1710 	xudc->nr_enabled_eps--;
1711 	if (usb_endpoint_xfer_isoc(ep->desc))
1712 		xudc->nr_isoch_eps--;
1713 
1714 	ep->desc = NULL;
1715 	ep->comp_desc = NULL;
1716 
1717 	memset(ep->context, 0, sizeof(*ep->context));
1718 
1719 	ep_unpause(xudc, ep->index);
1720 	ep_unhalt(xudc, ep->index);
1721 	if (xudc_readl(xudc, EP_STOPPED) & BIT(ep->index))
1722 		xudc_writel(xudc, BIT(ep->index), EP_STOPPED);
1723 
1724 	/*
1725 	 * If this is the last endpoint disabled in a de-configure request,
1726 	 * switch back to address state.
1727 	 */
1728 	if ((xudc->device_state == USB_STATE_CONFIGURED) &&
1729 	    (xudc->nr_enabled_eps == 1)) {
1730 		u32 val;
1731 
1732 		xudc->device_state = USB_STATE_ADDRESS;
1733 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1734 
1735 		val = xudc_readl(xudc, CTRL);
1736 		val &= ~CTRL_RUN;
1737 		xudc_writel(xudc, val, CTRL);
1738 	}
1739 
1740 	dev_info(xudc->dev, "ep %u disabled\n", ep->index);
1741 
1742 	return 0;
1743 }
1744 
1745 static int tegra_xudc_ep_disable(struct usb_ep *usb_ep)
1746 {
1747 	struct tegra_xudc_ep *ep;
1748 	struct tegra_xudc *xudc;
1749 	unsigned long flags;
1750 	int ret;
1751 
1752 	if (!usb_ep)
1753 		return -EINVAL;
1754 
1755 	ep = to_xudc_ep(usb_ep);
1756 	xudc = ep->xudc;
1757 
1758 	spin_lock_irqsave(&xudc->lock, flags);
1759 	if (xudc->powergated) {
1760 		ret = -ESHUTDOWN;
1761 		goto unlock;
1762 	}
1763 
1764 	ret = __tegra_xudc_ep_disable(ep);
1765 unlock:
1766 	spin_unlock_irqrestore(&xudc->lock, flags);
1767 
1768 	return ret;
1769 }
1770 
1771 static int __tegra_xudc_ep_enable(struct tegra_xudc_ep *ep,
1772 				  const struct usb_endpoint_descriptor *desc)
1773 {
1774 	struct tegra_xudc *xudc = ep->xudc;
1775 	unsigned int i;
1776 	u32 val;
1777 
1778 	if (xudc->gadget.speed == USB_SPEED_SUPER &&
1779 		!usb_endpoint_xfer_control(desc) && !ep->usb_ep.comp_desc)
1780 		return -EINVAL;
1781 
1782 	/* Disable the EP if it is not disabled */
1783 	if (ep_ctx_read_state(ep->context) != EP_STATE_DISABLED)
1784 		__tegra_xudc_ep_disable(ep);
1785 
1786 	ep->desc = desc;
1787 	ep->comp_desc = ep->usb_ep.comp_desc;
1788 
1789 	if (usb_endpoint_xfer_isoc(desc)) {
1790 		if (xudc->nr_isoch_eps > XUDC_MAX_ISOCH_EPS) {
1791 			dev_err(xudc->dev, "too many isochronous endpoints\n");
1792 			return -EBUSY;
1793 		}
1794 		xudc->nr_isoch_eps++;
1795 	}
1796 
1797 	memset(ep->transfer_ring, 0, XUDC_TRANSFER_RING_SIZE *
1798 	       sizeof(*ep->transfer_ring));
1799 	setup_link_trb(ep, &ep->transfer_ring[XUDC_TRANSFER_RING_SIZE - 1]);
1800 
1801 	ep->enq_ptr = 0;
1802 	ep->deq_ptr = 0;
1803 	ep->pcs = true;
1804 	ep->ring_full = false;
1805 	xudc->nr_enabled_eps++;
1806 
1807 	tegra_xudc_ep_context_setup(ep);
1808 
1809 	/*
1810 	 * No need to reload and un-halt EP0.  This will be done automatically
1811 	 * once a valid SETUP packet is received.
1812 	 */
1813 	if (usb_endpoint_xfer_control(desc))
1814 		goto out;
1815 
1816 	/*
1817 	 * Transition to configured state once the first non-control
1818 	 * endpoint is enabled.
1819 	 */
1820 	if (xudc->device_state == USB_STATE_ADDRESS) {
1821 		val = xudc_readl(xudc, CTRL);
1822 		val |= CTRL_RUN;
1823 		xudc_writel(xudc, val, CTRL);
1824 
1825 		xudc->device_state = USB_STATE_CONFIGURED;
1826 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1827 	}
1828 
1829 	if (usb_endpoint_xfer_isoc(desc)) {
1830 		/*
1831 		 * Pause all bulk endpoints when enabling an isoch endpoint
1832 		 * to ensure the isoch endpoint is allocated enough bandwidth.
1833 		 */
1834 		for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1835 			if (xudc->ep[i].desc &&
1836 			    usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1837 				ep_pause(xudc, i);
1838 		}
1839 	}
1840 
1841 	ep_reload(xudc, ep->index);
1842 	ep_unpause(xudc, ep->index);
1843 	ep_unhalt(xudc, ep->index);
1844 
1845 	if (usb_endpoint_xfer_isoc(desc)) {
1846 		for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
1847 			if (xudc->ep[i].desc &&
1848 			    usb_endpoint_xfer_bulk(xudc->ep[i].desc))
1849 				ep_unpause(xudc, i);
1850 		}
1851 	}
1852 
1853 out:
1854 	dev_info(xudc->dev, "EP %u (type: %s, dir: %s) enabled\n", ep->index,
1855 		 usb_ep_type_string(usb_endpoint_type(ep->desc)),
1856 		 usb_endpoint_dir_in(ep->desc) ? "in" : "out");
1857 
1858 	return 0;
1859 }
1860 
1861 static int tegra_xudc_ep_enable(struct usb_ep *usb_ep,
1862 				const struct usb_endpoint_descriptor *desc)
1863 {
1864 	struct tegra_xudc_ep *ep;
1865 	struct tegra_xudc *xudc;
1866 	unsigned long flags;
1867 	int ret;
1868 
1869 	if  (!usb_ep || !desc || (desc->bDescriptorType != USB_DT_ENDPOINT))
1870 		return -EINVAL;
1871 
1872 	ep = to_xudc_ep(usb_ep);
1873 	xudc = ep->xudc;
1874 
1875 	spin_lock_irqsave(&xudc->lock, flags);
1876 	if (xudc->powergated) {
1877 		ret = -ESHUTDOWN;
1878 		goto unlock;
1879 	}
1880 
1881 	ret = __tegra_xudc_ep_enable(ep, desc);
1882 unlock:
1883 	spin_unlock_irqrestore(&xudc->lock, flags);
1884 
1885 	return ret;
1886 }
1887 
1888 static struct usb_request *
1889 tegra_xudc_ep_alloc_request(struct usb_ep *usb_ep, gfp_t gfp)
1890 {
1891 	struct tegra_xudc_request *req;
1892 
1893 	req = kzalloc(sizeof(*req), gfp);
1894 	if (!req)
1895 		return NULL;
1896 
1897 	INIT_LIST_HEAD(&req->list);
1898 
1899 	return &req->usb_req;
1900 }
1901 
1902 static void tegra_xudc_ep_free_request(struct usb_ep *usb_ep,
1903 				       struct usb_request *usb_req)
1904 {
1905 	struct tegra_xudc_request *req = to_xudc_req(usb_req);
1906 
1907 	kfree(req);
1908 }
1909 
1910 static struct usb_ep_ops tegra_xudc_ep_ops = {
1911 	.enable = tegra_xudc_ep_enable,
1912 	.disable = tegra_xudc_ep_disable,
1913 	.alloc_request = tegra_xudc_ep_alloc_request,
1914 	.free_request = tegra_xudc_ep_free_request,
1915 	.queue = tegra_xudc_ep_queue,
1916 	.dequeue = tegra_xudc_ep_dequeue,
1917 	.set_halt = tegra_xudc_ep_set_halt,
1918 };
1919 
1920 static int tegra_xudc_ep0_enable(struct usb_ep *usb_ep,
1921 				 const struct usb_endpoint_descriptor *desc)
1922 {
1923 	return -EBUSY;
1924 }
1925 
1926 static int tegra_xudc_ep0_disable(struct usb_ep *usb_ep)
1927 {
1928 	return -EBUSY;
1929 }
1930 
1931 static struct usb_ep_ops tegra_xudc_ep0_ops = {
1932 	.enable = tegra_xudc_ep0_enable,
1933 	.disable = tegra_xudc_ep0_disable,
1934 	.alloc_request = tegra_xudc_ep_alloc_request,
1935 	.free_request = tegra_xudc_ep_free_request,
1936 	.queue = tegra_xudc_ep_queue,
1937 	.dequeue = tegra_xudc_ep_dequeue,
1938 	.set_halt = tegra_xudc_ep_set_halt,
1939 };
1940 
1941 static int tegra_xudc_gadget_get_frame(struct usb_gadget *gadget)
1942 {
1943 	struct tegra_xudc *xudc = to_xudc(gadget);
1944 	unsigned long flags;
1945 	int ret;
1946 
1947 	spin_lock_irqsave(&xudc->lock, flags);
1948 	if (xudc->powergated) {
1949 		ret = -ESHUTDOWN;
1950 		goto unlock;
1951 	}
1952 
1953 	ret = (xudc_readl(xudc, MFINDEX) & MFINDEX_FRAME_MASK) >>
1954 		MFINDEX_FRAME_SHIFT;
1955 unlock:
1956 	spin_unlock_irqrestore(&xudc->lock, flags);
1957 
1958 	return ret;
1959 }
1960 
1961 static void tegra_xudc_resume_device_state(struct tegra_xudc *xudc)
1962 {
1963 	unsigned int i;
1964 	u32 val;
1965 
1966 	ep_unpause_all(xudc);
1967 
1968 	/* Direct link to U0. */
1969 	val = xudc_readl(xudc, PORTSC);
1970 	if (((val & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT) != PORTSC_PLS_U0) {
1971 		val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
1972 		val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_U0);
1973 		xudc_writel(xudc, val, PORTSC);
1974 	}
1975 
1976 	if (xudc->device_state == USB_STATE_SUSPENDED) {
1977 		xudc->device_state = xudc->resume_state;
1978 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
1979 		xudc->resume_state = 0;
1980 	}
1981 
1982 	/*
1983 	 * Doorbells may be dropped if they are sent too soon (< ~200ns)
1984 	 * after unpausing the endpoint.  Wait for 500ns just to be safe.
1985 	 */
1986 	ndelay(500);
1987 	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
1988 		tegra_xudc_ep_ring_doorbell(&xudc->ep[i]);
1989 }
1990 
1991 static int tegra_xudc_gadget_wakeup(struct usb_gadget *gadget)
1992 {
1993 	struct tegra_xudc *xudc = to_xudc(gadget);
1994 	unsigned long flags;
1995 	int ret = 0;
1996 	u32 val;
1997 
1998 	spin_lock_irqsave(&xudc->lock, flags);
1999 
2000 	if (xudc->powergated) {
2001 		ret = -ESHUTDOWN;
2002 		goto unlock;
2003 	}
2004 	val = xudc_readl(xudc, PORTPM);
2005 	dev_dbg(xudc->dev, "%s: PORTPM=%#x, speed=%x\n", __func__,
2006 			val, gadget->speed);
2007 
2008 	if (((xudc->gadget.speed <= USB_SPEED_HIGH) &&
2009 	     (val & PORTPM_RWE)) ||
2010 	    ((xudc->gadget.speed == USB_SPEED_SUPER) &&
2011 	     (val & PORTPM_FRWE))) {
2012 		tegra_xudc_resume_device_state(xudc);
2013 
2014 		/* Send Device Notification packet. */
2015 		if (xudc->gadget.speed == USB_SPEED_SUPER) {
2016 			val = DEVNOTIF_LO_TYPE(DEVNOTIF_LO_TYPE_FUNCTION_WAKE)
2017 					     | DEVNOTIF_LO_TRIG;
2018 			xudc_writel(xudc, 0, DEVNOTIF_HI);
2019 			xudc_writel(xudc, val, DEVNOTIF_LO);
2020 		}
2021 	}
2022 
2023 unlock:
2024 	dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2025 	spin_unlock_irqrestore(&xudc->lock, flags);
2026 
2027 	return ret;
2028 }
2029 
2030 static int tegra_xudc_gadget_pullup(struct usb_gadget *gadget, int is_on)
2031 {
2032 	struct tegra_xudc *xudc = to_xudc(gadget);
2033 	unsigned long flags;
2034 	u32 val;
2035 
2036 	pm_runtime_get_sync(xudc->dev);
2037 
2038 	spin_lock_irqsave(&xudc->lock, flags);
2039 
2040 	if (is_on != xudc->pullup) {
2041 		val = xudc_readl(xudc, CTRL);
2042 		if (is_on)
2043 			val |= CTRL_ENABLE;
2044 		else
2045 			val &= ~CTRL_ENABLE;
2046 		xudc_writel(xudc, val, CTRL);
2047 	}
2048 
2049 	xudc->pullup = is_on;
2050 	dev_dbg(xudc->dev, "%s: pullup:%d", __func__, is_on);
2051 
2052 	spin_unlock_irqrestore(&xudc->lock, flags);
2053 
2054 	pm_runtime_put(xudc->dev);
2055 
2056 	return 0;
2057 }
2058 
2059 static int tegra_xudc_gadget_start(struct usb_gadget *gadget,
2060 				   struct usb_gadget_driver *driver)
2061 {
2062 	struct tegra_xudc *xudc = to_xudc(gadget);
2063 	unsigned long flags;
2064 	u32 val;
2065 	int ret;
2066 	unsigned int i;
2067 
2068 	if (!driver)
2069 		return -EINVAL;
2070 
2071 	pm_runtime_get_sync(xudc->dev);
2072 
2073 	spin_lock_irqsave(&xudc->lock, flags);
2074 
2075 	if (xudc->driver) {
2076 		ret = -EBUSY;
2077 		goto unlock;
2078 	}
2079 
2080 	xudc->setup_state = WAIT_FOR_SETUP;
2081 	xudc->device_state = USB_STATE_DEFAULT;
2082 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2083 
2084 	ret = __tegra_xudc_ep_enable(&xudc->ep[0], &tegra_xudc_ep0_desc);
2085 	if (ret < 0)
2086 		goto unlock;
2087 
2088 	val = xudc_readl(xudc, CTRL);
2089 	val |= CTRL_IE | CTRL_LSE;
2090 	xudc_writel(xudc, val, CTRL);
2091 
2092 	val = xudc_readl(xudc, PORTHALT);
2093 	val |= PORTHALT_STCHG_INTR_EN;
2094 	xudc_writel(xudc, val, PORTHALT);
2095 
2096 	if (xudc->pullup) {
2097 		val = xudc_readl(xudc, CTRL);
2098 		val |= CTRL_ENABLE;
2099 		xudc_writel(xudc, val, CTRL);
2100 	}
2101 
2102 	for (i = 0; i < xudc->soc->num_phys; i++)
2103 		if (xudc->usbphy[i])
2104 			otg_set_peripheral(xudc->usbphy[i]->otg, gadget);
2105 
2106 	xudc->driver = driver;
2107 unlock:
2108 	dev_dbg(xudc->dev, "%s: ret value is %d", __func__, ret);
2109 	spin_unlock_irqrestore(&xudc->lock, flags);
2110 
2111 	pm_runtime_put(xudc->dev);
2112 
2113 	return ret;
2114 }
2115 
2116 static int tegra_xudc_gadget_stop(struct usb_gadget *gadget)
2117 {
2118 	struct tegra_xudc *xudc = to_xudc(gadget);
2119 	unsigned long flags;
2120 	u32 val;
2121 	unsigned int i;
2122 
2123 	pm_runtime_get_sync(xudc->dev);
2124 
2125 	spin_lock_irqsave(&xudc->lock, flags);
2126 
2127 	for (i = 0; i < xudc->soc->num_phys; i++)
2128 		if (xudc->usbphy[i])
2129 			otg_set_peripheral(xudc->usbphy[i]->otg, NULL);
2130 
2131 	val = xudc_readl(xudc, CTRL);
2132 	val &= ~(CTRL_IE | CTRL_ENABLE);
2133 	xudc_writel(xudc, val, CTRL);
2134 
2135 	__tegra_xudc_ep_disable(&xudc->ep[0]);
2136 
2137 	xudc->driver = NULL;
2138 	dev_dbg(xudc->dev, "Gadget stopped");
2139 
2140 	spin_unlock_irqrestore(&xudc->lock, flags);
2141 
2142 	pm_runtime_put(xudc->dev);
2143 
2144 	return 0;
2145 }
2146 
2147 static int tegra_xudc_gadget_vbus_draw(struct usb_gadget *gadget,
2148 						unsigned int m_a)
2149 {
2150 	int ret = 0;
2151 	struct tegra_xudc *xudc = to_xudc(gadget);
2152 
2153 	dev_dbg(xudc->dev, "%s: %u mA\n", __func__, m_a);
2154 
2155 	if (xudc->curr_usbphy->chg_type == SDP_TYPE)
2156 		ret = usb_phy_set_power(xudc->curr_usbphy, m_a);
2157 
2158 	return ret;
2159 }
2160 
2161 static int tegra_xudc_set_selfpowered(struct usb_gadget *gadget, int is_on)
2162 {
2163 	struct tegra_xudc *xudc = to_xudc(gadget);
2164 
2165 	dev_dbg(xudc->dev, "%s: %d\n", __func__, is_on);
2166 	xudc->selfpowered = !!is_on;
2167 
2168 	return 0;
2169 }
2170 
2171 static struct usb_gadget_ops tegra_xudc_gadget_ops = {
2172 	.get_frame = tegra_xudc_gadget_get_frame,
2173 	.wakeup = tegra_xudc_gadget_wakeup,
2174 	.pullup = tegra_xudc_gadget_pullup,
2175 	.udc_start = tegra_xudc_gadget_start,
2176 	.udc_stop = tegra_xudc_gadget_stop,
2177 	.vbus_draw = tegra_xudc_gadget_vbus_draw,
2178 	.set_selfpowered = tegra_xudc_set_selfpowered,
2179 };
2180 
2181 static void no_op_complete(struct usb_ep *ep, struct usb_request *req)
2182 {
2183 }
2184 
2185 static int
2186 tegra_xudc_ep0_queue_status(struct tegra_xudc *xudc,
2187 		void (*cmpl)(struct usb_ep *, struct usb_request *))
2188 {
2189 	xudc->ep0_req->usb_req.buf = NULL;
2190 	xudc->ep0_req->usb_req.dma = 0;
2191 	xudc->ep0_req->usb_req.length = 0;
2192 	xudc->ep0_req->usb_req.complete = cmpl;
2193 	xudc->ep0_req->usb_req.context = xudc;
2194 
2195 	return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2196 }
2197 
2198 static int
2199 tegra_xudc_ep0_queue_data(struct tegra_xudc *xudc, void *buf, size_t len,
2200 		void (*cmpl)(struct usb_ep *, struct usb_request *))
2201 {
2202 	xudc->ep0_req->usb_req.buf = buf;
2203 	xudc->ep0_req->usb_req.length = len;
2204 	xudc->ep0_req->usb_req.complete = cmpl;
2205 	xudc->ep0_req->usb_req.context = xudc;
2206 
2207 	return __tegra_xudc_ep_queue(&xudc->ep[0], xudc->ep0_req);
2208 }
2209 
2210 static void tegra_xudc_ep0_req_done(struct tegra_xudc *xudc)
2211 {
2212 	switch (xudc->setup_state) {
2213 	case DATA_STAGE_XFER:
2214 		xudc->setup_state = STATUS_STAGE_RECV;
2215 		tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2216 		break;
2217 	case DATA_STAGE_RECV:
2218 		xudc->setup_state = STATUS_STAGE_XFER;
2219 		tegra_xudc_ep0_queue_status(xudc, no_op_complete);
2220 		break;
2221 	default:
2222 		xudc->setup_state = WAIT_FOR_SETUP;
2223 		break;
2224 	}
2225 }
2226 
2227 static int tegra_xudc_ep0_delegate_req(struct tegra_xudc *xudc,
2228 				       struct usb_ctrlrequest *ctrl)
2229 {
2230 	int ret;
2231 
2232 	spin_unlock(&xudc->lock);
2233 	ret = xudc->driver->setup(&xudc->gadget, ctrl);
2234 	spin_lock(&xudc->lock);
2235 
2236 	return ret;
2237 }
2238 
2239 static void set_feature_complete(struct usb_ep *ep, struct usb_request *req)
2240 {
2241 	struct tegra_xudc *xudc = req->context;
2242 
2243 	if (xudc->test_mode_pattern) {
2244 		xudc_writel(xudc, xudc->test_mode_pattern, PORT_TM);
2245 		xudc->test_mode_pattern = 0;
2246 	}
2247 }
2248 
2249 static int tegra_xudc_ep0_set_feature(struct tegra_xudc *xudc,
2250 				      struct usb_ctrlrequest *ctrl)
2251 {
2252 	bool set = (ctrl->bRequest == USB_REQ_SET_FEATURE);
2253 	u32 feature = le16_to_cpu(ctrl->wValue);
2254 	u32 index = le16_to_cpu(ctrl->wIndex);
2255 	u32 val, ep;
2256 	int ret;
2257 
2258 	if (le16_to_cpu(ctrl->wLength) != 0)
2259 		return -EINVAL;
2260 
2261 	switch (ctrl->bRequestType & USB_RECIP_MASK) {
2262 	case USB_RECIP_DEVICE:
2263 		switch (feature) {
2264 		case USB_DEVICE_REMOTE_WAKEUP:
2265 			if ((xudc->gadget.speed == USB_SPEED_SUPER) ||
2266 			    (xudc->device_state == USB_STATE_DEFAULT))
2267 				return -EINVAL;
2268 
2269 			val = xudc_readl(xudc, PORTPM);
2270 			if (set)
2271 				val |= PORTPM_RWE;
2272 			else
2273 				val &= ~PORTPM_RWE;
2274 
2275 			xudc_writel(xudc, val, PORTPM);
2276 			break;
2277 		case USB_DEVICE_U1_ENABLE:
2278 		case USB_DEVICE_U2_ENABLE:
2279 			if ((xudc->device_state != USB_STATE_CONFIGURED) ||
2280 			    (xudc->gadget.speed != USB_SPEED_SUPER))
2281 				return -EINVAL;
2282 
2283 			val = xudc_readl(xudc, PORTPM);
2284 			if ((feature == USB_DEVICE_U1_ENABLE) &&
2285 			     xudc->soc->u1_enable) {
2286 				if (set)
2287 					val |= PORTPM_U1E;
2288 				else
2289 					val &= ~PORTPM_U1E;
2290 			}
2291 
2292 			if ((feature == USB_DEVICE_U2_ENABLE) &&
2293 			     xudc->soc->u2_enable) {
2294 				if (set)
2295 					val |= PORTPM_U2E;
2296 				else
2297 					val &= ~PORTPM_U2E;
2298 			}
2299 
2300 			xudc_writel(xudc, val, PORTPM);
2301 			break;
2302 		case USB_DEVICE_TEST_MODE:
2303 			if (xudc->gadget.speed != USB_SPEED_HIGH)
2304 				return -EINVAL;
2305 
2306 			if (!set)
2307 				return -EINVAL;
2308 
2309 			xudc->test_mode_pattern = index >> 8;
2310 			break;
2311 		default:
2312 			return -EINVAL;
2313 		}
2314 
2315 		break;
2316 	case USB_RECIP_INTERFACE:
2317 		if (xudc->device_state != USB_STATE_CONFIGURED)
2318 			return -EINVAL;
2319 
2320 		switch (feature) {
2321 		case USB_INTRF_FUNC_SUSPEND:
2322 			if (set) {
2323 				val = xudc_readl(xudc, PORTPM);
2324 
2325 				if (index & USB_INTRF_FUNC_SUSPEND_RW)
2326 					val |= PORTPM_FRWE;
2327 				else
2328 					val &= ~PORTPM_FRWE;
2329 
2330 				xudc_writel(xudc, val, PORTPM);
2331 			}
2332 
2333 			return tegra_xudc_ep0_delegate_req(xudc, ctrl);
2334 		default:
2335 			return -EINVAL;
2336 		}
2337 
2338 		break;
2339 	case USB_RECIP_ENDPOINT:
2340 		ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2341 			((index & USB_DIR_IN) ? 1 : 0);
2342 
2343 		if ((xudc->device_state == USB_STATE_DEFAULT) ||
2344 		    ((xudc->device_state == USB_STATE_ADDRESS) &&
2345 		     (index != 0)))
2346 			return -EINVAL;
2347 
2348 		ret = __tegra_xudc_ep_set_halt(&xudc->ep[ep], set);
2349 		if (ret < 0)
2350 			return ret;
2351 		break;
2352 	default:
2353 		return -EINVAL;
2354 	}
2355 
2356 	return tegra_xudc_ep0_queue_status(xudc, set_feature_complete);
2357 }
2358 
2359 static int tegra_xudc_ep0_get_status(struct tegra_xudc *xudc,
2360 				     struct usb_ctrlrequest *ctrl)
2361 {
2362 	struct tegra_xudc_ep_context *ep_ctx;
2363 	u32 val, ep, index = le16_to_cpu(ctrl->wIndex);
2364 	u16 status = 0;
2365 
2366 	if (!(ctrl->bRequestType & USB_DIR_IN))
2367 		return -EINVAL;
2368 
2369 	if ((le16_to_cpu(ctrl->wValue) != 0) ||
2370 	    (le16_to_cpu(ctrl->wLength) != 2))
2371 		return -EINVAL;
2372 
2373 	switch (ctrl->bRequestType & USB_RECIP_MASK) {
2374 	case USB_RECIP_DEVICE:
2375 		val = xudc_readl(xudc, PORTPM);
2376 
2377 		if (xudc->selfpowered)
2378 			status |= BIT(USB_DEVICE_SELF_POWERED);
2379 
2380 		if ((xudc->gadget.speed < USB_SPEED_SUPER) &&
2381 		    (val & PORTPM_RWE))
2382 			status |= BIT(USB_DEVICE_REMOTE_WAKEUP);
2383 
2384 		if (xudc->gadget.speed == USB_SPEED_SUPER) {
2385 			if (val & PORTPM_U1E)
2386 				status |= BIT(USB_DEV_STAT_U1_ENABLED);
2387 			if (val & PORTPM_U2E)
2388 				status |= BIT(USB_DEV_STAT_U2_ENABLED);
2389 		}
2390 		break;
2391 	case USB_RECIP_INTERFACE:
2392 		if (xudc->gadget.speed == USB_SPEED_SUPER) {
2393 			status |= USB_INTRF_STAT_FUNC_RW_CAP;
2394 			val = xudc_readl(xudc, PORTPM);
2395 			if (val & PORTPM_FRWE)
2396 				status |= USB_INTRF_STAT_FUNC_RW;
2397 		}
2398 		break;
2399 	case USB_RECIP_ENDPOINT:
2400 		ep = (index & USB_ENDPOINT_NUMBER_MASK) * 2 +
2401 			((index & USB_DIR_IN) ? 1 : 0);
2402 		ep_ctx = &xudc->ep_context[ep];
2403 
2404 		if ((xudc->device_state != USB_STATE_CONFIGURED) &&
2405 		    ((xudc->device_state != USB_STATE_ADDRESS) || (ep != 0)))
2406 			return -EINVAL;
2407 
2408 		if (ep_ctx_read_state(ep_ctx) == EP_STATE_DISABLED)
2409 			return -EINVAL;
2410 
2411 		if (xudc_readl(xudc, EP_HALT) & BIT(ep))
2412 			status |= BIT(USB_ENDPOINT_HALT);
2413 		break;
2414 	default:
2415 		return -EINVAL;
2416 	}
2417 
2418 	xudc->status_buf = cpu_to_le16(status);
2419 	return tegra_xudc_ep0_queue_data(xudc, &xudc->status_buf,
2420 					 sizeof(xudc->status_buf),
2421 					 no_op_complete);
2422 }
2423 
2424 static void set_sel_complete(struct usb_ep *ep, struct usb_request *req)
2425 {
2426 	/* Nothing to do with SEL values */
2427 }
2428 
2429 static int tegra_xudc_ep0_set_sel(struct tegra_xudc *xudc,
2430 				  struct usb_ctrlrequest *ctrl)
2431 {
2432 	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2433 				     USB_TYPE_STANDARD))
2434 		return -EINVAL;
2435 
2436 	if (xudc->device_state == USB_STATE_DEFAULT)
2437 		return -EINVAL;
2438 
2439 	if ((le16_to_cpu(ctrl->wIndex) != 0) ||
2440 	    (le16_to_cpu(ctrl->wValue) != 0) ||
2441 	    (le16_to_cpu(ctrl->wLength) != 6))
2442 		return -EINVAL;
2443 
2444 	return tegra_xudc_ep0_queue_data(xudc, &xudc->sel_timing,
2445 					 sizeof(xudc->sel_timing),
2446 					 set_sel_complete);
2447 }
2448 
2449 static void set_isoch_delay_complete(struct usb_ep *ep, struct usb_request *req)
2450 {
2451 	/* Nothing to do with isoch delay */
2452 }
2453 
2454 static int tegra_xudc_ep0_set_isoch_delay(struct tegra_xudc *xudc,
2455 					  struct usb_ctrlrequest *ctrl)
2456 {
2457 	u32 delay = le16_to_cpu(ctrl->wValue);
2458 
2459 	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2460 				   USB_TYPE_STANDARD))
2461 		return -EINVAL;
2462 
2463 	if ((delay > 65535) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2464 	    (le16_to_cpu(ctrl->wLength) != 0))
2465 		return -EINVAL;
2466 
2467 	xudc->isoch_delay = delay;
2468 
2469 	return tegra_xudc_ep0_queue_status(xudc, set_isoch_delay_complete);
2470 }
2471 
2472 static void set_address_complete(struct usb_ep *ep, struct usb_request *req)
2473 {
2474 	struct tegra_xudc *xudc = req->context;
2475 
2476 	if ((xudc->device_state == USB_STATE_DEFAULT) &&
2477 	    (xudc->dev_addr != 0)) {
2478 		xudc->device_state = USB_STATE_ADDRESS;
2479 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2480 	} else if ((xudc->device_state == USB_STATE_ADDRESS) &&
2481 		   (xudc->dev_addr == 0)) {
2482 		xudc->device_state = USB_STATE_DEFAULT;
2483 		usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2484 	}
2485 }
2486 
2487 static int tegra_xudc_ep0_set_address(struct tegra_xudc *xudc,
2488 				      struct usb_ctrlrequest *ctrl)
2489 {
2490 	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2491 	u32 val, addr = le16_to_cpu(ctrl->wValue);
2492 
2493 	if (ctrl->bRequestType != (USB_DIR_OUT | USB_RECIP_DEVICE |
2494 				     USB_TYPE_STANDARD))
2495 		return -EINVAL;
2496 
2497 	if ((addr > 127) || (le16_to_cpu(ctrl->wIndex) != 0) ||
2498 	    (le16_to_cpu(ctrl->wLength) != 0))
2499 		return -EINVAL;
2500 
2501 	if (xudc->device_state == USB_STATE_CONFIGURED)
2502 		return -EINVAL;
2503 
2504 	dev_dbg(xudc->dev, "set address: %u\n", addr);
2505 
2506 	xudc->dev_addr = addr;
2507 	val = xudc_readl(xudc, CTRL);
2508 	val &= ~(CTRL_DEVADDR_MASK);
2509 	val |= CTRL_DEVADDR(addr);
2510 	xudc_writel(xudc, val, CTRL);
2511 
2512 	ep_ctx_write_devaddr(ep0->context, addr);
2513 
2514 	return tegra_xudc_ep0_queue_status(xudc, set_address_complete);
2515 }
2516 
2517 static int tegra_xudc_ep0_standard_req(struct tegra_xudc *xudc,
2518 				      struct usb_ctrlrequest *ctrl)
2519 {
2520 	int ret;
2521 
2522 	switch (ctrl->bRequest) {
2523 	case USB_REQ_GET_STATUS:
2524 		dev_dbg(xudc->dev, "USB_REQ_GET_STATUS\n");
2525 		ret = tegra_xudc_ep0_get_status(xudc, ctrl);
2526 		break;
2527 	case USB_REQ_SET_ADDRESS:
2528 		dev_dbg(xudc->dev, "USB_REQ_SET_ADDRESS\n");
2529 		ret = tegra_xudc_ep0_set_address(xudc, ctrl);
2530 		break;
2531 	case USB_REQ_SET_SEL:
2532 		dev_dbg(xudc->dev, "USB_REQ_SET_SEL\n");
2533 		ret = tegra_xudc_ep0_set_sel(xudc, ctrl);
2534 		break;
2535 	case USB_REQ_SET_ISOCH_DELAY:
2536 		dev_dbg(xudc->dev, "USB_REQ_SET_ISOCH_DELAY\n");
2537 		ret = tegra_xudc_ep0_set_isoch_delay(xudc, ctrl);
2538 		break;
2539 	case USB_REQ_CLEAR_FEATURE:
2540 	case USB_REQ_SET_FEATURE:
2541 		dev_dbg(xudc->dev, "USB_REQ_CLEAR/SET_FEATURE\n");
2542 		ret = tegra_xudc_ep0_set_feature(xudc, ctrl);
2543 		break;
2544 	case USB_REQ_SET_CONFIGURATION:
2545 		dev_dbg(xudc->dev, "USB_REQ_SET_CONFIGURATION\n");
2546 		/*
2547 		 * In theory we need to clear RUN bit before status stage of
2548 		 * deconfig request sent, but this seems to be causing problems.
2549 		 * Clear RUN once all endpoints are disabled instead.
2550 		 */
2551 		fallthrough;
2552 	default:
2553 		ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2554 		break;
2555 	}
2556 
2557 	return ret;
2558 }
2559 
2560 static void tegra_xudc_handle_ep0_setup_packet(struct tegra_xudc *xudc,
2561 					       struct usb_ctrlrequest *ctrl,
2562 					       u16 seq_num)
2563 {
2564 	int ret;
2565 
2566 	xudc->setup_seq_num = seq_num;
2567 
2568 	/* Ensure EP0 is unhalted. */
2569 	ep_unhalt(xudc, 0);
2570 
2571 	/*
2572 	 * On Tegra210, setup packets with sequence numbers 0xfffe or 0xffff
2573 	 * are invalid.  Halt EP0 until we get a valid packet.
2574 	 */
2575 	if (xudc->soc->invalid_seq_num &&
2576 	    (seq_num == 0xfffe || seq_num == 0xffff)) {
2577 		dev_warn(xudc->dev, "invalid sequence number detected\n");
2578 		ep_halt(xudc, 0);
2579 		return;
2580 	}
2581 
2582 	if (ctrl->wLength)
2583 		xudc->setup_state = (ctrl->bRequestType & USB_DIR_IN) ?
2584 			DATA_STAGE_XFER :  DATA_STAGE_RECV;
2585 	else
2586 		xudc->setup_state = STATUS_STAGE_XFER;
2587 
2588 	if ((ctrl->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD)
2589 		ret = tegra_xudc_ep0_standard_req(xudc, ctrl);
2590 	else
2591 		ret = tegra_xudc_ep0_delegate_req(xudc, ctrl);
2592 
2593 	if (ret < 0) {
2594 		dev_warn(xudc->dev, "setup request failed: %d\n", ret);
2595 		xudc->setup_state = WAIT_FOR_SETUP;
2596 		ep_halt(xudc, 0);
2597 	}
2598 }
2599 
2600 static void tegra_xudc_handle_ep0_event(struct tegra_xudc *xudc,
2601 					struct tegra_xudc_trb *event)
2602 {
2603 	struct usb_ctrlrequest *ctrl = (struct usb_ctrlrequest *)event;
2604 	u16 seq_num = trb_read_seq_num(event);
2605 
2606 	if (xudc->setup_state != WAIT_FOR_SETUP) {
2607 		/*
2608 		 * The controller is in the process of handling another
2609 		 * setup request.  Queue subsequent requests and handle
2610 		 * the last one once the controller reports a sequence
2611 		 * number error.
2612 		 */
2613 		memcpy(&xudc->setup_packet.ctrl_req, ctrl, sizeof(*ctrl));
2614 		xudc->setup_packet.seq_num = seq_num;
2615 		xudc->queued_setup_packet = true;
2616 	} else {
2617 		tegra_xudc_handle_ep0_setup_packet(xudc, ctrl, seq_num);
2618 	}
2619 }
2620 
2621 static struct tegra_xudc_request *
2622 trb_to_request(struct tegra_xudc_ep *ep, struct tegra_xudc_trb *trb)
2623 {
2624 	struct tegra_xudc_request *req;
2625 
2626 	list_for_each_entry(req, &ep->queue, list) {
2627 		if (!req->trbs_queued)
2628 			break;
2629 
2630 		if (trb_in_request(ep, req, trb))
2631 			return req;
2632 	}
2633 
2634 	return NULL;
2635 }
2636 
2637 static void tegra_xudc_handle_transfer_completion(struct tegra_xudc *xudc,
2638 						  struct tegra_xudc_ep *ep,
2639 						  struct tegra_xudc_trb *event)
2640 {
2641 	struct tegra_xudc_request *req;
2642 	struct tegra_xudc_trb *trb;
2643 	bool short_packet;
2644 
2645 	short_packet = (trb_read_cmpl_code(event) ==
2646 			TRB_CMPL_CODE_SHORT_PACKET);
2647 
2648 	trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2649 	req = trb_to_request(ep, trb);
2650 
2651 	/*
2652 	 * TDs are complete on short packet or when the completed TRB is the
2653 	 * last TRB in the TD (the CHAIN bit is unset).
2654 	 */
2655 	if (req && (short_packet || (!trb_read_chain(trb) &&
2656 		(req->trbs_needed == req->trbs_queued)))) {
2657 		struct tegra_xudc_trb *last = req->last_trb;
2658 		unsigned int residual;
2659 
2660 		residual = trb_read_transfer_len(event);
2661 		req->usb_req.actual = req->usb_req.length - residual;
2662 
2663 		dev_dbg(xudc->dev, "bytes transferred %u / %u\n",
2664 			req->usb_req.actual, req->usb_req.length);
2665 
2666 		tegra_xudc_req_done(ep, req, 0);
2667 
2668 		if (ep->desc && usb_endpoint_xfer_control(ep->desc))
2669 			tegra_xudc_ep0_req_done(xudc);
2670 
2671 		/*
2672 		 * Advance the dequeue pointer past the end of the current TD
2673 		 * on short packet completion.
2674 		 */
2675 		if (short_packet) {
2676 			ep->deq_ptr = (last - ep->transfer_ring) + 1;
2677 			if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2678 				ep->deq_ptr = 0;
2679 		}
2680 	} else if (!req) {
2681 		dev_warn(xudc->dev, "transfer event on dequeued request\n");
2682 	}
2683 
2684 	if (ep->desc)
2685 		tegra_xudc_ep_kick_queue(ep);
2686 }
2687 
2688 static void tegra_xudc_handle_transfer_event(struct tegra_xudc *xudc,
2689 					     struct tegra_xudc_trb *event)
2690 {
2691 	unsigned int ep_index = trb_read_endpoint_id(event);
2692 	struct tegra_xudc_ep *ep = &xudc->ep[ep_index];
2693 	struct tegra_xudc_trb *trb;
2694 	u16 comp_code;
2695 
2696 	if (ep_ctx_read_state(ep->context) == EP_STATE_DISABLED) {
2697 		dev_warn(xudc->dev, "transfer event on disabled EP %u\n",
2698 			 ep_index);
2699 		return;
2700 	}
2701 
2702 	/* Update transfer ring dequeue pointer. */
2703 	trb = trb_phys_to_virt(ep, trb_read_data_ptr(event));
2704 	comp_code = trb_read_cmpl_code(event);
2705 	if (comp_code != TRB_CMPL_CODE_BABBLE_DETECTED_ERR) {
2706 		ep->deq_ptr = (trb - ep->transfer_ring) + 1;
2707 
2708 		if (ep->deq_ptr == XUDC_TRANSFER_RING_SIZE - 1)
2709 			ep->deq_ptr = 0;
2710 		ep->ring_full = false;
2711 	}
2712 
2713 	switch (comp_code) {
2714 	case TRB_CMPL_CODE_SUCCESS:
2715 	case TRB_CMPL_CODE_SHORT_PACKET:
2716 		tegra_xudc_handle_transfer_completion(xudc, ep, event);
2717 		break;
2718 	case TRB_CMPL_CODE_HOST_REJECTED:
2719 		dev_info(xudc->dev, "stream rejected on EP %u\n", ep_index);
2720 
2721 		ep->stream_rejected = true;
2722 		break;
2723 	case TRB_CMPL_CODE_PRIME_PIPE_RECEIVED:
2724 		dev_info(xudc->dev, "prime pipe received on EP %u\n", ep_index);
2725 
2726 		if (ep->stream_rejected) {
2727 			ep->stream_rejected = false;
2728 			/*
2729 			 * An EP is stopped when a stream is rejected.  Wait
2730 			 * for the EP to report that it is stopped and then
2731 			 * un-stop it.
2732 			 */
2733 			ep_wait_for_stopped(xudc, ep_index);
2734 		}
2735 		tegra_xudc_ep_ring_doorbell(ep);
2736 		break;
2737 	case TRB_CMPL_CODE_BABBLE_DETECTED_ERR:
2738 		/*
2739 		 * Wait for the EP to be stopped so the controller stops
2740 		 * processing doorbells.
2741 		 */
2742 		ep_wait_for_stopped(xudc, ep_index);
2743 		ep->enq_ptr = ep->deq_ptr;
2744 		tegra_xudc_ep_nuke(ep, -EIO);
2745 		fallthrough;
2746 	case TRB_CMPL_CODE_STREAM_NUMP_ERROR:
2747 	case TRB_CMPL_CODE_CTRL_DIR_ERR:
2748 	case TRB_CMPL_CODE_INVALID_STREAM_TYPE_ERR:
2749 	case TRB_CMPL_CODE_RING_UNDERRUN:
2750 	case TRB_CMPL_CODE_RING_OVERRUN:
2751 	case TRB_CMPL_CODE_ISOCH_BUFFER_OVERRUN:
2752 	case TRB_CMPL_CODE_USB_TRANS_ERR:
2753 	case TRB_CMPL_CODE_TRB_ERR:
2754 		dev_err(xudc->dev, "completion error %#x on EP %u\n",
2755 			comp_code, ep_index);
2756 
2757 		ep_halt(xudc, ep_index);
2758 		break;
2759 	case TRB_CMPL_CODE_CTRL_SEQNUM_ERR:
2760 		dev_info(xudc->dev, "sequence number error\n");
2761 
2762 		/*
2763 		 * Kill any queued control request and skip to the last
2764 		 * setup packet we received.
2765 		 */
2766 		tegra_xudc_ep_nuke(ep, -EINVAL);
2767 		xudc->setup_state = WAIT_FOR_SETUP;
2768 		if (!xudc->queued_setup_packet)
2769 			break;
2770 
2771 		tegra_xudc_handle_ep0_setup_packet(xudc,
2772 						   &xudc->setup_packet.ctrl_req,
2773 						   xudc->setup_packet.seq_num);
2774 		xudc->queued_setup_packet = false;
2775 		break;
2776 	case TRB_CMPL_CODE_STOPPED:
2777 		dev_dbg(xudc->dev, "stop completion code on EP %u\n",
2778 			ep_index);
2779 
2780 		/* Disconnected. */
2781 		tegra_xudc_ep_nuke(ep, -ECONNREFUSED);
2782 		break;
2783 	default:
2784 		dev_dbg(xudc->dev, "completion event %#x on EP %u\n",
2785 			comp_code, ep_index);
2786 		break;
2787 	}
2788 }
2789 
2790 static void tegra_xudc_reset(struct tegra_xudc *xudc)
2791 {
2792 	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2793 	dma_addr_t deq_ptr;
2794 	unsigned int i;
2795 
2796 	xudc->setup_state = WAIT_FOR_SETUP;
2797 	xudc->device_state = USB_STATE_DEFAULT;
2798 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2799 
2800 	ep_unpause_all(xudc);
2801 
2802 	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
2803 		tegra_xudc_ep_nuke(&xudc->ep[i], -ESHUTDOWN);
2804 
2805 	/*
2806 	 * Reset sequence number and dequeue pointer to flush the transfer
2807 	 * ring.
2808 	 */
2809 	ep0->deq_ptr = ep0->enq_ptr;
2810 	ep0->ring_full = false;
2811 
2812 	xudc->setup_seq_num = 0;
2813 	xudc->queued_setup_packet = false;
2814 
2815 	ep_ctx_write_seq_num(ep0->context, xudc->setup_seq_num);
2816 
2817 	deq_ptr = trb_virt_to_phys(ep0, &ep0->transfer_ring[ep0->deq_ptr]);
2818 
2819 	if (!dma_mapping_error(xudc->dev, deq_ptr)) {
2820 		ep_ctx_write_deq_ptr(ep0->context, deq_ptr);
2821 		ep_ctx_write_dcs(ep0->context, ep0->pcs);
2822 	}
2823 
2824 	ep_unhalt_all(xudc);
2825 	ep_reload(xudc, 0);
2826 	ep_unpause(xudc, 0);
2827 }
2828 
2829 static void tegra_xudc_port_connect(struct tegra_xudc *xudc)
2830 {
2831 	struct tegra_xudc_ep *ep0 = &xudc->ep[0];
2832 	u16 maxpacket;
2833 	u32 val;
2834 
2835 	val = (xudc_readl(xudc, PORTSC) & PORTSC_PS_MASK) >> PORTSC_PS_SHIFT;
2836 	switch (val) {
2837 	case PORTSC_PS_LS:
2838 		xudc->gadget.speed = USB_SPEED_LOW;
2839 		break;
2840 	case PORTSC_PS_FS:
2841 		xudc->gadget.speed = USB_SPEED_FULL;
2842 		break;
2843 	case PORTSC_PS_HS:
2844 		xudc->gadget.speed = USB_SPEED_HIGH;
2845 		break;
2846 	case PORTSC_PS_SS:
2847 		xudc->gadget.speed = USB_SPEED_SUPER;
2848 		break;
2849 	default:
2850 		xudc->gadget.speed = USB_SPEED_UNKNOWN;
2851 		break;
2852 	}
2853 
2854 	xudc->device_state = USB_STATE_DEFAULT;
2855 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2856 
2857 	xudc->setup_state = WAIT_FOR_SETUP;
2858 
2859 	if (xudc->gadget.speed == USB_SPEED_SUPER)
2860 		maxpacket = 512;
2861 	else
2862 		maxpacket = 64;
2863 
2864 	ep_ctx_write_max_packet_size(ep0->context, maxpacket);
2865 	tegra_xudc_ep0_desc.wMaxPacketSize = cpu_to_le16(maxpacket);
2866 	usb_ep_set_maxpacket_limit(&ep0->usb_ep, maxpacket);
2867 
2868 	if (!xudc->soc->u1_enable) {
2869 		val = xudc_readl(xudc, PORTPM);
2870 		val &= ~(PORTPM_U1TIMEOUT_MASK);
2871 		xudc_writel(xudc, val, PORTPM);
2872 	}
2873 
2874 	if (!xudc->soc->u2_enable) {
2875 		val = xudc_readl(xudc, PORTPM);
2876 		val &= ~(PORTPM_U2TIMEOUT_MASK);
2877 		xudc_writel(xudc, val, PORTPM);
2878 	}
2879 
2880 	if (xudc->gadget.speed <= USB_SPEED_HIGH) {
2881 		val = xudc_readl(xudc, PORTPM);
2882 		val &= ~(PORTPM_L1S_MASK);
2883 		if (xudc->soc->lpm_enable)
2884 			val |= PORTPM_L1S(PORTPM_L1S_ACCEPT);
2885 		else
2886 			val |= PORTPM_L1S(PORTPM_L1S_NYET);
2887 		xudc_writel(xudc, val, PORTPM);
2888 	}
2889 
2890 	val = xudc_readl(xudc, ST);
2891 	if (val & ST_RC)
2892 		xudc_writel(xudc, ST_RC, ST);
2893 }
2894 
2895 static void tegra_xudc_port_disconnect(struct tegra_xudc *xudc)
2896 {
2897 	tegra_xudc_reset(xudc);
2898 
2899 	if (xudc->driver && xudc->driver->disconnect) {
2900 		spin_unlock(&xudc->lock);
2901 		xudc->driver->disconnect(&xudc->gadget);
2902 		spin_lock(&xudc->lock);
2903 	}
2904 
2905 	xudc->device_state = USB_STATE_NOTATTACHED;
2906 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2907 
2908 	complete(&xudc->disconnect_complete);
2909 }
2910 
2911 static void tegra_xudc_port_reset(struct tegra_xudc *xudc)
2912 {
2913 	tegra_xudc_reset(xudc);
2914 
2915 	if (xudc->driver) {
2916 		spin_unlock(&xudc->lock);
2917 		usb_gadget_udc_reset(&xudc->gadget, xudc->driver);
2918 		spin_lock(&xudc->lock);
2919 	}
2920 
2921 	tegra_xudc_port_connect(xudc);
2922 }
2923 
2924 static void tegra_xudc_port_suspend(struct tegra_xudc *xudc)
2925 {
2926 	dev_dbg(xudc->dev, "port suspend\n");
2927 
2928 	xudc->resume_state = xudc->device_state;
2929 	xudc->device_state = USB_STATE_SUSPENDED;
2930 	usb_gadget_set_state(&xudc->gadget, xudc->device_state);
2931 
2932 	if (xudc->driver->suspend) {
2933 		spin_unlock(&xudc->lock);
2934 		xudc->driver->suspend(&xudc->gadget);
2935 		spin_lock(&xudc->lock);
2936 	}
2937 }
2938 
2939 static void tegra_xudc_port_resume(struct tegra_xudc *xudc)
2940 {
2941 	dev_dbg(xudc->dev, "port resume\n");
2942 
2943 	tegra_xudc_resume_device_state(xudc);
2944 
2945 	if (xudc->driver->resume) {
2946 		spin_unlock(&xudc->lock);
2947 		xudc->driver->resume(&xudc->gadget);
2948 		spin_lock(&xudc->lock);
2949 	}
2950 }
2951 
2952 static inline void clear_port_change(struct tegra_xudc *xudc, u32 flag)
2953 {
2954 	u32 val;
2955 
2956 	val = xudc_readl(xudc, PORTSC);
2957 	val &= ~PORTSC_CHANGE_MASK;
2958 	val |= flag;
2959 	xudc_writel(xudc, val, PORTSC);
2960 }
2961 
2962 static void __tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
2963 {
2964 	u32 portsc, porthalt;
2965 
2966 	porthalt = xudc_readl(xudc, PORTHALT);
2967 	if ((porthalt & PORTHALT_STCHG_REQ) &&
2968 	    (porthalt & PORTHALT_HALT_LTSSM)) {
2969 		dev_dbg(xudc->dev, "STCHG_REQ, PORTHALT = %#x\n", porthalt);
2970 		porthalt &= ~PORTHALT_HALT_LTSSM;
2971 		xudc_writel(xudc, porthalt, PORTHALT);
2972 	}
2973 
2974 	portsc = xudc_readl(xudc, PORTSC);
2975 	if ((portsc & PORTSC_PRC) && (portsc & PORTSC_PR)) {
2976 		dev_dbg(xudc->dev, "PRC, PR, PORTSC = %#x\n", portsc);
2977 		clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
2978 #define TOGGLE_VBUS_WAIT_MS 100
2979 		if (xudc->soc->port_reset_quirk) {
2980 			schedule_delayed_work(&xudc->port_reset_war_work,
2981 				msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
2982 			xudc->wait_for_sec_prc = 1;
2983 		}
2984 	}
2985 
2986 	if ((portsc & PORTSC_PRC) && !(portsc & PORTSC_PR)) {
2987 		dev_dbg(xudc->dev, "PRC, Not PR, PORTSC = %#x\n", portsc);
2988 		clear_port_change(xudc, PORTSC_PRC | PORTSC_PED);
2989 		tegra_xudc_port_reset(xudc);
2990 		cancel_delayed_work(&xudc->port_reset_war_work);
2991 		xudc->wait_for_sec_prc = 0;
2992 	}
2993 
2994 	portsc = xudc_readl(xudc, PORTSC);
2995 	if (portsc & PORTSC_WRC) {
2996 		dev_dbg(xudc->dev, "WRC, PORTSC = %#x\n", portsc);
2997 		clear_port_change(xudc, PORTSC_WRC | PORTSC_PED);
2998 		if (!(xudc_readl(xudc, PORTSC) & PORTSC_WPR))
2999 			tegra_xudc_port_reset(xudc);
3000 	}
3001 
3002 	portsc = xudc_readl(xudc, PORTSC);
3003 	if (portsc & PORTSC_CSC) {
3004 		dev_dbg(xudc->dev, "CSC, PORTSC = %#x\n", portsc);
3005 		clear_port_change(xudc, PORTSC_CSC);
3006 
3007 		if (portsc & PORTSC_CCS)
3008 			tegra_xudc_port_connect(xudc);
3009 		else
3010 			tegra_xudc_port_disconnect(xudc);
3011 
3012 		if (xudc->wait_csc) {
3013 			cancel_delayed_work(&xudc->plc_reset_work);
3014 			xudc->wait_csc = false;
3015 		}
3016 	}
3017 
3018 	portsc = xudc_readl(xudc, PORTSC);
3019 	if (portsc & PORTSC_PLC) {
3020 		u32 pls = (portsc & PORTSC_PLS_MASK) >> PORTSC_PLS_SHIFT;
3021 
3022 		dev_dbg(xudc->dev, "PLC, PORTSC = %#x\n", portsc);
3023 		clear_port_change(xudc, PORTSC_PLC);
3024 		switch (pls) {
3025 		case PORTSC_PLS_U3:
3026 			tegra_xudc_port_suspend(xudc);
3027 			break;
3028 		case PORTSC_PLS_U0:
3029 			if (xudc->gadget.speed < USB_SPEED_SUPER)
3030 				tegra_xudc_port_resume(xudc);
3031 			break;
3032 		case PORTSC_PLS_RESUME:
3033 			if (xudc->gadget.speed == USB_SPEED_SUPER)
3034 				tegra_xudc_port_resume(xudc);
3035 			break;
3036 		case PORTSC_PLS_INACTIVE:
3037 			schedule_delayed_work(&xudc->plc_reset_work,
3038 					msecs_to_jiffies(TOGGLE_VBUS_WAIT_MS));
3039 			xudc->wait_csc = true;
3040 			break;
3041 		default:
3042 			break;
3043 		}
3044 	}
3045 
3046 	if (portsc & PORTSC_CEC) {
3047 		dev_warn(xudc->dev, "CEC, PORTSC = %#x\n", portsc);
3048 		clear_port_change(xudc, PORTSC_CEC);
3049 	}
3050 
3051 	dev_dbg(xudc->dev, "PORTSC = %#x\n", xudc_readl(xudc, PORTSC));
3052 }
3053 
3054 static void tegra_xudc_handle_port_status(struct tegra_xudc *xudc)
3055 {
3056 	while ((xudc_readl(xudc, PORTSC) & PORTSC_CHANGE_MASK) ||
3057 	       (xudc_readl(xudc, PORTHALT) & PORTHALT_STCHG_REQ))
3058 		__tegra_xudc_handle_port_status(xudc);
3059 }
3060 
3061 static void tegra_xudc_handle_event(struct tegra_xudc *xudc,
3062 				    struct tegra_xudc_trb *event)
3063 {
3064 	u32 type = trb_read_type(event);
3065 
3066 	dump_trb(xudc, "EVENT", event);
3067 
3068 	switch (type) {
3069 	case TRB_TYPE_PORT_STATUS_CHANGE_EVENT:
3070 		tegra_xudc_handle_port_status(xudc);
3071 		break;
3072 	case TRB_TYPE_TRANSFER_EVENT:
3073 		tegra_xudc_handle_transfer_event(xudc, event);
3074 		break;
3075 	case TRB_TYPE_SETUP_PACKET_EVENT:
3076 		tegra_xudc_handle_ep0_event(xudc, event);
3077 		break;
3078 	default:
3079 		dev_info(xudc->dev, "Unrecognized TRB type = %#x\n", type);
3080 		break;
3081 	}
3082 }
3083 
3084 static void tegra_xudc_process_event_ring(struct tegra_xudc *xudc)
3085 {
3086 	struct tegra_xudc_trb *event;
3087 	dma_addr_t erdp;
3088 
3089 	while (true) {
3090 		event = xudc->event_ring[xudc->event_ring_index] +
3091 			xudc->event_ring_deq_ptr;
3092 
3093 		if (trb_read_cycle(event) != xudc->ccs)
3094 			break;
3095 
3096 		tegra_xudc_handle_event(xudc, event);
3097 
3098 		xudc->event_ring_deq_ptr++;
3099 		if (xudc->event_ring_deq_ptr == XUDC_EVENT_RING_SIZE) {
3100 			xudc->event_ring_deq_ptr = 0;
3101 			xudc->event_ring_index++;
3102 		}
3103 
3104 		if (xudc->event_ring_index == XUDC_NR_EVENT_RINGS) {
3105 			xudc->event_ring_index = 0;
3106 			xudc->ccs = !xudc->ccs;
3107 		}
3108 	}
3109 
3110 	erdp = xudc->event_ring_phys[xudc->event_ring_index] +
3111 		xudc->event_ring_deq_ptr * sizeof(*event);
3112 
3113 	xudc_writel(xudc, upper_32_bits(erdp), ERDPHI);
3114 	xudc_writel(xudc, lower_32_bits(erdp) | ERDPLO_EHB, ERDPLO);
3115 }
3116 
3117 static irqreturn_t tegra_xudc_irq(int irq, void *data)
3118 {
3119 	struct tegra_xudc *xudc = data;
3120 	unsigned long flags;
3121 	u32 val;
3122 
3123 	val = xudc_readl(xudc, ST);
3124 	if (!(val & ST_IP))
3125 		return IRQ_NONE;
3126 	xudc_writel(xudc, ST_IP, ST);
3127 
3128 	spin_lock_irqsave(&xudc->lock, flags);
3129 	tegra_xudc_process_event_ring(xudc);
3130 	spin_unlock_irqrestore(&xudc->lock, flags);
3131 
3132 	return IRQ_HANDLED;
3133 }
3134 
3135 static int tegra_xudc_alloc_ep(struct tegra_xudc *xudc, unsigned int index)
3136 {
3137 	struct tegra_xudc_ep *ep = &xudc->ep[index];
3138 
3139 	ep->xudc = xudc;
3140 	ep->index = index;
3141 	ep->context = &xudc->ep_context[index];
3142 	INIT_LIST_HEAD(&ep->queue);
3143 
3144 	/*
3145 	 * EP1 would be the input endpoint corresponding to EP0, but since
3146 	 * EP0 is bi-directional, EP1 is unused.
3147 	 */
3148 	if (index == 1)
3149 		return 0;
3150 
3151 	ep->transfer_ring = dma_pool_alloc(xudc->transfer_ring_pool,
3152 					   GFP_KERNEL,
3153 					   &ep->transfer_ring_phys);
3154 	if (!ep->transfer_ring)
3155 		return -ENOMEM;
3156 
3157 	if (index) {
3158 		snprintf(ep->name, sizeof(ep->name), "ep%u%s", index / 2,
3159 			 (index % 2 == 0) ? "out" : "in");
3160 		ep->usb_ep.name = ep->name;
3161 		usb_ep_set_maxpacket_limit(&ep->usb_ep, 1024);
3162 		ep->usb_ep.max_streams = 16;
3163 		ep->usb_ep.ops = &tegra_xudc_ep_ops;
3164 		ep->usb_ep.caps.type_bulk = true;
3165 		ep->usb_ep.caps.type_int = true;
3166 		if (index & 1)
3167 			ep->usb_ep.caps.dir_in = true;
3168 		else
3169 			ep->usb_ep.caps.dir_out = true;
3170 		list_add_tail(&ep->usb_ep.ep_list, &xudc->gadget.ep_list);
3171 	} else {
3172 		strscpy(ep->name, "ep0", 3);
3173 		ep->usb_ep.name = ep->name;
3174 		usb_ep_set_maxpacket_limit(&ep->usb_ep, 512);
3175 		ep->usb_ep.ops = &tegra_xudc_ep0_ops;
3176 		ep->usb_ep.caps.type_control = true;
3177 		ep->usb_ep.caps.dir_in = true;
3178 		ep->usb_ep.caps.dir_out = true;
3179 	}
3180 
3181 	return 0;
3182 }
3183 
3184 static void tegra_xudc_free_ep(struct tegra_xudc *xudc, unsigned int index)
3185 {
3186 	struct tegra_xudc_ep *ep = &xudc->ep[index];
3187 
3188 	/*
3189 	 * EP1 would be the input endpoint corresponding to EP0, but since
3190 	 * EP0 is bi-directional, EP1 is unused.
3191 	 */
3192 	if (index == 1)
3193 		return;
3194 
3195 	dma_pool_free(xudc->transfer_ring_pool, ep->transfer_ring,
3196 		      ep->transfer_ring_phys);
3197 }
3198 
3199 static int tegra_xudc_alloc_eps(struct tegra_xudc *xudc)
3200 {
3201 	struct usb_request *req;
3202 	unsigned int i;
3203 	int err;
3204 
3205 	xudc->ep_context =
3206 		dma_alloc_coherent(xudc->dev, XUDC_NR_EPS *
3207 				    sizeof(*xudc->ep_context),
3208 				    &xudc->ep_context_phys, GFP_KERNEL);
3209 	if (!xudc->ep_context)
3210 		return -ENOMEM;
3211 
3212 	xudc->transfer_ring_pool =
3213 		dmam_pool_create(dev_name(xudc->dev), xudc->dev,
3214 				 XUDC_TRANSFER_RING_SIZE *
3215 				 sizeof(struct tegra_xudc_trb),
3216 				 sizeof(struct tegra_xudc_trb), 0);
3217 	if (!xudc->transfer_ring_pool) {
3218 		err = -ENOMEM;
3219 		goto free_ep_context;
3220 	}
3221 
3222 	INIT_LIST_HEAD(&xudc->gadget.ep_list);
3223 	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++) {
3224 		err = tegra_xudc_alloc_ep(xudc, i);
3225 		if (err < 0)
3226 			goto free_eps;
3227 	}
3228 
3229 	req = tegra_xudc_ep_alloc_request(&xudc->ep[0].usb_ep, GFP_KERNEL);
3230 	if (!req) {
3231 		err = -ENOMEM;
3232 		goto free_eps;
3233 	}
3234 	xudc->ep0_req = to_xudc_req(req);
3235 
3236 	return 0;
3237 
3238 free_eps:
3239 	for (; i > 0; i--)
3240 		tegra_xudc_free_ep(xudc, i - 1);
3241 free_ep_context:
3242 	dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3243 			  xudc->ep_context, xudc->ep_context_phys);
3244 	return err;
3245 }
3246 
3247 static void tegra_xudc_init_eps(struct tegra_xudc *xudc)
3248 {
3249 	xudc_writel(xudc, lower_32_bits(xudc->ep_context_phys), ECPLO);
3250 	xudc_writel(xudc, upper_32_bits(xudc->ep_context_phys), ECPHI);
3251 }
3252 
3253 static void tegra_xudc_free_eps(struct tegra_xudc *xudc)
3254 {
3255 	unsigned int i;
3256 
3257 	tegra_xudc_ep_free_request(&xudc->ep[0].usb_ep,
3258 				   &xudc->ep0_req->usb_req);
3259 
3260 	for (i = 0; i < ARRAY_SIZE(xudc->ep); i++)
3261 		tegra_xudc_free_ep(xudc, i);
3262 
3263 	dma_free_coherent(xudc->dev, XUDC_NR_EPS * sizeof(*xudc->ep_context),
3264 			  xudc->ep_context, xudc->ep_context_phys);
3265 }
3266 
3267 static int tegra_xudc_alloc_event_ring(struct tegra_xudc *xudc)
3268 {
3269 	unsigned int i;
3270 
3271 	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3272 		xudc->event_ring[i] =
3273 			dma_alloc_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3274 					   sizeof(*xudc->event_ring[i]),
3275 					   &xudc->event_ring_phys[i],
3276 					   GFP_KERNEL);
3277 		if (!xudc->event_ring[i])
3278 			goto free_dma;
3279 	}
3280 
3281 	return 0;
3282 
3283 free_dma:
3284 	for (; i > 0; i--) {
3285 		dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3286 				  sizeof(*xudc->event_ring[i - 1]),
3287 				  xudc->event_ring[i - 1],
3288 				  xudc->event_ring_phys[i - 1]);
3289 	}
3290 	return -ENOMEM;
3291 }
3292 
3293 static void tegra_xudc_init_event_ring(struct tegra_xudc *xudc)
3294 {
3295 	unsigned int i;
3296 	u32 val;
3297 
3298 	val = xudc_readl(xudc, SPARAM);
3299 	val &= ~(SPARAM_ERSTMAX_MASK);
3300 	val |= SPARAM_ERSTMAX(XUDC_NR_EVENT_RINGS);
3301 	xudc_writel(xudc, val, SPARAM);
3302 
3303 	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3304 		memset(xudc->event_ring[i], 0, XUDC_EVENT_RING_SIZE *
3305 		       sizeof(*xudc->event_ring[i]));
3306 
3307 		val = xudc_readl(xudc, ERSTSZ);
3308 		val &= ~(ERSTSZ_ERSTXSZ_MASK << ERSTSZ_ERSTXSZ_SHIFT(i));
3309 		val |= XUDC_EVENT_RING_SIZE << ERSTSZ_ERSTXSZ_SHIFT(i);
3310 		xudc_writel(xudc, val, ERSTSZ);
3311 
3312 		xudc_writel(xudc, lower_32_bits(xudc->event_ring_phys[i]),
3313 			    ERSTXBALO(i));
3314 		xudc_writel(xudc, upper_32_bits(xudc->event_ring_phys[i]),
3315 			    ERSTXBAHI(i));
3316 	}
3317 
3318 	val = lower_32_bits(xudc->event_ring_phys[0]);
3319 	xudc_writel(xudc, val, ERDPLO);
3320 	val |= EREPLO_ECS;
3321 	xudc_writel(xudc, val, EREPLO);
3322 
3323 	val = upper_32_bits(xudc->event_ring_phys[0]);
3324 	xudc_writel(xudc, val, ERDPHI);
3325 	xudc_writel(xudc, val, EREPHI);
3326 
3327 	xudc->ccs = true;
3328 	xudc->event_ring_index = 0;
3329 	xudc->event_ring_deq_ptr = 0;
3330 }
3331 
3332 static void tegra_xudc_free_event_ring(struct tegra_xudc *xudc)
3333 {
3334 	unsigned int i;
3335 
3336 	for (i = 0; i < ARRAY_SIZE(xudc->event_ring); i++) {
3337 		dma_free_coherent(xudc->dev, XUDC_EVENT_RING_SIZE *
3338 				  sizeof(*xudc->event_ring[i]),
3339 				  xudc->event_ring[i],
3340 				  xudc->event_ring_phys[i]);
3341 	}
3342 }
3343 
3344 static void tegra_xudc_fpci_ipfs_init(struct tegra_xudc *xudc)
3345 {
3346 	u32 val;
3347 
3348 	if (xudc->soc->has_ipfs) {
3349 		val = ipfs_readl(xudc, XUSB_DEV_CONFIGURATION_0);
3350 		val |= XUSB_DEV_CONFIGURATION_0_EN_FPCI;
3351 		ipfs_writel(xudc, val, XUSB_DEV_CONFIGURATION_0);
3352 		usleep_range(10, 15);
3353 	}
3354 
3355 	/* Enable bus master */
3356 	val = XUSB_DEV_CFG_1_IO_SPACE_EN | XUSB_DEV_CFG_1_MEMORY_SPACE_EN |
3357 		XUSB_DEV_CFG_1_BUS_MASTER_EN;
3358 	fpci_writel(xudc, val, XUSB_DEV_CFG_1);
3359 
3360 	/* Program BAR0 space */
3361 	val = fpci_readl(xudc, XUSB_DEV_CFG_4);
3362 	val &= ~(XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3363 	val |= xudc->phys_base & (XUSB_DEV_CFG_4_BASE_ADDR_MASK);
3364 
3365 	fpci_writel(xudc, val, XUSB_DEV_CFG_4);
3366 	fpci_writel(xudc, upper_32_bits(xudc->phys_base), XUSB_DEV_CFG_5);
3367 
3368 	usleep_range(100, 200);
3369 
3370 	if (xudc->soc->has_ipfs) {
3371 		/* Enable interrupt assertion */
3372 		val = ipfs_readl(xudc, XUSB_DEV_INTR_MASK_0);
3373 		val |= XUSB_DEV_INTR_MASK_0_IP_INT_MASK;
3374 		ipfs_writel(xudc, val, XUSB_DEV_INTR_MASK_0);
3375 	}
3376 }
3377 
3378 static void tegra_xudc_device_params_init(struct tegra_xudc *xudc)
3379 {
3380 	u32 val, imod;
3381 
3382 	if (xudc->soc->has_ipfs) {
3383 		val = xudc_readl(xudc, BLCG);
3384 		val |= BLCG_ALL;
3385 		val &= ~(BLCG_DFPCI | BLCG_UFPCI | BLCG_FE |
3386 				BLCG_COREPLL_PWRDN);
3387 		val |= BLCG_IOPLL_0_PWRDN;
3388 		val |= BLCG_IOPLL_1_PWRDN;
3389 		val |= BLCG_IOPLL_2_PWRDN;
3390 
3391 		xudc_writel(xudc, val, BLCG);
3392 	}
3393 
3394 	if (xudc->soc->port_speed_quirk)
3395 		tegra_xudc_limit_port_speed(xudc);
3396 
3397 	/* Set a reasonable U3 exit timer value. */
3398 	val = xudc_readl(xudc, SSPX_CORE_PADCTL4);
3399 	val &= ~(SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3_MASK);
3400 	val |= SSPX_CORE_PADCTL4_RXDAT_VLD_TIMEOUT_U3(0x5dc0);
3401 	xudc_writel(xudc, val, SSPX_CORE_PADCTL4);
3402 
3403 	/* Default ping LFPS tBurst is too large. */
3404 	val = xudc_readl(xudc, SSPX_CORE_CNT0);
3405 	val &= ~(SSPX_CORE_CNT0_PING_TBURST_MASK);
3406 	val |= SSPX_CORE_CNT0_PING_TBURST(0xa);
3407 	xudc_writel(xudc, val, SSPX_CORE_CNT0);
3408 
3409 	/* Default tPortConfiguration timeout is too small. */
3410 	val = xudc_readl(xudc, SSPX_CORE_CNT30);
3411 	val &= ~(SSPX_CORE_CNT30_LMPITP_TIMER_MASK);
3412 	val |= SSPX_CORE_CNT30_LMPITP_TIMER(0x978);
3413 	xudc_writel(xudc, val, SSPX_CORE_CNT30);
3414 
3415 	if (xudc->soc->lpm_enable) {
3416 		/* Set L1 resume duration to 95 us. */
3417 		val = xudc_readl(xudc, HSFSPI_COUNT13);
3418 		val &= ~(HSFSPI_COUNT13_U2_RESUME_K_DURATION_MASK);
3419 		val |= HSFSPI_COUNT13_U2_RESUME_K_DURATION(0x2c88);
3420 		xudc_writel(xudc, val, HSFSPI_COUNT13);
3421 	}
3422 
3423 	/*
3424 	 * Compliacne suite appears to be violating polling LFPS tBurst max
3425 	 * of 1.4us.  Send 1.45us instead.
3426 	 */
3427 	val = xudc_readl(xudc, SSPX_CORE_CNT32);
3428 	val &= ~(SSPX_CORE_CNT32_POLL_TBURST_MAX_MASK);
3429 	val |= SSPX_CORE_CNT32_POLL_TBURST_MAX(0xb0);
3430 	xudc_writel(xudc, val, SSPX_CORE_CNT32);
3431 
3432 	/* Direct HS/FS port instance to RxDetect. */
3433 	val = xudc_readl(xudc, CFG_DEV_FE);
3434 	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3435 	val |= CFG_DEV_FE_PORTREGSEL(CFG_DEV_FE_PORTREGSEL_HSFS_PI);
3436 	xudc_writel(xudc, val, CFG_DEV_FE);
3437 
3438 	val = xudc_readl(xudc, PORTSC);
3439 	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3440 	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3441 	xudc_writel(xudc, val, PORTSC);
3442 
3443 	/* Direct SS port instance to RxDetect. */
3444 	val = xudc_readl(xudc, CFG_DEV_FE);
3445 	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3446 	val |= CFG_DEV_FE_PORTREGSEL_SS_PI & CFG_DEV_FE_PORTREGSEL_MASK;
3447 	xudc_writel(xudc, val, CFG_DEV_FE);
3448 
3449 	val = xudc_readl(xudc, PORTSC);
3450 	val &= ~(PORTSC_CHANGE_MASK | PORTSC_PLS_MASK);
3451 	val |= PORTSC_LWS | PORTSC_PLS(PORTSC_PLS_RXDETECT);
3452 	xudc_writel(xudc, val, PORTSC);
3453 
3454 	/* Restore port instance. */
3455 	val = xudc_readl(xudc, CFG_DEV_FE);
3456 	val &= ~(CFG_DEV_FE_PORTREGSEL_MASK);
3457 	xudc_writel(xudc, val, CFG_DEV_FE);
3458 
3459 	/*
3460 	 * Enable INFINITE_SS_RETRY to prevent device from entering
3461 	 * Disabled.Error when attached to buggy SuperSpeed hubs.
3462 	 */
3463 	val = xudc_readl(xudc, CFG_DEV_FE);
3464 	val |= CFG_DEV_FE_INFINITE_SS_RETRY;
3465 	xudc_writel(xudc, val, CFG_DEV_FE);
3466 
3467 	/* Set interrupt moderation. */
3468 	imod = XUDC_INTERRUPT_MODERATION_US * 4;
3469 	val = xudc_readl(xudc, RT_IMOD);
3470 	val &= ~((RT_IMOD_IMODI_MASK) | (RT_IMOD_IMODC_MASK));
3471 	val |= (RT_IMOD_IMODI(imod) | RT_IMOD_IMODC(imod));
3472 	xudc_writel(xudc, val, RT_IMOD);
3473 
3474 	/* increase SSPI transaction timeout from 32us to 512us */
3475 	val = xudc_readl(xudc, CFG_DEV_SSPI_XFER);
3476 	val &= ~(CFG_DEV_SSPI_XFER_ACKTIMEOUT_MASK);
3477 	val |= CFG_DEV_SSPI_XFER_ACKTIMEOUT(0xf000);
3478 	xudc_writel(xudc, val, CFG_DEV_SSPI_XFER);
3479 }
3480 
3481 static int tegra_xudc_phy_get(struct tegra_xudc *xudc)
3482 {
3483 	int err = 0, usb3;
3484 	unsigned int i;
3485 
3486 	xudc->utmi_phy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3487 					   sizeof(*xudc->utmi_phy), GFP_KERNEL);
3488 	if (!xudc->utmi_phy)
3489 		return -ENOMEM;
3490 
3491 	xudc->usb3_phy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3492 					   sizeof(*xudc->usb3_phy), GFP_KERNEL);
3493 	if (!xudc->usb3_phy)
3494 		return -ENOMEM;
3495 
3496 	xudc->usbphy = devm_kcalloc(xudc->dev, xudc->soc->num_phys,
3497 					   sizeof(*xudc->usbphy), GFP_KERNEL);
3498 	if (!xudc->usbphy)
3499 		return -ENOMEM;
3500 
3501 	xudc->vbus_nb.notifier_call = tegra_xudc_vbus_notify;
3502 
3503 	for (i = 0; i < xudc->soc->num_phys; i++) {
3504 		char phy_name[] = "usb.-.";
3505 
3506 		/* Get USB2 phy */
3507 		snprintf(phy_name, sizeof(phy_name), "usb2-%d", i);
3508 		xudc->utmi_phy[i] = devm_phy_optional_get(xudc->dev, phy_name);
3509 		if (IS_ERR(xudc->utmi_phy[i])) {
3510 			err = PTR_ERR(xudc->utmi_phy[i]);
3511 			if (err != -EPROBE_DEFER)
3512 				dev_err(xudc->dev, "failed to get usb2-%d PHY: %d\n",
3513 					i, err);
3514 
3515 			goto clean_up;
3516 		} else if (xudc->utmi_phy[i]) {
3517 			/* Get usb-phy, if utmi phy is available */
3518 			xudc->usbphy[i] = devm_usb_get_phy_by_node(xudc->dev,
3519 						xudc->utmi_phy[i]->dev.of_node,
3520 						&xudc->vbus_nb);
3521 			if (IS_ERR(xudc->usbphy[i])) {
3522 				err = PTR_ERR(xudc->usbphy[i]);
3523 				dev_err(xudc->dev, "failed to get usbphy-%d: %d\n",
3524 					i, err);
3525 				goto clean_up;
3526 			}
3527 		} else if (!xudc->utmi_phy[i]) {
3528 			/* if utmi phy is not available, ignore USB3 phy get */
3529 			continue;
3530 		}
3531 
3532 		/* Get USB3 phy */
3533 		usb3 = tegra_xusb_padctl_get_usb3_companion(xudc->padctl, i);
3534 		if (usb3 < 0)
3535 			continue;
3536 
3537 		snprintf(phy_name, sizeof(phy_name), "usb3-%d", usb3);
3538 		xudc->usb3_phy[i] = devm_phy_optional_get(xudc->dev, phy_name);
3539 		if (IS_ERR(xudc->usb3_phy[i])) {
3540 			err = PTR_ERR(xudc->usb3_phy[i]);
3541 			if (err != -EPROBE_DEFER)
3542 				dev_err(xudc->dev, "failed to get usb3-%d PHY: %d\n",
3543 					usb3, err);
3544 
3545 			goto clean_up;
3546 		} else if (xudc->usb3_phy[i])
3547 			dev_dbg(xudc->dev, "usb3-%d PHY registered", usb3);
3548 	}
3549 
3550 	return err;
3551 
3552 clean_up:
3553 	for (i = 0; i < xudc->soc->num_phys; i++) {
3554 		xudc->usb3_phy[i] = NULL;
3555 		xudc->utmi_phy[i] = NULL;
3556 		xudc->usbphy[i] = NULL;
3557 	}
3558 
3559 	return err;
3560 }
3561 
3562 static void tegra_xudc_phy_exit(struct tegra_xudc *xudc)
3563 {
3564 	unsigned int i;
3565 
3566 	for (i = 0; i < xudc->soc->num_phys; i++) {
3567 		phy_exit(xudc->usb3_phy[i]);
3568 		phy_exit(xudc->utmi_phy[i]);
3569 	}
3570 }
3571 
3572 static int tegra_xudc_phy_init(struct tegra_xudc *xudc)
3573 {
3574 	int err;
3575 	unsigned int i;
3576 
3577 	for (i = 0; i < xudc->soc->num_phys; i++) {
3578 		err = phy_init(xudc->utmi_phy[i]);
3579 		if (err < 0) {
3580 			dev_err(xudc->dev, "UTMI PHY #%u initialization failed: %d\n", i, err);
3581 			goto exit_phy;
3582 		}
3583 
3584 		err = phy_init(xudc->usb3_phy[i]);
3585 		if (err < 0) {
3586 			dev_err(xudc->dev, "USB3 PHY #%u initialization failed: %d\n", i, err);
3587 			goto exit_phy;
3588 		}
3589 	}
3590 	return 0;
3591 
3592 exit_phy:
3593 	tegra_xudc_phy_exit(xudc);
3594 	return err;
3595 }
3596 
3597 static const char * const tegra210_xudc_supply_names[] = {
3598 	"hvdd-usb",
3599 	"avddio-usb",
3600 };
3601 
3602 static const char * const tegra210_xudc_clock_names[] = {
3603 	"dev",
3604 	"ss",
3605 	"ss_src",
3606 	"hs_src",
3607 	"fs_src",
3608 };
3609 
3610 static const char * const tegra186_xudc_clock_names[] = {
3611 	"dev",
3612 	"ss",
3613 	"ss_src",
3614 	"fs_src",
3615 };
3616 
3617 static struct tegra_xudc_soc tegra210_xudc_soc_data = {
3618 	.supply_names = tegra210_xudc_supply_names,
3619 	.num_supplies = ARRAY_SIZE(tegra210_xudc_supply_names),
3620 	.clock_names = tegra210_xudc_clock_names,
3621 	.num_clks = ARRAY_SIZE(tegra210_xudc_clock_names),
3622 	.num_phys = 4,
3623 	.u1_enable = false,
3624 	.u2_enable = true,
3625 	.lpm_enable = false,
3626 	.invalid_seq_num = true,
3627 	.pls_quirk = true,
3628 	.port_reset_quirk = true,
3629 	.port_speed_quirk = false,
3630 	.has_ipfs = true,
3631 };
3632 
3633 static struct tegra_xudc_soc tegra186_xudc_soc_data = {
3634 	.clock_names = tegra186_xudc_clock_names,
3635 	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3636 	.num_phys = 4,
3637 	.u1_enable = true,
3638 	.u2_enable = true,
3639 	.lpm_enable = false,
3640 	.invalid_seq_num = false,
3641 	.pls_quirk = false,
3642 	.port_reset_quirk = false,
3643 	.port_speed_quirk = false,
3644 	.has_ipfs = false,
3645 };
3646 
3647 static struct tegra_xudc_soc tegra194_xudc_soc_data = {
3648 	.clock_names = tegra186_xudc_clock_names,
3649 	.num_clks = ARRAY_SIZE(tegra186_xudc_clock_names),
3650 	.num_phys = 4,
3651 	.u1_enable = true,
3652 	.u2_enable = true,
3653 	.lpm_enable = true,
3654 	.invalid_seq_num = false,
3655 	.pls_quirk = false,
3656 	.port_reset_quirk = false,
3657 	.port_speed_quirk = true,
3658 	.has_ipfs = false,
3659 };
3660 
3661 static const struct of_device_id tegra_xudc_of_match[] = {
3662 	{
3663 		.compatible = "nvidia,tegra210-xudc",
3664 		.data = &tegra210_xudc_soc_data
3665 	},
3666 	{
3667 		.compatible = "nvidia,tegra186-xudc",
3668 		.data = &tegra186_xudc_soc_data
3669 	},
3670 	{
3671 		.compatible = "nvidia,tegra194-xudc",
3672 		.data = &tegra194_xudc_soc_data
3673 	},
3674 	{ }
3675 };
3676 MODULE_DEVICE_TABLE(of, tegra_xudc_of_match);
3677 
3678 static void tegra_xudc_powerdomain_remove(struct tegra_xudc *xudc)
3679 {
3680 	if (xudc->genpd_dl_ss)
3681 		device_link_del(xudc->genpd_dl_ss);
3682 	if (xudc->genpd_dl_device)
3683 		device_link_del(xudc->genpd_dl_device);
3684 	if (xudc->genpd_dev_ss)
3685 		dev_pm_domain_detach(xudc->genpd_dev_ss, true);
3686 	if (xudc->genpd_dev_device)
3687 		dev_pm_domain_detach(xudc->genpd_dev_device, true);
3688 }
3689 
3690 static int tegra_xudc_powerdomain_init(struct tegra_xudc *xudc)
3691 {
3692 	struct device *dev = xudc->dev;
3693 	int err;
3694 
3695 	xudc->genpd_dev_device = dev_pm_domain_attach_by_name(dev, "dev");
3696 	if (IS_ERR(xudc->genpd_dev_device)) {
3697 		err = PTR_ERR(xudc->genpd_dev_device);
3698 		dev_err(dev, "failed to get device power domain: %d\n", err);
3699 		return err;
3700 	}
3701 
3702 	xudc->genpd_dev_ss = dev_pm_domain_attach_by_name(dev, "ss");
3703 	if (IS_ERR(xudc->genpd_dev_ss)) {
3704 		err = PTR_ERR(xudc->genpd_dev_ss);
3705 		dev_err(dev, "failed to get SuperSpeed power domain: %d\n", err);
3706 		return err;
3707 	}
3708 
3709 	xudc->genpd_dl_device = device_link_add(dev, xudc->genpd_dev_device,
3710 						DL_FLAG_PM_RUNTIME |
3711 						DL_FLAG_STATELESS);
3712 	if (!xudc->genpd_dl_device) {
3713 		dev_err(dev, "failed to add USB device link\n");
3714 		return -ENODEV;
3715 	}
3716 
3717 	xudc->genpd_dl_ss = device_link_add(dev, xudc->genpd_dev_ss,
3718 					    DL_FLAG_PM_RUNTIME |
3719 					    DL_FLAG_STATELESS);
3720 	if (!xudc->genpd_dl_ss) {
3721 		dev_err(dev, "failed to add SuperSpeed device link\n");
3722 		return -ENODEV;
3723 	}
3724 
3725 	return 0;
3726 }
3727 
3728 static int tegra_xudc_probe(struct platform_device *pdev)
3729 {
3730 	struct tegra_xudc *xudc;
3731 	struct resource *res;
3732 	unsigned int i;
3733 	int err;
3734 
3735 	xudc = devm_kzalloc(&pdev->dev, sizeof(*xudc), GFP_KERNEL);
3736 	if (!xudc)
3737 		return -ENOMEM;
3738 
3739 	xudc->dev = &pdev->dev;
3740 	platform_set_drvdata(pdev, xudc);
3741 
3742 	xudc->soc = of_device_get_match_data(&pdev->dev);
3743 	if (!xudc->soc)
3744 		return -ENODEV;
3745 
3746 	res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "base");
3747 	xudc->base = devm_ioremap_resource(&pdev->dev, res);
3748 	if (IS_ERR(xudc->base))
3749 		return PTR_ERR(xudc->base);
3750 	xudc->phys_base = res->start;
3751 
3752 	xudc->fpci = devm_platform_ioremap_resource_byname(pdev, "fpci");
3753 	if (IS_ERR(xudc->fpci))
3754 		return PTR_ERR(xudc->fpci);
3755 
3756 	if (xudc->soc->has_ipfs) {
3757 		xudc->ipfs = devm_platform_ioremap_resource_byname(pdev, "ipfs");
3758 		if (IS_ERR(xudc->ipfs))
3759 			return PTR_ERR(xudc->ipfs);
3760 	}
3761 
3762 	xudc->irq = platform_get_irq(pdev, 0);
3763 	if (xudc->irq < 0)
3764 		return xudc->irq;
3765 
3766 	err = devm_request_irq(&pdev->dev, xudc->irq, tegra_xudc_irq, 0,
3767 			       dev_name(&pdev->dev), xudc);
3768 	if (err < 0) {
3769 		dev_err(xudc->dev, "failed to claim IRQ#%u: %d\n", xudc->irq,
3770 			err);
3771 		return err;
3772 	}
3773 
3774 	xudc->clks = devm_kcalloc(&pdev->dev, xudc->soc->num_clks, sizeof(*xudc->clks),
3775 				  GFP_KERNEL);
3776 	if (!xudc->clks)
3777 		return -ENOMEM;
3778 
3779 	for (i = 0; i < xudc->soc->num_clks; i++)
3780 		xudc->clks[i].id = xudc->soc->clock_names[i];
3781 
3782 	err = devm_clk_bulk_get(&pdev->dev, xudc->soc->num_clks, xudc->clks);
3783 	if (err) {
3784 		if (err != -EPROBE_DEFER)
3785 			dev_err(xudc->dev, "failed to request clocks: %d\n", err);
3786 
3787 		return err;
3788 	}
3789 
3790 	xudc->supplies = devm_kcalloc(&pdev->dev, xudc->soc->num_supplies,
3791 				      sizeof(*xudc->supplies), GFP_KERNEL);
3792 	if (!xudc->supplies)
3793 		return -ENOMEM;
3794 
3795 	for (i = 0; i < xudc->soc->num_supplies; i++)
3796 		xudc->supplies[i].supply = xudc->soc->supply_names[i];
3797 
3798 	err = devm_regulator_bulk_get(&pdev->dev, xudc->soc->num_supplies,
3799 				      xudc->supplies);
3800 	if (err) {
3801 		if (err != -EPROBE_DEFER)
3802 			dev_err(xudc->dev, "failed to request regulators: %d\n", err);
3803 
3804 		return err;
3805 	}
3806 
3807 	xudc->padctl = tegra_xusb_padctl_get(&pdev->dev);
3808 	if (IS_ERR(xudc->padctl))
3809 		return PTR_ERR(xudc->padctl);
3810 
3811 	err = regulator_bulk_enable(xudc->soc->num_supplies, xudc->supplies);
3812 	if (err) {
3813 		dev_err(xudc->dev, "failed to enable regulators: %d\n", err);
3814 		goto put_padctl;
3815 	}
3816 
3817 	err = tegra_xudc_phy_get(xudc);
3818 	if (err)
3819 		goto disable_regulator;
3820 
3821 	err = tegra_xudc_powerdomain_init(xudc);
3822 	if (err)
3823 		goto put_powerdomains;
3824 
3825 	err = tegra_xudc_phy_init(xudc);
3826 	if (err)
3827 		goto put_powerdomains;
3828 
3829 	err = tegra_xudc_alloc_event_ring(xudc);
3830 	if (err)
3831 		goto disable_phy;
3832 
3833 	err = tegra_xudc_alloc_eps(xudc);
3834 	if (err)
3835 		goto free_event_ring;
3836 
3837 	spin_lock_init(&xudc->lock);
3838 
3839 	init_completion(&xudc->disconnect_complete);
3840 
3841 	INIT_WORK(&xudc->usb_role_sw_work, tegra_xudc_usb_role_sw_work);
3842 
3843 	INIT_DELAYED_WORK(&xudc->plc_reset_work, tegra_xudc_plc_reset_work);
3844 
3845 	INIT_DELAYED_WORK(&xudc->port_reset_war_work,
3846 				tegra_xudc_port_reset_war_work);
3847 
3848 	pm_runtime_enable(&pdev->dev);
3849 
3850 	xudc->gadget.ops = &tegra_xudc_gadget_ops;
3851 	xudc->gadget.ep0 = &xudc->ep[0].usb_ep;
3852 	xudc->gadget.name = "tegra-xudc";
3853 	xudc->gadget.max_speed = USB_SPEED_SUPER;
3854 
3855 	err = usb_add_gadget_udc(&pdev->dev, &xudc->gadget);
3856 	if (err) {
3857 		dev_err(&pdev->dev, "failed to add USB gadget: %d\n", err);
3858 		goto free_eps;
3859 	}
3860 
3861 	return 0;
3862 
3863 free_eps:
3864 	tegra_xudc_free_eps(xudc);
3865 free_event_ring:
3866 	tegra_xudc_free_event_ring(xudc);
3867 disable_phy:
3868 	tegra_xudc_phy_exit(xudc);
3869 put_powerdomains:
3870 	tegra_xudc_powerdomain_remove(xudc);
3871 disable_regulator:
3872 	regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3873 put_padctl:
3874 	tegra_xusb_padctl_put(xudc->padctl);
3875 
3876 	return err;
3877 }
3878 
3879 static int tegra_xudc_remove(struct platform_device *pdev)
3880 {
3881 	struct tegra_xudc *xudc = platform_get_drvdata(pdev);
3882 	unsigned int i;
3883 
3884 	pm_runtime_get_sync(xudc->dev);
3885 
3886 	cancel_delayed_work_sync(&xudc->plc_reset_work);
3887 	cancel_work_sync(&xudc->usb_role_sw_work);
3888 
3889 	usb_del_gadget_udc(&xudc->gadget);
3890 
3891 	tegra_xudc_free_eps(xudc);
3892 	tegra_xudc_free_event_ring(xudc);
3893 
3894 	tegra_xudc_powerdomain_remove(xudc);
3895 
3896 	regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3897 
3898 	for (i = 0; i < xudc->soc->num_phys; i++) {
3899 		phy_power_off(xudc->utmi_phy[i]);
3900 		phy_power_off(xudc->usb3_phy[i]);
3901 	}
3902 
3903 	tegra_xudc_phy_exit(xudc);
3904 
3905 	pm_runtime_disable(xudc->dev);
3906 	pm_runtime_put(xudc->dev);
3907 
3908 	tegra_xusb_padctl_put(xudc->padctl);
3909 
3910 	return 0;
3911 }
3912 
3913 static int __maybe_unused tegra_xudc_powergate(struct tegra_xudc *xudc)
3914 {
3915 	unsigned long flags;
3916 
3917 	dev_dbg(xudc->dev, "entering ELPG\n");
3918 
3919 	spin_lock_irqsave(&xudc->lock, flags);
3920 
3921 	xudc->powergated = true;
3922 	xudc->saved_regs.ctrl = xudc_readl(xudc, CTRL);
3923 	xudc->saved_regs.portpm = xudc_readl(xudc, PORTPM);
3924 	xudc_writel(xudc, 0, CTRL);
3925 
3926 	spin_unlock_irqrestore(&xudc->lock, flags);
3927 
3928 	clk_bulk_disable_unprepare(xudc->soc->num_clks, xudc->clks);
3929 
3930 	regulator_bulk_disable(xudc->soc->num_supplies, xudc->supplies);
3931 
3932 	dev_dbg(xudc->dev, "entering ELPG done\n");
3933 	return 0;
3934 }
3935 
3936 static int __maybe_unused tegra_xudc_unpowergate(struct tegra_xudc *xudc)
3937 {
3938 	unsigned long flags;
3939 	int err;
3940 
3941 	dev_dbg(xudc->dev, "exiting ELPG\n");
3942 
3943 	err = regulator_bulk_enable(xudc->soc->num_supplies,
3944 			xudc->supplies);
3945 	if (err < 0)
3946 		return err;
3947 
3948 	err = clk_bulk_prepare_enable(xudc->soc->num_clks, xudc->clks);
3949 	if (err < 0)
3950 		return err;
3951 
3952 	tegra_xudc_fpci_ipfs_init(xudc);
3953 
3954 	tegra_xudc_device_params_init(xudc);
3955 
3956 	tegra_xudc_init_event_ring(xudc);
3957 
3958 	tegra_xudc_init_eps(xudc);
3959 
3960 	xudc_writel(xudc, xudc->saved_regs.portpm, PORTPM);
3961 	xudc_writel(xudc, xudc->saved_regs.ctrl, CTRL);
3962 
3963 	spin_lock_irqsave(&xudc->lock, flags);
3964 	xudc->powergated = false;
3965 	spin_unlock_irqrestore(&xudc->lock, flags);
3966 
3967 	dev_dbg(xudc->dev, "exiting ELPG done\n");
3968 	return 0;
3969 }
3970 
3971 static int __maybe_unused tegra_xudc_suspend(struct device *dev)
3972 {
3973 	struct tegra_xudc *xudc = dev_get_drvdata(dev);
3974 	unsigned long flags;
3975 
3976 	spin_lock_irqsave(&xudc->lock, flags);
3977 	xudc->suspended = true;
3978 	spin_unlock_irqrestore(&xudc->lock, flags);
3979 
3980 	flush_work(&xudc->usb_role_sw_work);
3981 
3982 	if (!pm_runtime_status_suspended(dev)) {
3983 		/* Forcibly disconnect before powergating. */
3984 		tegra_xudc_device_mode_off(xudc);
3985 		tegra_xudc_powergate(xudc);
3986 	}
3987 
3988 	pm_runtime_disable(dev);
3989 
3990 	return 0;
3991 }
3992 
3993 static int __maybe_unused tegra_xudc_resume(struct device *dev)
3994 {
3995 	struct tegra_xudc *xudc = dev_get_drvdata(dev);
3996 	unsigned long flags;
3997 	int err;
3998 
3999 	err = tegra_xudc_unpowergate(xudc);
4000 	if (err < 0)
4001 		return err;
4002 
4003 	spin_lock_irqsave(&xudc->lock, flags);
4004 	xudc->suspended = false;
4005 	spin_unlock_irqrestore(&xudc->lock, flags);
4006 
4007 	schedule_work(&xudc->usb_role_sw_work);
4008 
4009 	pm_runtime_enable(dev);
4010 
4011 	return 0;
4012 }
4013 
4014 static int __maybe_unused tegra_xudc_runtime_suspend(struct device *dev)
4015 {
4016 	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4017 
4018 	return tegra_xudc_powergate(xudc);
4019 }
4020 
4021 static int __maybe_unused tegra_xudc_runtime_resume(struct device *dev)
4022 {
4023 	struct tegra_xudc *xudc = dev_get_drvdata(dev);
4024 
4025 	return tegra_xudc_unpowergate(xudc);
4026 }
4027 
4028 static const struct dev_pm_ops tegra_xudc_pm_ops = {
4029 	SET_SYSTEM_SLEEP_PM_OPS(tegra_xudc_suspend, tegra_xudc_resume)
4030 	SET_RUNTIME_PM_OPS(tegra_xudc_runtime_suspend,
4031 			   tegra_xudc_runtime_resume, NULL)
4032 };
4033 
4034 static struct platform_driver tegra_xudc_driver = {
4035 	.probe = tegra_xudc_probe,
4036 	.remove = tegra_xudc_remove,
4037 	.driver = {
4038 		.name = "tegra-xudc",
4039 		.pm = &tegra_xudc_pm_ops,
4040 		.of_match_table = tegra_xudc_of_match,
4041 	},
4042 };
4043 module_platform_driver(tegra_xudc_driver);
4044 
4045 MODULE_DESCRIPTION("NVIDIA Tegra XUSB Device Controller");
4046 MODULE_AUTHOR("Andrew Bresticker <abrestic@chromium.org>");
4047 MODULE_AUTHOR("Hui Fu <hfu@nvidia.com>");
4048 MODULE_AUTHOR("Nagarjuna Kristam <nkristam@nvidia.com>");
4049 MODULE_LICENSE("GPL v2");
4050