xref: /openbmc/linux/drivers/usb/host/xhci.c (revision ae88357c)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * xHCI host controller driver
4  *
5  * Copyright (C) 2008 Intel Corp.
6  *
7  * Author: Sarah Sharp
8  * Some code borrowed from the Linux EHCI driver.
9  */
10 
11 #include <linux/pci.h>
12 #include <linux/iopoll.h>
13 #include <linux/irq.h>
14 #include <linux/log2.h>
15 #include <linux/module.h>
16 #include <linux/moduleparam.h>
17 #include <linux/slab.h>
18 #include <linux/dmi.h>
19 #include <linux/dma-mapping.h>
20 
21 #include "xhci.h"
22 #include "xhci-trace.h"
23 #include "xhci-debugfs.h"
24 #include "xhci-dbgcap.h"
25 
26 #define DRIVER_AUTHOR "Sarah Sharp"
27 #define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"
28 
29 #define	PORT_WAKE_BITS	(PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E)
30 
31 /* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */
32 static int link_quirk;
33 module_param(link_quirk, int, S_IRUGO | S_IWUSR);
34 MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");
35 
36 static unsigned long long quirks;
37 module_param(quirks, ullong, S_IRUGO);
38 MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default");
39 
40 static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring)
41 {
42 	struct xhci_segment *seg = ring->first_seg;
43 
44 	if (!td || !td->start_seg)
45 		return false;
46 	do {
47 		if (seg == td->start_seg)
48 			return true;
49 		seg = seg->next;
50 	} while (seg && seg != ring->first_seg);
51 
52 	return false;
53 }
54 
55 /*
56  * xhci_handshake - spin reading hc until handshake completes or fails
57  * @ptr: address of hc register to be read
58  * @mask: bits to look at in result of read
59  * @done: value of those bits when handshake succeeds
60  * @usec: timeout in microseconds
61  *
62  * Returns negative errno, or zero on success
63  *
64  * Success happens when the "mask" bits have the specified value (hardware
65  * handshake done).  There are two failure modes:  "usec" have passed (major
66  * hardware flakeout), or the register reads as all-ones (hardware removed).
67  */
68 int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec)
69 {
70 	u32	result;
71 	int	ret;
72 
73 	ret = readl_poll_timeout_atomic(ptr, result,
74 					(result & mask) == done ||
75 					result == U32_MAX,
76 					1, usec);
77 	if (result == U32_MAX)		/* card removed */
78 		return -ENODEV;
79 
80 	return ret;
81 }
82 
83 /*
84  * Disable interrupts and begin the xHCI halting process.
85  */
86 void xhci_quiesce(struct xhci_hcd *xhci)
87 {
88 	u32 halted;
89 	u32 cmd;
90 	u32 mask;
91 
92 	mask = ~(XHCI_IRQS);
93 	halted = readl(&xhci->op_regs->status) & STS_HALT;
94 	if (!halted)
95 		mask &= ~CMD_RUN;
96 
97 	cmd = readl(&xhci->op_regs->command);
98 	cmd &= mask;
99 	writel(cmd, &xhci->op_regs->command);
100 }
101 
102 /*
103  * Force HC into halt state.
104  *
105  * Disable any IRQs and clear the run/stop bit.
106  * HC will complete any current and actively pipelined transactions, and
107  * should halt within 16 ms of the run/stop bit being cleared.
108  * Read HC Halted bit in the status register to see when the HC is finished.
109  */
110 int xhci_halt(struct xhci_hcd *xhci)
111 {
112 	int ret;
113 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC");
114 	xhci_quiesce(xhci);
115 
116 	ret = xhci_handshake(&xhci->op_regs->status,
117 			STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
118 	if (ret) {
119 		xhci_warn(xhci, "Host halt failed, %d\n", ret);
120 		return ret;
121 	}
122 	xhci->xhc_state |= XHCI_STATE_HALTED;
123 	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
124 	return ret;
125 }
126 
127 /*
128  * Set the run bit and wait for the host to be running.
129  */
130 int xhci_start(struct xhci_hcd *xhci)
131 {
132 	u32 temp;
133 	int ret;
134 
135 	temp = readl(&xhci->op_regs->command);
136 	temp |= (CMD_RUN);
137 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.",
138 			temp);
139 	writel(temp, &xhci->op_regs->command);
140 
141 	/*
142 	 * Wait for the HCHalted Status bit to be 0 to indicate the host is
143 	 * running.
144 	 */
145 	ret = xhci_handshake(&xhci->op_regs->status,
146 			STS_HALT, 0, XHCI_MAX_HALT_USEC);
147 	if (ret == -ETIMEDOUT)
148 		xhci_err(xhci, "Host took too long to start, "
149 				"waited %u microseconds.\n",
150 				XHCI_MAX_HALT_USEC);
151 	if (!ret)
152 		/* clear state flags. Including dying, halted or removing */
153 		xhci->xhc_state = 0;
154 
155 	return ret;
156 }
157 
158 /*
159  * Reset a halted HC.
160  *
161  * This resets pipelines, timers, counters, state machines, etc.
162  * Transactions will be terminated immediately, and operational registers
163  * will be set to their defaults.
164  */
165 int xhci_reset(struct xhci_hcd *xhci)
166 {
167 	u32 command;
168 	u32 state;
169 	int ret;
170 
171 	state = readl(&xhci->op_regs->status);
172 
173 	if (state == ~(u32)0) {
174 		xhci_warn(xhci, "Host not accessible, reset failed.\n");
175 		return -ENODEV;
176 	}
177 
178 	if ((state & STS_HALT) == 0) {
179 		xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
180 		return 0;
181 	}
182 
183 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC");
184 	command = readl(&xhci->op_regs->command);
185 	command |= CMD_RESET;
186 	writel(command, &xhci->op_regs->command);
187 
188 	/* Existing Intel xHCI controllers require a delay of 1 mS,
189 	 * after setting the CMD_RESET bit, and before accessing any
190 	 * HC registers. This allows the HC to complete the
191 	 * reset operation and be ready for HC register access.
192 	 * Without this delay, the subsequent HC register access,
193 	 * may result in a system hang very rarely.
194 	 */
195 	if (xhci->quirks & XHCI_INTEL_HOST)
196 		udelay(1000);
197 
198 	ret = xhci_handshake(&xhci->op_regs->command,
199 			CMD_RESET, 0, 10 * 1000 * 1000);
200 	if (ret)
201 		return ret;
202 
203 	if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
204 		usb_asmedia_modifyflowcontrol(to_pci_dev(xhci_to_hcd(xhci)->self.controller));
205 
206 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
207 			 "Wait for controller to be ready for doorbell rings");
208 	/*
209 	 * xHCI cannot write to any doorbells or operational registers other
210 	 * than status until the "Controller Not Ready" flag is cleared.
211 	 */
212 	ret = xhci_handshake(&xhci->op_regs->status,
213 			STS_CNR, 0, 10 * 1000 * 1000);
214 
215 	xhci->usb2_rhub.bus_state.port_c_suspend = 0;
216 	xhci->usb2_rhub.bus_state.suspended_ports = 0;
217 	xhci->usb2_rhub.bus_state.resuming_ports = 0;
218 	xhci->usb3_rhub.bus_state.port_c_suspend = 0;
219 	xhci->usb3_rhub.bus_state.suspended_ports = 0;
220 	xhci->usb3_rhub.bus_state.resuming_ports = 0;
221 
222 	return ret;
223 }
224 
225 static void xhci_zero_64b_regs(struct xhci_hcd *xhci)
226 {
227 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
228 	int err, i;
229 	u64 val;
230 	u32 intrs;
231 
232 	/*
233 	 * Some Renesas controllers get into a weird state if they are
234 	 * reset while programmed with 64bit addresses (they will preserve
235 	 * the top half of the address in internal, non visible
236 	 * registers). You end up with half the address coming from the
237 	 * kernel, and the other half coming from the firmware. Also,
238 	 * changing the programming leads to extra accesses even if the
239 	 * controller is supposed to be halted. The controller ends up with
240 	 * a fatal fault, and is then ripe for being properly reset.
241 	 *
242 	 * Special care is taken to only apply this if the device is behind
243 	 * an iommu. Doing anything when there is no iommu is definitely
244 	 * unsafe...
245 	 */
246 	if (!(xhci->quirks & XHCI_ZERO_64B_REGS) || !device_iommu_mapped(dev))
247 		return;
248 
249 	xhci_info(xhci, "Zeroing 64bit base registers, expecting fault\n");
250 
251 	/* Clear HSEIE so that faults do not get signaled */
252 	val = readl(&xhci->op_regs->command);
253 	val &= ~CMD_HSEIE;
254 	writel(val, &xhci->op_regs->command);
255 
256 	/* Clear HSE (aka FATAL) */
257 	val = readl(&xhci->op_regs->status);
258 	val |= STS_FATAL;
259 	writel(val, &xhci->op_regs->status);
260 
261 	/* Now zero the registers, and brace for impact */
262 	val = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
263 	if (upper_32_bits(val))
264 		xhci_write_64(xhci, 0, &xhci->op_regs->dcbaa_ptr);
265 	val = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
266 	if (upper_32_bits(val))
267 		xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring);
268 
269 	intrs = min_t(u32, HCS_MAX_INTRS(xhci->hcs_params1),
270 		      ARRAY_SIZE(xhci->run_regs->ir_set));
271 
272 	for (i = 0; i < intrs; i++) {
273 		struct xhci_intr_reg __iomem *ir;
274 
275 		ir = &xhci->run_regs->ir_set[i];
276 		val = xhci_read_64(xhci, &ir->erst_base);
277 		if (upper_32_bits(val))
278 			xhci_write_64(xhci, 0, &ir->erst_base);
279 		val= xhci_read_64(xhci, &ir->erst_dequeue);
280 		if (upper_32_bits(val))
281 			xhci_write_64(xhci, 0, &ir->erst_dequeue);
282 	}
283 
284 	/* Wait for the fault to appear. It will be cleared on reset */
285 	err = xhci_handshake(&xhci->op_regs->status,
286 			     STS_FATAL, STS_FATAL,
287 			     XHCI_MAX_HALT_USEC);
288 	if (!err)
289 		xhci_info(xhci, "Fault detected\n");
290 }
291 
292 #ifdef CONFIG_USB_PCI
293 /*
294  * Set up MSI
295  */
296 static int xhci_setup_msi(struct xhci_hcd *xhci)
297 {
298 	int ret;
299 	/*
300 	 * TODO:Check with MSI Soc for sysdev
301 	 */
302 	struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
303 
304 	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
305 	if (ret < 0) {
306 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
307 				"failed to allocate MSI entry");
308 		return ret;
309 	}
310 
311 	ret = request_irq(pdev->irq, xhci_msi_irq,
312 				0, "xhci_hcd", xhci_to_hcd(xhci));
313 	if (ret) {
314 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
315 				"disable MSI interrupt");
316 		pci_free_irq_vectors(pdev);
317 	}
318 
319 	return ret;
320 }
321 
322 /*
323  * Set up MSI-X
324  */
325 static int xhci_setup_msix(struct xhci_hcd *xhci)
326 {
327 	int i, ret = 0;
328 	struct usb_hcd *hcd = xhci_to_hcd(xhci);
329 	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
330 
331 	/*
332 	 * calculate number of msi-x vectors supported.
333 	 * - HCS_MAX_INTRS: the max number of interrupts the host can handle,
334 	 *   with max number of interrupters based on the xhci HCSPARAMS1.
335 	 * - num_online_cpus: maximum msi-x vectors per CPUs core.
336 	 *   Add additional 1 vector to ensure always available interrupt.
337 	 */
338 	xhci->msix_count = min(num_online_cpus() + 1,
339 				HCS_MAX_INTRS(xhci->hcs_params1));
340 
341 	ret = pci_alloc_irq_vectors(pdev, xhci->msix_count, xhci->msix_count,
342 			PCI_IRQ_MSIX);
343 	if (ret < 0) {
344 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
345 				"Failed to enable MSI-X");
346 		return ret;
347 	}
348 
349 	for (i = 0; i < xhci->msix_count; i++) {
350 		ret = request_irq(pci_irq_vector(pdev, i), xhci_msi_irq, 0,
351 				"xhci_hcd", xhci_to_hcd(xhci));
352 		if (ret)
353 			goto disable_msix;
354 	}
355 
356 	hcd->msix_enabled = 1;
357 	return ret;
358 
359 disable_msix:
360 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "disable MSI-X interrupt");
361 	while (--i >= 0)
362 		free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
363 	pci_free_irq_vectors(pdev);
364 	return ret;
365 }
366 
367 /* Free any IRQs and disable MSI-X */
368 static void xhci_cleanup_msix(struct xhci_hcd *xhci)
369 {
370 	struct usb_hcd *hcd = xhci_to_hcd(xhci);
371 	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
372 
373 	if (xhci->quirks & XHCI_PLAT)
374 		return;
375 
376 	/* return if using legacy interrupt */
377 	if (hcd->irq > 0)
378 		return;
379 
380 	if (hcd->msix_enabled) {
381 		int i;
382 
383 		for (i = 0; i < xhci->msix_count; i++)
384 			free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
385 	} else {
386 		free_irq(pci_irq_vector(pdev, 0), xhci_to_hcd(xhci));
387 	}
388 
389 	pci_free_irq_vectors(pdev);
390 	hcd->msix_enabled = 0;
391 }
392 
393 static void __maybe_unused xhci_msix_sync_irqs(struct xhci_hcd *xhci)
394 {
395 	struct usb_hcd *hcd = xhci_to_hcd(xhci);
396 
397 	if (hcd->msix_enabled) {
398 		struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
399 		int i;
400 
401 		for (i = 0; i < xhci->msix_count; i++)
402 			synchronize_irq(pci_irq_vector(pdev, i));
403 	}
404 }
405 
406 static int xhci_try_enable_msi(struct usb_hcd *hcd)
407 {
408 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
409 	struct pci_dev  *pdev;
410 	int ret;
411 
412 	/* The xhci platform device has set up IRQs through usb_add_hcd. */
413 	if (xhci->quirks & XHCI_PLAT)
414 		return 0;
415 
416 	pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
417 	/*
418 	 * Some Fresco Logic host controllers advertise MSI, but fail to
419 	 * generate interrupts.  Don't even try to enable MSI.
420 	 */
421 	if (xhci->quirks & XHCI_BROKEN_MSI)
422 		goto legacy_irq;
423 
424 	/* unregister the legacy interrupt */
425 	if (hcd->irq)
426 		free_irq(hcd->irq, hcd);
427 	hcd->irq = 0;
428 
429 	ret = xhci_setup_msix(xhci);
430 	if (ret)
431 		/* fall back to msi*/
432 		ret = xhci_setup_msi(xhci);
433 
434 	if (!ret) {
435 		hcd->msi_enabled = 1;
436 		return 0;
437 	}
438 
439 	if (!pdev->irq) {
440 		xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");
441 		return -EINVAL;
442 	}
443 
444  legacy_irq:
445 	if (!strlen(hcd->irq_descr))
446 		snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",
447 			 hcd->driver->description, hcd->self.busnum);
448 
449 	/* fall back to legacy interrupt*/
450 	ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
451 			hcd->irq_descr, hcd);
452 	if (ret) {
453 		xhci_err(xhci, "request interrupt %d failed\n",
454 				pdev->irq);
455 		return ret;
456 	}
457 	hcd->irq = pdev->irq;
458 	return 0;
459 }
460 
461 #else
462 
463 static inline int xhci_try_enable_msi(struct usb_hcd *hcd)
464 {
465 	return 0;
466 }
467 
468 static inline void xhci_cleanup_msix(struct xhci_hcd *xhci)
469 {
470 }
471 
472 static inline void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
473 {
474 }
475 
476 #endif
477 
478 static void compliance_mode_recovery(struct timer_list *t)
479 {
480 	struct xhci_hcd *xhci;
481 	struct usb_hcd *hcd;
482 	struct xhci_hub *rhub;
483 	u32 temp;
484 	int i;
485 
486 	xhci = from_timer(xhci, t, comp_mode_recovery_timer);
487 	rhub = &xhci->usb3_rhub;
488 
489 	for (i = 0; i < rhub->num_ports; i++) {
490 		temp = readl(rhub->ports[i]->addr);
491 		if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) {
492 			/*
493 			 * Compliance Mode Detected. Letting USB Core
494 			 * handle the Warm Reset
495 			 */
496 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
497 					"Compliance mode detected->port %d",
498 					i + 1);
499 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
500 					"Attempting compliance mode recovery");
501 			hcd = xhci->shared_hcd;
502 
503 			if (hcd->state == HC_STATE_SUSPENDED)
504 				usb_hcd_resume_root_hub(hcd);
505 
506 			usb_hcd_poll_rh_status(hcd);
507 		}
508 	}
509 
510 	if (xhci->port_status_u0 != ((1 << rhub->num_ports) - 1))
511 		mod_timer(&xhci->comp_mode_recovery_timer,
512 			jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
513 }
514 
515 /*
516  * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver
517  * that causes ports behind that hardware to enter compliance mode sometimes.
518  * The quirk creates a timer that polls every 2 seconds the link state of
519  * each host controller's port and recovers it by issuing a Warm reset
520  * if Compliance mode is detected, otherwise the port will become "dead" (no
521  * device connections or disconnections will be detected anymore). Becasue no
522  * status event is generated when entering compliance mode (per xhci spec),
523  * this quirk is needed on systems that have the failing hardware installed.
524  */
525 static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)
526 {
527 	xhci->port_status_u0 = 0;
528 	timer_setup(&xhci->comp_mode_recovery_timer, compliance_mode_recovery,
529 		    0);
530 	xhci->comp_mode_recovery_timer.expires = jiffies +
531 			msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);
532 
533 	add_timer(&xhci->comp_mode_recovery_timer);
534 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
535 			"Compliance mode recovery timer initialized");
536 }
537 
538 /*
539  * This function identifies the systems that have installed the SN65LVPE502CP
540  * USB3.0 re-driver and that need the Compliance Mode Quirk.
541  * Systems:
542  * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820
543  */
544 static bool xhci_compliance_mode_recovery_timer_quirk_check(void)
545 {
546 	const char *dmi_product_name, *dmi_sys_vendor;
547 
548 	dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
549 	dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
550 	if (!dmi_product_name || !dmi_sys_vendor)
551 		return false;
552 
553 	if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))
554 		return false;
555 
556 	if (strstr(dmi_product_name, "Z420") ||
557 			strstr(dmi_product_name, "Z620") ||
558 			strstr(dmi_product_name, "Z820") ||
559 			strstr(dmi_product_name, "Z1 Workstation"))
560 		return true;
561 
562 	return false;
563 }
564 
565 static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)
566 {
567 	return (xhci->port_status_u0 == ((1 << xhci->usb3_rhub.num_ports) - 1));
568 }
569 
570 
571 /*
572  * Initialize memory for HCD and xHC (one-time init).
573  *
574  * Program the PAGESIZE register, initialize the device context array, create
575  * device contexts (?), set up a command ring segment (or two?), create event
576  * ring (one for now).
577  */
578 static int xhci_init(struct usb_hcd *hcd)
579 {
580 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
581 	int retval = 0;
582 
583 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init");
584 	spin_lock_init(&xhci->lock);
585 	if (xhci->hci_version == 0x95 && link_quirk) {
586 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
587 				"QUIRK: Not clearing Link TRB chain bits.");
588 		xhci->quirks |= XHCI_LINK_TRB_QUIRK;
589 	} else {
590 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
591 				"xHCI doesn't need link TRB QUIRK");
592 	}
593 	retval = xhci_mem_init(xhci, GFP_KERNEL);
594 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init");
595 
596 	/* Initializing Compliance Mode Recovery Data If Needed */
597 	if (xhci_compliance_mode_recovery_timer_quirk_check()) {
598 		xhci->quirks |= XHCI_COMP_MODE_QUIRK;
599 		compliance_mode_recovery_timer_init(xhci);
600 	}
601 
602 	return retval;
603 }
604 
605 /*-------------------------------------------------------------------------*/
606 
607 
608 static int xhci_run_finished(struct xhci_hcd *xhci)
609 {
610 	if (xhci_start(xhci)) {
611 		xhci_halt(xhci);
612 		return -ENODEV;
613 	}
614 	xhci->shared_hcd->state = HC_STATE_RUNNING;
615 	xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
616 
617 	if (xhci->quirks & XHCI_NEC_HOST)
618 		xhci_ring_cmd_db(xhci);
619 
620 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
621 			"Finished xhci_run for USB3 roothub");
622 	return 0;
623 }
624 
625 /*
626  * Start the HC after it was halted.
627  *
628  * This function is called by the USB core when the HC driver is added.
629  * Its opposite is xhci_stop().
630  *
631  * xhci_init() must be called once before this function can be called.
632  * Reset the HC, enable device slot contexts, program DCBAAP, and
633  * set command ring pointer and event ring pointer.
634  *
635  * Setup MSI-X vectors and enable interrupts.
636  */
637 int xhci_run(struct usb_hcd *hcd)
638 {
639 	u32 temp;
640 	u64 temp_64;
641 	int ret;
642 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
643 
644 	/* Start the xHCI host controller running only after the USB 2.0 roothub
645 	 * is setup.
646 	 */
647 
648 	hcd->uses_new_polling = 1;
649 	if (!usb_hcd_is_primary_hcd(hcd))
650 		return xhci_run_finished(xhci);
651 
652 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");
653 
654 	ret = xhci_try_enable_msi(hcd);
655 	if (ret)
656 		return ret;
657 
658 	temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
659 	temp_64 &= ~ERST_PTR_MASK;
660 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
661 			"ERST deq = 64'h%0lx", (long unsigned int) temp_64);
662 
663 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
664 			"// Set the interrupt modulation register");
665 	temp = readl(&xhci->ir_set->irq_control);
666 	temp &= ~ER_IRQ_INTERVAL_MASK;
667 	temp |= (xhci->imod_interval / 250) & ER_IRQ_INTERVAL_MASK;
668 	writel(temp, &xhci->ir_set->irq_control);
669 
670 	/* Set the HCD state before we enable the irqs */
671 	temp = readl(&xhci->op_regs->command);
672 	temp |= (CMD_EIE);
673 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
674 			"// Enable interrupts, cmd = 0x%x.", temp);
675 	writel(temp, &xhci->op_regs->command);
676 
677 	temp = readl(&xhci->ir_set->irq_pending);
678 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
679 			"// Enabling event ring interrupter %p by writing 0x%x to irq_pending",
680 			xhci->ir_set, (unsigned int) ER_IRQ_ENABLE(temp));
681 	writel(ER_IRQ_ENABLE(temp), &xhci->ir_set->irq_pending);
682 
683 	if (xhci->quirks & XHCI_NEC_HOST) {
684 		struct xhci_command *command;
685 
686 		command = xhci_alloc_command(xhci, false, GFP_KERNEL);
687 		if (!command)
688 			return -ENOMEM;
689 
690 		ret = xhci_queue_vendor_command(xhci, command, 0, 0, 0,
691 				TRB_TYPE(TRB_NEC_GET_FW));
692 		if (ret)
693 			xhci_free_command(xhci, command);
694 	}
695 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
696 			"Finished xhci_run for USB2 roothub");
697 
698 	xhci_dbc_init(xhci);
699 
700 	xhci_debugfs_init(xhci);
701 
702 	return 0;
703 }
704 EXPORT_SYMBOL_GPL(xhci_run);
705 
706 /*
707  * Stop xHCI driver.
708  *
709  * This function is called by the USB core when the HC driver is removed.
710  * Its opposite is xhci_run().
711  *
712  * Disable device contexts, disable IRQs, and quiesce the HC.
713  * Reset the HC, finish any completed transactions, and cleanup memory.
714  */
715 static void xhci_stop(struct usb_hcd *hcd)
716 {
717 	u32 temp;
718 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
719 
720 	mutex_lock(&xhci->mutex);
721 
722 	/* Only halt host and free memory after both hcds are removed */
723 	if (!usb_hcd_is_primary_hcd(hcd)) {
724 		mutex_unlock(&xhci->mutex);
725 		return;
726 	}
727 
728 	xhci_dbc_exit(xhci);
729 
730 	spin_lock_irq(&xhci->lock);
731 	xhci->xhc_state |= XHCI_STATE_HALTED;
732 	xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
733 	xhci_halt(xhci);
734 	xhci_reset(xhci);
735 	spin_unlock_irq(&xhci->lock);
736 
737 	xhci_cleanup_msix(xhci);
738 
739 	/* Deleting Compliance Mode Recovery Timer */
740 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
741 			(!(xhci_all_ports_seen_u0(xhci)))) {
742 		del_timer_sync(&xhci->comp_mode_recovery_timer);
743 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
744 				"%s: compliance mode recovery timer deleted",
745 				__func__);
746 	}
747 
748 	if (xhci->quirks & XHCI_AMD_PLL_FIX)
749 		usb_amd_dev_put();
750 
751 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
752 			"// Disabling event ring interrupts");
753 	temp = readl(&xhci->op_regs->status);
754 	writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
755 	temp = readl(&xhci->ir_set->irq_pending);
756 	writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
757 
758 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory");
759 	xhci_mem_cleanup(xhci);
760 	xhci_debugfs_exit(xhci);
761 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
762 			"xhci_stop completed - status = %x",
763 			readl(&xhci->op_regs->status));
764 	mutex_unlock(&xhci->mutex);
765 }
766 
767 /*
768  * Shutdown HC (not bus-specific)
769  *
770  * This is called when the machine is rebooting or halting.  We assume that the
771  * machine will be powered off, and the HC's internal state will be reset.
772  * Don't bother to free memory.
773  *
774  * This will only ever be called with the main usb_hcd (the USB3 roothub).
775  */
776 void xhci_shutdown(struct usb_hcd *hcd)
777 {
778 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
779 
780 	if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
781 		usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev));
782 
783 	spin_lock_irq(&xhci->lock);
784 	xhci_halt(xhci);
785 	/* Workaround for spurious wakeups at shutdown with HSW */
786 	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
787 		xhci_reset(xhci);
788 	spin_unlock_irq(&xhci->lock);
789 
790 	xhci_cleanup_msix(xhci);
791 
792 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
793 			"xhci_shutdown completed - status = %x",
794 			readl(&xhci->op_regs->status));
795 }
796 EXPORT_SYMBOL_GPL(xhci_shutdown);
797 
798 #ifdef CONFIG_PM
799 static void xhci_save_registers(struct xhci_hcd *xhci)
800 {
801 	xhci->s3.command = readl(&xhci->op_regs->command);
802 	xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification);
803 	xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
804 	xhci->s3.config_reg = readl(&xhci->op_regs->config_reg);
805 	xhci->s3.erst_size = readl(&xhci->ir_set->erst_size);
806 	xhci->s3.erst_base = xhci_read_64(xhci, &xhci->ir_set->erst_base);
807 	xhci->s3.erst_dequeue = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
808 	xhci->s3.irq_pending = readl(&xhci->ir_set->irq_pending);
809 	xhci->s3.irq_control = readl(&xhci->ir_set->irq_control);
810 }
811 
812 static void xhci_restore_registers(struct xhci_hcd *xhci)
813 {
814 	writel(xhci->s3.command, &xhci->op_regs->command);
815 	writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
816 	xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
817 	writel(xhci->s3.config_reg, &xhci->op_regs->config_reg);
818 	writel(xhci->s3.erst_size, &xhci->ir_set->erst_size);
819 	xhci_write_64(xhci, xhci->s3.erst_base, &xhci->ir_set->erst_base);
820 	xhci_write_64(xhci, xhci->s3.erst_dequeue, &xhci->ir_set->erst_dequeue);
821 	writel(xhci->s3.irq_pending, &xhci->ir_set->irq_pending);
822 	writel(xhci->s3.irq_control, &xhci->ir_set->irq_control);
823 }
824 
825 static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
826 {
827 	u64	val_64;
828 
829 	/* step 2: initialize command ring buffer */
830 	val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
831 	val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
832 		(xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
833 				      xhci->cmd_ring->dequeue) &
834 		 (u64) ~CMD_RING_RSVD_BITS) |
835 		xhci->cmd_ring->cycle_state;
836 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
837 			"// Setting command ring address to 0x%llx",
838 			(long unsigned long) val_64);
839 	xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
840 }
841 
842 /*
843  * The whole command ring must be cleared to zero when we suspend the host.
844  *
845  * The host doesn't save the command ring pointer in the suspend well, so we
846  * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
847  * aligned, because of the reserved bits in the command ring dequeue pointer
848  * register.  Therefore, we can't just set the dequeue pointer back in the
849  * middle of the ring (TRBs are 16-byte aligned).
850  */
851 static void xhci_clear_command_ring(struct xhci_hcd *xhci)
852 {
853 	struct xhci_ring *ring;
854 	struct xhci_segment *seg;
855 
856 	ring = xhci->cmd_ring;
857 	seg = ring->deq_seg;
858 	do {
859 		memset(seg->trbs, 0,
860 			sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
861 		seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
862 			cpu_to_le32(~TRB_CYCLE);
863 		seg = seg->next;
864 	} while (seg != ring->deq_seg);
865 
866 	/* Reset the software enqueue and dequeue pointers */
867 	ring->deq_seg = ring->first_seg;
868 	ring->dequeue = ring->first_seg->trbs;
869 	ring->enq_seg = ring->deq_seg;
870 	ring->enqueue = ring->dequeue;
871 
872 	ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
873 	/*
874 	 * Ring is now zeroed, so the HW should look for change of ownership
875 	 * when the cycle bit is set to 1.
876 	 */
877 	ring->cycle_state = 1;
878 
879 	/*
880 	 * Reset the hardware dequeue pointer.
881 	 * Yes, this will need to be re-written after resume, but we're paranoid
882 	 * and want to make sure the hardware doesn't access bogus memory
883 	 * because, say, the BIOS or an SMI started the host without changing
884 	 * the command ring pointers.
885 	 */
886 	xhci_set_cmd_ring_deq(xhci);
887 }
888 
889 /*
890  * Disable port wake bits if do_wakeup is not set.
891  *
892  * Also clear a possible internal port wake state left hanging for ports that
893  * detected termination but never successfully enumerated (trained to 0U).
894  * Internal wake causes immediate xHCI wake after suspend. PORT_CSC write done
895  * at enumeration clears this wake, force one here as well for unconnected ports
896  */
897 
898 static void xhci_disable_hub_port_wake(struct xhci_hcd *xhci,
899 				       struct xhci_hub *rhub,
900 				       bool do_wakeup)
901 {
902 	unsigned long flags;
903 	u32 t1, t2, portsc;
904 	int i;
905 
906 	spin_lock_irqsave(&xhci->lock, flags);
907 
908 	for (i = 0; i < rhub->num_ports; i++) {
909 		portsc = readl(rhub->ports[i]->addr);
910 		t1 = xhci_port_state_to_neutral(portsc);
911 		t2 = t1;
912 
913 		/* clear wake bits if do_wake is not set */
914 		if (!do_wakeup)
915 			t2 &= ~PORT_WAKE_BITS;
916 
917 		/* Don't touch csc bit if connected or connect change is set */
918 		if (!(portsc & (PORT_CSC | PORT_CONNECT)))
919 			t2 |= PORT_CSC;
920 
921 		if (t1 != t2) {
922 			writel(t2, rhub->ports[i]->addr);
923 			xhci_dbg(xhci, "config port %d-%d wake bits, portsc: 0x%x, write: 0x%x\n",
924 				 rhub->hcd->self.busnum, i + 1, portsc, t2);
925 		}
926 	}
927 	spin_unlock_irqrestore(&xhci->lock, flags);
928 }
929 
930 static bool xhci_pending_portevent(struct xhci_hcd *xhci)
931 {
932 	struct xhci_port	**ports;
933 	int			port_index;
934 	u32			status;
935 	u32			portsc;
936 
937 	status = readl(&xhci->op_regs->status);
938 	if (status & STS_EINT)
939 		return true;
940 	/*
941 	 * Checking STS_EINT is not enough as there is a lag between a change
942 	 * bit being set and the Port Status Change Event that it generated
943 	 * being written to the Event Ring. See note in xhci 1.1 section 4.19.2.
944 	 */
945 
946 	port_index = xhci->usb2_rhub.num_ports;
947 	ports = xhci->usb2_rhub.ports;
948 	while (port_index--) {
949 		portsc = readl(ports[port_index]->addr);
950 		if (portsc & PORT_CHANGE_MASK ||
951 		    (portsc & PORT_PLS_MASK) == XDEV_RESUME)
952 			return true;
953 	}
954 	port_index = xhci->usb3_rhub.num_ports;
955 	ports = xhci->usb3_rhub.ports;
956 	while (port_index--) {
957 		portsc = readl(ports[port_index]->addr);
958 		if (portsc & PORT_CHANGE_MASK ||
959 		    (portsc & PORT_PLS_MASK) == XDEV_RESUME)
960 			return true;
961 	}
962 	return false;
963 }
964 
965 /*
966  * Stop HC (not bus-specific)
967  *
968  * This is called when the machine transition into S3/S4 mode.
969  *
970  */
971 int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)
972 {
973 	int			rc = 0;
974 	unsigned int		delay = XHCI_MAX_HALT_USEC * 2;
975 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
976 	u32			command;
977 	u32			res;
978 
979 	if (!hcd->state)
980 		return 0;
981 
982 	if (hcd->state != HC_STATE_SUSPENDED ||
983 			xhci->shared_hcd->state != HC_STATE_SUSPENDED)
984 		return -EINVAL;
985 
986 	/* Clear root port wake on bits if wakeup not allowed. */
987 	xhci_disable_hub_port_wake(xhci, &xhci->usb3_rhub, do_wakeup);
988 	xhci_disable_hub_port_wake(xhci, &xhci->usb2_rhub, do_wakeup);
989 
990 	if (!HCD_HW_ACCESSIBLE(hcd))
991 		return 0;
992 
993 	xhci_dbc_suspend(xhci);
994 
995 	/* Don't poll the roothubs on bus suspend. */
996 	xhci_dbg(xhci, "%s: stopping port polling.\n", __func__);
997 	clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
998 	del_timer_sync(&hcd->rh_timer);
999 	clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
1000 	del_timer_sync(&xhci->shared_hcd->rh_timer);
1001 
1002 	if (xhci->quirks & XHCI_SUSPEND_DELAY)
1003 		usleep_range(1000, 1500);
1004 
1005 	spin_lock_irq(&xhci->lock);
1006 	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
1007 	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
1008 	/* step 1: stop endpoint */
1009 	/* skipped assuming that port suspend has done */
1010 
1011 	/* step 2: clear Run/Stop bit */
1012 	command = readl(&xhci->op_regs->command);
1013 	command &= ~CMD_RUN;
1014 	writel(command, &xhci->op_regs->command);
1015 
1016 	/* Some chips from Fresco Logic need an extraordinary delay */
1017 	delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;
1018 
1019 	if (xhci_handshake(&xhci->op_regs->status,
1020 		      STS_HALT, STS_HALT, delay)) {
1021 		xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
1022 		spin_unlock_irq(&xhci->lock);
1023 		return -ETIMEDOUT;
1024 	}
1025 	xhci_clear_command_ring(xhci);
1026 
1027 	/* step 3: save registers */
1028 	xhci_save_registers(xhci);
1029 
1030 	/* step 4: set CSS flag */
1031 	command = readl(&xhci->op_regs->command);
1032 	command |= CMD_CSS;
1033 	writel(command, &xhci->op_regs->command);
1034 	xhci->broken_suspend = 0;
1035 	if (xhci_handshake(&xhci->op_regs->status,
1036 				STS_SAVE, 0, 20 * 1000)) {
1037 	/*
1038 	 * AMD SNPS xHC 3.0 occasionally does not clear the
1039 	 * SSS bit of USBSTS and when driver tries to poll
1040 	 * to see if the xHC clears BIT(8) which never happens
1041 	 * and driver assumes that controller is not responding
1042 	 * and times out. To workaround this, its good to check
1043 	 * if SRE and HCE bits are not set (as per xhci
1044 	 * Section 5.4.2) and bypass the timeout.
1045 	 */
1046 		res = readl(&xhci->op_regs->status);
1047 		if ((xhci->quirks & XHCI_SNPS_BROKEN_SUSPEND) &&
1048 		    (((res & STS_SRE) == 0) &&
1049 				((res & STS_HCE) == 0))) {
1050 			xhci->broken_suspend = 1;
1051 		} else {
1052 			xhci_warn(xhci, "WARN: xHC save state timeout\n");
1053 			spin_unlock_irq(&xhci->lock);
1054 			return -ETIMEDOUT;
1055 		}
1056 	}
1057 	spin_unlock_irq(&xhci->lock);
1058 
1059 	/*
1060 	 * Deleting Compliance Mode Recovery Timer because the xHCI Host
1061 	 * is about to be suspended.
1062 	 */
1063 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
1064 			(!(xhci_all_ports_seen_u0(xhci)))) {
1065 		del_timer_sync(&xhci->comp_mode_recovery_timer);
1066 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1067 				"%s: compliance mode recovery timer deleted",
1068 				__func__);
1069 	}
1070 
1071 	/* step 5: remove core well power */
1072 	/* synchronize irq when using MSI-X */
1073 	xhci_msix_sync_irqs(xhci);
1074 
1075 	return rc;
1076 }
1077 EXPORT_SYMBOL_GPL(xhci_suspend);
1078 
1079 /*
1080  * start xHC (not bus-specific)
1081  *
1082  * This is called when the machine transition from S3/S4 mode.
1083  *
1084  */
1085 int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
1086 {
1087 	u32			command, temp = 0;
1088 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
1089 	struct usb_hcd		*secondary_hcd;
1090 	int			retval = 0;
1091 	bool			comp_timer_running = false;
1092 	bool			pending_portevent = false;
1093 
1094 	if (!hcd->state)
1095 		return 0;
1096 
1097 	/* Wait a bit if either of the roothubs need to settle from the
1098 	 * transition into bus suspend.
1099 	 */
1100 
1101 	if (time_before(jiffies, xhci->usb2_rhub.bus_state.next_statechange) ||
1102 	    time_before(jiffies, xhci->usb3_rhub.bus_state.next_statechange))
1103 		msleep(100);
1104 
1105 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
1106 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
1107 
1108 	spin_lock_irq(&xhci->lock);
1109 	if ((xhci->quirks & XHCI_RESET_ON_RESUME) || xhci->broken_suspend)
1110 		hibernated = true;
1111 
1112 	if (!hibernated) {
1113 		/*
1114 		 * Some controllers might lose power during suspend, so wait
1115 		 * for controller not ready bit to clear, just as in xHC init.
1116 		 */
1117 		retval = xhci_handshake(&xhci->op_regs->status,
1118 					STS_CNR, 0, 10 * 1000 * 1000);
1119 		if (retval) {
1120 			xhci_warn(xhci, "Controller not ready at resume %d\n",
1121 				  retval);
1122 			spin_unlock_irq(&xhci->lock);
1123 			return retval;
1124 		}
1125 		/* step 1: restore register */
1126 		xhci_restore_registers(xhci);
1127 		/* step 2: initialize command ring buffer */
1128 		xhci_set_cmd_ring_deq(xhci);
1129 		/* step 3: restore state and start state*/
1130 		/* step 3: set CRS flag */
1131 		command = readl(&xhci->op_regs->command);
1132 		command |= CMD_CRS;
1133 		writel(command, &xhci->op_regs->command);
1134 		/*
1135 		 * Some controllers take up to 55+ ms to complete the controller
1136 		 * restore so setting the timeout to 100ms. Xhci specification
1137 		 * doesn't mention any timeout value.
1138 		 */
1139 		if (xhci_handshake(&xhci->op_regs->status,
1140 			      STS_RESTORE, 0, 100 * 1000)) {
1141 			xhci_warn(xhci, "WARN: xHC restore state timeout\n");
1142 			spin_unlock_irq(&xhci->lock);
1143 			return -ETIMEDOUT;
1144 		}
1145 		temp = readl(&xhci->op_regs->status);
1146 	}
1147 
1148 	/* If restore operation fails, re-initialize the HC during resume */
1149 	if ((temp & STS_SRE) || hibernated) {
1150 
1151 		if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
1152 				!(xhci_all_ports_seen_u0(xhci))) {
1153 			del_timer_sync(&xhci->comp_mode_recovery_timer);
1154 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1155 				"Compliance Mode Recovery Timer deleted!");
1156 		}
1157 
1158 		/* Let the USB core know _both_ roothubs lost power. */
1159 		usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
1160 		usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
1161 
1162 		xhci_dbg(xhci, "Stop HCD\n");
1163 		xhci_halt(xhci);
1164 		xhci_zero_64b_regs(xhci);
1165 		retval = xhci_reset(xhci);
1166 		spin_unlock_irq(&xhci->lock);
1167 		if (retval)
1168 			return retval;
1169 		xhci_cleanup_msix(xhci);
1170 
1171 		xhci_dbg(xhci, "// Disabling event ring interrupts\n");
1172 		temp = readl(&xhci->op_regs->status);
1173 		writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
1174 		temp = readl(&xhci->ir_set->irq_pending);
1175 		writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
1176 
1177 		xhci_dbg(xhci, "cleaning up memory\n");
1178 		xhci_mem_cleanup(xhci);
1179 		xhci_debugfs_exit(xhci);
1180 		xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
1181 			    readl(&xhci->op_regs->status));
1182 
1183 		/* USB core calls the PCI reinit and start functions twice:
1184 		 * first with the primary HCD, and then with the secondary HCD.
1185 		 * If we don't do the same, the host will never be started.
1186 		 */
1187 		if (!usb_hcd_is_primary_hcd(hcd))
1188 			secondary_hcd = hcd;
1189 		else
1190 			secondary_hcd = xhci->shared_hcd;
1191 
1192 		xhci_dbg(xhci, "Initialize the xhci_hcd\n");
1193 		retval = xhci_init(hcd->primary_hcd);
1194 		if (retval)
1195 			return retval;
1196 		comp_timer_running = true;
1197 
1198 		xhci_dbg(xhci, "Start the primary HCD\n");
1199 		retval = xhci_run(hcd->primary_hcd);
1200 		if (!retval) {
1201 			xhci_dbg(xhci, "Start the secondary HCD\n");
1202 			retval = xhci_run(secondary_hcd);
1203 		}
1204 		hcd->state = HC_STATE_SUSPENDED;
1205 		xhci->shared_hcd->state = HC_STATE_SUSPENDED;
1206 		goto done;
1207 	}
1208 
1209 	/* step 4: set Run/Stop bit */
1210 	command = readl(&xhci->op_regs->command);
1211 	command |= CMD_RUN;
1212 	writel(command, &xhci->op_regs->command);
1213 	xhci_handshake(&xhci->op_regs->status, STS_HALT,
1214 		  0, 250 * 1000);
1215 
1216 	/* step 5: walk topology and initialize portsc,
1217 	 * portpmsc and portli
1218 	 */
1219 	/* this is done in bus_resume */
1220 
1221 	/* step 6: restart each of the previously
1222 	 * Running endpoints by ringing their doorbells
1223 	 */
1224 
1225 	spin_unlock_irq(&xhci->lock);
1226 
1227 	xhci_dbc_resume(xhci);
1228 
1229  done:
1230 	if (retval == 0) {
1231 		/*
1232 		 * Resume roothubs only if there are pending events.
1233 		 * USB 3 devices resend U3 LFPS wake after a 100ms delay if
1234 		 * the first wake signalling failed, give it that chance.
1235 		 */
1236 		pending_portevent = xhci_pending_portevent(xhci);
1237 		if (!pending_portevent) {
1238 			msleep(120);
1239 			pending_portevent = xhci_pending_portevent(xhci);
1240 		}
1241 
1242 		if (pending_portevent) {
1243 			usb_hcd_resume_root_hub(xhci->shared_hcd);
1244 			usb_hcd_resume_root_hub(hcd);
1245 		}
1246 	}
1247 	/*
1248 	 * If system is subject to the Quirk, Compliance Mode Timer needs to
1249 	 * be re-initialized Always after a system resume. Ports are subject
1250 	 * to suffer the Compliance Mode issue again. It doesn't matter if
1251 	 * ports have entered previously to U0 before system's suspension.
1252 	 */
1253 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)
1254 		compliance_mode_recovery_timer_init(xhci);
1255 
1256 	if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
1257 		usb_asmedia_modifyflowcontrol(to_pci_dev(hcd->self.controller));
1258 
1259 	/* Re-enable port polling. */
1260 	xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1261 	set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
1262 	usb_hcd_poll_rh_status(xhci->shared_hcd);
1263 	set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1264 	usb_hcd_poll_rh_status(hcd);
1265 
1266 	return retval;
1267 }
1268 EXPORT_SYMBOL_GPL(xhci_resume);
1269 #endif	/* CONFIG_PM */
1270 
1271 /*-------------------------------------------------------------------------*/
1272 
1273 static int xhci_map_temp_buffer(struct usb_hcd *hcd, struct urb *urb)
1274 {
1275 	void *temp;
1276 	int ret = 0;
1277 	unsigned int buf_len;
1278 	enum dma_data_direction dir;
1279 
1280 	dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1281 	buf_len = urb->transfer_buffer_length;
1282 
1283 	temp = kzalloc_node(buf_len, GFP_ATOMIC,
1284 			    dev_to_node(hcd->self.sysdev));
1285 
1286 	if (usb_urb_dir_out(urb))
1287 		sg_pcopy_to_buffer(urb->sg, urb->num_sgs,
1288 				   temp, buf_len, 0);
1289 
1290 	urb->transfer_buffer = temp;
1291 	urb->transfer_dma = dma_map_single(hcd->self.sysdev,
1292 					   urb->transfer_buffer,
1293 					   urb->transfer_buffer_length,
1294 					   dir);
1295 
1296 	if (dma_mapping_error(hcd->self.sysdev,
1297 			      urb->transfer_dma)) {
1298 		ret = -EAGAIN;
1299 		kfree(temp);
1300 	} else {
1301 		urb->transfer_flags |= URB_DMA_MAP_SINGLE;
1302 	}
1303 
1304 	return ret;
1305 }
1306 
1307 static bool xhci_urb_temp_buffer_required(struct usb_hcd *hcd,
1308 					  struct urb *urb)
1309 {
1310 	bool ret = false;
1311 	unsigned int i;
1312 	unsigned int len = 0;
1313 	unsigned int trb_size;
1314 	unsigned int max_pkt;
1315 	struct scatterlist *sg;
1316 	struct scatterlist *tail_sg;
1317 
1318 	tail_sg = urb->sg;
1319 	max_pkt = usb_endpoint_maxp(&urb->ep->desc);
1320 
1321 	if (!urb->num_sgs)
1322 		return ret;
1323 
1324 	if (urb->dev->speed >= USB_SPEED_SUPER)
1325 		trb_size = TRB_CACHE_SIZE_SS;
1326 	else
1327 		trb_size = TRB_CACHE_SIZE_HS;
1328 
1329 	if (urb->transfer_buffer_length != 0 &&
1330 	    !(urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP)) {
1331 		for_each_sg(urb->sg, sg, urb->num_sgs, i) {
1332 			len = len + sg->length;
1333 			if (i > trb_size - 2) {
1334 				len = len - tail_sg->length;
1335 				if (len < max_pkt) {
1336 					ret = true;
1337 					break;
1338 				}
1339 
1340 				tail_sg = sg_next(tail_sg);
1341 			}
1342 		}
1343 	}
1344 	return ret;
1345 }
1346 
1347 static void xhci_unmap_temp_buf(struct usb_hcd *hcd, struct urb *urb)
1348 {
1349 	unsigned int len;
1350 	unsigned int buf_len;
1351 	enum dma_data_direction dir;
1352 
1353 	dir = usb_urb_dir_in(urb) ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1354 
1355 	buf_len = urb->transfer_buffer_length;
1356 
1357 	if (IS_ENABLED(CONFIG_HAS_DMA) &&
1358 	    (urb->transfer_flags & URB_DMA_MAP_SINGLE))
1359 		dma_unmap_single(hcd->self.sysdev,
1360 				 urb->transfer_dma,
1361 				 urb->transfer_buffer_length,
1362 				 dir);
1363 
1364 	if (usb_urb_dir_in(urb))
1365 		len = sg_pcopy_from_buffer(urb->sg, urb->num_sgs,
1366 					   urb->transfer_buffer,
1367 					   buf_len,
1368 					   0);
1369 
1370 	urb->transfer_flags &= ~URB_DMA_MAP_SINGLE;
1371 	kfree(urb->transfer_buffer);
1372 	urb->transfer_buffer = NULL;
1373 }
1374 
1375 /*
1376  * Bypass the DMA mapping if URB is suitable for Immediate Transfer (IDT),
1377  * we'll copy the actual data into the TRB address register. This is limited to
1378  * transfers up to 8 bytes on output endpoints of any kind with wMaxPacketSize
1379  * >= 8 bytes. If suitable for IDT only one Transfer TRB per TD is allowed.
1380  */
1381 static int xhci_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
1382 				gfp_t mem_flags)
1383 {
1384 	struct xhci_hcd *xhci;
1385 
1386 	xhci = hcd_to_xhci(hcd);
1387 
1388 	if (xhci_urb_suitable_for_idt(urb))
1389 		return 0;
1390 
1391 	if (xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) {
1392 		if (xhci_urb_temp_buffer_required(hcd, urb))
1393 			return xhci_map_temp_buffer(hcd, urb);
1394 	}
1395 	return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
1396 }
1397 
1398 static void xhci_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
1399 {
1400 	struct xhci_hcd *xhci;
1401 	bool unmap_temp_buf = false;
1402 
1403 	xhci = hcd_to_xhci(hcd);
1404 
1405 	if (urb->num_sgs && (urb->transfer_flags & URB_DMA_MAP_SINGLE))
1406 		unmap_temp_buf = true;
1407 
1408 	if ((xhci->quirks & XHCI_SG_TRB_CACHE_SIZE_QUIRK) && unmap_temp_buf)
1409 		xhci_unmap_temp_buf(hcd, urb);
1410 	else
1411 		usb_hcd_unmap_urb_for_dma(hcd, urb);
1412 }
1413 
1414 /**
1415  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
1416  * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
1417  * value to right shift 1 for the bitmask.
1418  *
1419  * Index  = (epnum * 2) + direction - 1,
1420  * where direction = 0 for OUT, 1 for IN.
1421  * For control endpoints, the IN index is used (OUT index is unused), so
1422  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
1423  */
1424 unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
1425 {
1426 	unsigned int index;
1427 	if (usb_endpoint_xfer_control(desc))
1428 		index = (unsigned int) (usb_endpoint_num(desc)*2);
1429 	else
1430 		index = (unsigned int) (usb_endpoint_num(desc)*2) +
1431 			(usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
1432 	return index;
1433 }
1434 EXPORT_SYMBOL_GPL(xhci_get_endpoint_index);
1435 
1436 /* The reverse operation to xhci_get_endpoint_index. Calculate the USB endpoint
1437  * address from the XHCI endpoint index.
1438  */
1439 unsigned int xhci_get_endpoint_address(unsigned int ep_index)
1440 {
1441 	unsigned int number = DIV_ROUND_UP(ep_index, 2);
1442 	unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN;
1443 	return direction | number;
1444 }
1445 
1446 /* Find the flag for this endpoint (for use in the control context).  Use the
1447  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
1448  * bit 1, etc.
1449  */
1450 static unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
1451 {
1452 	return 1 << (xhci_get_endpoint_index(desc) + 1);
1453 }
1454 
1455 /* Compute the last valid endpoint context index.  Basically, this is the
1456  * endpoint index plus one.  For slot contexts with more than valid endpoint,
1457  * we find the most significant bit set in the added contexts flags.
1458  * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000
1459  * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one.
1460  */
1461 unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
1462 {
1463 	return fls(added_ctxs) - 1;
1464 }
1465 
1466 /* Returns 1 if the arguments are OK;
1467  * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
1468  */
1469 static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
1470 		struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
1471 		const char *func) {
1472 	struct xhci_hcd	*xhci;
1473 	struct xhci_virt_device	*virt_dev;
1474 
1475 	if (!hcd || (check_ep && !ep) || !udev) {
1476 		pr_debug("xHCI %s called with invalid args\n", func);
1477 		return -EINVAL;
1478 	}
1479 	if (!udev->parent) {
1480 		pr_debug("xHCI %s called for root hub\n", func);
1481 		return 0;
1482 	}
1483 
1484 	xhci = hcd_to_xhci(hcd);
1485 	if (check_virt_dev) {
1486 		if (!udev->slot_id || !xhci->devs[udev->slot_id]) {
1487 			xhci_dbg(xhci, "xHCI %s called with unaddressed device\n",
1488 					func);
1489 			return -EINVAL;
1490 		}
1491 
1492 		virt_dev = xhci->devs[udev->slot_id];
1493 		if (virt_dev->udev != udev) {
1494 			xhci_dbg(xhci, "xHCI %s called with udev and "
1495 					  "virt_dev does not match\n", func);
1496 			return -EINVAL;
1497 		}
1498 	}
1499 
1500 	if (xhci->xhc_state & XHCI_STATE_HALTED)
1501 		return -ENODEV;
1502 
1503 	return 1;
1504 }
1505 
1506 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1507 		struct usb_device *udev, struct xhci_command *command,
1508 		bool ctx_change, bool must_succeed);
1509 
1510 /*
1511  * Full speed devices may have a max packet size greater than 8 bytes, but the
1512  * USB core doesn't know that until it reads the first 8 bytes of the
1513  * descriptor.  If the usb_device's max packet size changes after that point,
1514  * we need to issue an evaluate context command and wait on it.
1515  */
1516 static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
1517 		unsigned int ep_index, struct urb *urb, gfp_t mem_flags)
1518 {
1519 	struct xhci_container_ctx *out_ctx;
1520 	struct xhci_input_control_ctx *ctrl_ctx;
1521 	struct xhci_ep_ctx *ep_ctx;
1522 	struct xhci_command *command;
1523 	int max_packet_size;
1524 	int hw_max_packet_size;
1525 	int ret = 0;
1526 
1527 	out_ctx = xhci->devs[slot_id]->out_ctx;
1528 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1529 	hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
1530 	max_packet_size = usb_endpoint_maxp(&urb->dev->ep0.desc);
1531 	if (hw_max_packet_size != max_packet_size) {
1532 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1533 				"Max Packet Size for ep 0 changed.");
1534 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1535 				"Max packet size in usb_device = %d",
1536 				max_packet_size);
1537 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1538 				"Max packet size in xHCI HW = %d",
1539 				hw_max_packet_size);
1540 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1541 				"Issuing evaluate context command.");
1542 
1543 		/* Set up the input context flags for the command */
1544 		/* FIXME: This won't work if a non-default control endpoint
1545 		 * changes max packet sizes.
1546 		 */
1547 
1548 		command = xhci_alloc_command(xhci, true, mem_flags);
1549 		if (!command)
1550 			return -ENOMEM;
1551 
1552 		command->in_ctx = xhci->devs[slot_id]->in_ctx;
1553 		ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
1554 		if (!ctrl_ctx) {
1555 			xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1556 					__func__);
1557 			ret = -ENOMEM;
1558 			goto command_cleanup;
1559 		}
1560 		/* Set up the modified control endpoint 0 */
1561 		xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
1562 				xhci->devs[slot_id]->out_ctx, ep_index);
1563 
1564 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
1565 		ep_ctx->ep_info &= cpu_to_le32(~EP_STATE_MASK);/* must clear */
1566 		ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
1567 		ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
1568 
1569 		ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
1570 		ctrl_ctx->drop_flags = 0;
1571 
1572 		ret = xhci_configure_endpoint(xhci, urb->dev, command,
1573 				true, false);
1574 
1575 		/* Clean up the input context for later use by bandwidth
1576 		 * functions.
1577 		 */
1578 		ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
1579 command_cleanup:
1580 		kfree(command->completion);
1581 		kfree(command);
1582 	}
1583 	return ret;
1584 }
1585 
1586 /*
1587  * non-error returns are a promise to giveback() the urb later
1588  * we drop ownership so next owner (or urb unlink) can get it
1589  */
1590 static int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
1591 {
1592 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
1593 	unsigned long flags;
1594 	int ret = 0;
1595 	unsigned int slot_id, ep_index;
1596 	unsigned int *ep_state;
1597 	struct urb_priv	*urb_priv;
1598 	int num_tds;
1599 
1600 	if (!urb || xhci_check_args(hcd, urb->dev, urb->ep,
1601 					true, true, __func__) <= 0)
1602 		return -EINVAL;
1603 
1604 	slot_id = urb->dev->slot_id;
1605 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1606 	ep_state = &xhci->devs[slot_id]->eps[ep_index].ep_state;
1607 
1608 	if (!HCD_HW_ACCESSIBLE(hcd))
1609 		return -ESHUTDOWN;
1610 
1611 	if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR) {
1612 		xhci_dbg(xhci, "Can't queue urb, port error, link inactive\n");
1613 		return -ENODEV;
1614 	}
1615 
1616 	if (usb_endpoint_xfer_isoc(&urb->ep->desc))
1617 		num_tds = urb->number_of_packets;
1618 	else if (usb_endpoint_is_bulk_out(&urb->ep->desc) &&
1619 	    urb->transfer_buffer_length > 0 &&
1620 	    urb->transfer_flags & URB_ZERO_PACKET &&
1621 	    !(urb->transfer_buffer_length % usb_endpoint_maxp(&urb->ep->desc)))
1622 		num_tds = 2;
1623 	else
1624 		num_tds = 1;
1625 
1626 	urb_priv = kzalloc(struct_size(urb_priv, td, num_tds), mem_flags);
1627 	if (!urb_priv)
1628 		return -ENOMEM;
1629 
1630 	urb_priv->num_tds = num_tds;
1631 	urb_priv->num_tds_done = 0;
1632 	urb->hcpriv = urb_priv;
1633 
1634 	trace_xhci_urb_enqueue(urb);
1635 
1636 	if (usb_endpoint_xfer_control(&urb->ep->desc)) {
1637 		/* Check to see if the max packet size for the default control
1638 		 * endpoint changed during FS device enumeration
1639 		 */
1640 		if (urb->dev->speed == USB_SPEED_FULL) {
1641 			ret = xhci_check_maxpacket(xhci, slot_id,
1642 					ep_index, urb, mem_flags);
1643 			if (ret < 0) {
1644 				xhci_urb_free_priv(urb_priv);
1645 				urb->hcpriv = NULL;
1646 				return ret;
1647 			}
1648 		}
1649 	}
1650 
1651 	spin_lock_irqsave(&xhci->lock, flags);
1652 
1653 	if (xhci->xhc_state & XHCI_STATE_DYING) {
1654 		xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for non-responsive xHCI host.\n",
1655 			 urb->ep->desc.bEndpointAddress, urb);
1656 		ret = -ESHUTDOWN;
1657 		goto free_priv;
1658 	}
1659 	if (*ep_state & (EP_GETTING_STREAMS | EP_GETTING_NO_STREAMS)) {
1660 		xhci_warn(xhci, "WARN: Can't enqueue URB, ep in streams transition state %x\n",
1661 			  *ep_state);
1662 		ret = -EINVAL;
1663 		goto free_priv;
1664 	}
1665 	if (*ep_state & EP_SOFT_CLEAR_TOGGLE) {
1666 		xhci_warn(xhci, "Can't enqueue URB while manually clearing toggle\n");
1667 		ret = -EINVAL;
1668 		goto free_priv;
1669 	}
1670 
1671 	switch (usb_endpoint_type(&urb->ep->desc)) {
1672 
1673 	case USB_ENDPOINT_XFER_CONTROL:
1674 		ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
1675 					 slot_id, ep_index);
1676 		break;
1677 	case USB_ENDPOINT_XFER_BULK:
1678 		ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
1679 					 slot_id, ep_index);
1680 		break;
1681 	case USB_ENDPOINT_XFER_INT:
1682 		ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
1683 				slot_id, ep_index);
1684 		break;
1685 	case USB_ENDPOINT_XFER_ISOC:
1686 		ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
1687 				slot_id, ep_index);
1688 	}
1689 
1690 	if (ret) {
1691 free_priv:
1692 		xhci_urb_free_priv(urb_priv);
1693 		urb->hcpriv = NULL;
1694 	}
1695 	spin_unlock_irqrestore(&xhci->lock, flags);
1696 	return ret;
1697 }
1698 
1699 /*
1700  * Remove the URB's TD from the endpoint ring.  This may cause the HC to stop
1701  * USB transfers, potentially stopping in the middle of a TRB buffer.  The HC
1702  * should pick up where it left off in the TD, unless a Set Transfer Ring
1703  * Dequeue Pointer is issued.
1704  *
1705  * The TRBs that make up the buffers for the canceled URB will be "removed" from
1706  * the ring.  Since the ring is a contiguous structure, they can't be physically
1707  * removed.  Instead, there are two options:
1708  *
1709  *  1) If the HC is in the middle of processing the URB to be canceled, we
1710  *     simply move the ring's dequeue pointer past those TRBs using the Set
1711  *     Transfer Ring Dequeue Pointer command.  This will be the common case,
1712  *     when drivers timeout on the last submitted URB and attempt to cancel.
1713  *
1714  *  2) If the HC is in the middle of a different TD, we turn the TRBs into a
1715  *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
1716  *     HC will need to invalidate the any TRBs it has cached after the stop
1717  *     endpoint command, as noted in the xHCI 0.95 errata.
1718  *
1719  *  3) The TD may have completed by the time the Stop Endpoint Command
1720  *     completes, so software needs to handle that case too.
1721  *
1722  * This function should protect against the TD enqueueing code ringing the
1723  * doorbell while this code is waiting for a Stop Endpoint command to complete.
1724  * It also needs to account for multiple cancellations on happening at the same
1725  * time for the same endpoint.
1726  *
1727  * Note that this function can be called in any context, or so says
1728  * usb_hcd_unlink_urb()
1729  */
1730 static int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
1731 {
1732 	unsigned long flags;
1733 	int ret, i;
1734 	u32 temp;
1735 	struct xhci_hcd *xhci;
1736 	struct urb_priv	*urb_priv;
1737 	struct xhci_td *td;
1738 	unsigned int ep_index;
1739 	struct xhci_ring *ep_ring;
1740 	struct xhci_virt_ep *ep;
1741 	struct xhci_command *command;
1742 	struct xhci_virt_device *vdev;
1743 
1744 	xhci = hcd_to_xhci(hcd);
1745 	spin_lock_irqsave(&xhci->lock, flags);
1746 
1747 	trace_xhci_urb_dequeue(urb);
1748 
1749 	/* Make sure the URB hasn't completed or been unlinked already */
1750 	ret = usb_hcd_check_unlink_urb(hcd, urb, status);
1751 	if (ret)
1752 		goto done;
1753 
1754 	/* give back URB now if we can't queue it for cancel */
1755 	vdev = xhci->devs[urb->dev->slot_id];
1756 	urb_priv = urb->hcpriv;
1757 	if (!vdev || !urb_priv)
1758 		goto err_giveback;
1759 
1760 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1761 	ep = &vdev->eps[ep_index];
1762 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
1763 	if (!ep || !ep_ring)
1764 		goto err_giveback;
1765 
1766 	/* If xHC is dead take it down and return ALL URBs in xhci_hc_died() */
1767 	temp = readl(&xhci->op_regs->status);
1768 	if (temp == ~(u32)0 || xhci->xhc_state & XHCI_STATE_DYING) {
1769 		xhci_hc_died(xhci);
1770 		goto done;
1771 	}
1772 
1773 	/*
1774 	 * check ring is not re-allocated since URB was enqueued. If it is, then
1775 	 * make sure none of the ring related pointers in this URB private data
1776 	 * are touched, such as td_list, otherwise we overwrite freed data
1777 	 */
1778 	if (!td_on_ring(&urb_priv->td[0], ep_ring)) {
1779 		xhci_err(xhci, "Canceled URB td not found on endpoint ring");
1780 		for (i = urb_priv->num_tds_done; i < urb_priv->num_tds; i++) {
1781 			td = &urb_priv->td[i];
1782 			if (!list_empty(&td->cancelled_td_list))
1783 				list_del_init(&td->cancelled_td_list);
1784 		}
1785 		goto err_giveback;
1786 	}
1787 
1788 	if (xhci->xhc_state & XHCI_STATE_HALTED) {
1789 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1790 				"HC halted, freeing TD manually.");
1791 		for (i = urb_priv->num_tds_done;
1792 		     i < urb_priv->num_tds;
1793 		     i++) {
1794 			td = &urb_priv->td[i];
1795 			if (!list_empty(&td->td_list))
1796 				list_del_init(&td->td_list);
1797 			if (!list_empty(&td->cancelled_td_list))
1798 				list_del_init(&td->cancelled_td_list);
1799 		}
1800 		goto err_giveback;
1801 	}
1802 
1803 	i = urb_priv->num_tds_done;
1804 	if (i < urb_priv->num_tds)
1805 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1806 				"Cancel URB %p, dev %s, ep 0x%x, "
1807 				"starting at offset 0x%llx",
1808 				urb, urb->dev->devpath,
1809 				urb->ep->desc.bEndpointAddress,
1810 				(unsigned long long) xhci_trb_virt_to_dma(
1811 					urb_priv->td[i].start_seg,
1812 					urb_priv->td[i].first_trb));
1813 
1814 	for (; i < urb_priv->num_tds; i++) {
1815 		td = &urb_priv->td[i];
1816 		/* TD can already be on cancelled list if ep halted on it */
1817 		if (list_empty(&td->cancelled_td_list)) {
1818 			td->cancel_status = TD_DIRTY;
1819 			list_add_tail(&td->cancelled_td_list,
1820 				      &ep->cancelled_td_list);
1821 		}
1822 	}
1823 
1824 	/* Queue a stop endpoint command, but only if this is
1825 	 * the first cancellation to be handled.
1826 	 */
1827 	if (!(ep->ep_state & EP_STOP_CMD_PENDING)) {
1828 		command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
1829 		if (!command) {
1830 			ret = -ENOMEM;
1831 			goto done;
1832 		}
1833 		ep->ep_state |= EP_STOP_CMD_PENDING;
1834 		ep->stop_cmd_timer.expires = jiffies +
1835 			XHCI_STOP_EP_CMD_TIMEOUT * HZ;
1836 		add_timer(&ep->stop_cmd_timer);
1837 		xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id,
1838 					 ep_index, 0);
1839 		xhci_ring_cmd_db(xhci);
1840 	}
1841 done:
1842 	spin_unlock_irqrestore(&xhci->lock, flags);
1843 	return ret;
1844 
1845 err_giveback:
1846 	if (urb_priv)
1847 		xhci_urb_free_priv(urb_priv);
1848 	usb_hcd_unlink_urb_from_ep(hcd, urb);
1849 	spin_unlock_irqrestore(&xhci->lock, flags);
1850 	usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
1851 	return ret;
1852 }
1853 
1854 /* Drop an endpoint from a new bandwidth configuration for this device.
1855  * Only one call to this function is allowed per endpoint before
1856  * check_bandwidth() or reset_bandwidth() must be called.
1857  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1858  * add the endpoint to the schedule with possibly new parameters denoted by a
1859  * different endpoint descriptor in usb_host_endpoint.
1860  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1861  * not allowed.
1862  *
1863  * The USB core will not allow URBs to be queued to an endpoint that is being
1864  * disabled, so there's no need for mutual exclusion to protect
1865  * the xhci->devs[slot_id] structure.
1866  */
1867 int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1868 		       struct usb_host_endpoint *ep)
1869 {
1870 	struct xhci_hcd *xhci;
1871 	struct xhci_container_ctx *in_ctx, *out_ctx;
1872 	struct xhci_input_control_ctx *ctrl_ctx;
1873 	unsigned int ep_index;
1874 	struct xhci_ep_ctx *ep_ctx;
1875 	u32 drop_flag;
1876 	u32 new_add_flags, new_drop_flags;
1877 	int ret;
1878 
1879 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1880 	if (ret <= 0)
1881 		return ret;
1882 	xhci = hcd_to_xhci(hcd);
1883 	if (xhci->xhc_state & XHCI_STATE_DYING)
1884 		return -ENODEV;
1885 
1886 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1887 	drop_flag = xhci_get_endpoint_flag(&ep->desc);
1888 	if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
1889 		xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
1890 				__func__, drop_flag);
1891 		return 0;
1892 	}
1893 
1894 	in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1895 	out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1896 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1897 	if (!ctrl_ctx) {
1898 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1899 				__func__);
1900 		return 0;
1901 	}
1902 
1903 	ep_index = xhci_get_endpoint_index(&ep->desc);
1904 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1905 	/* If the HC already knows the endpoint is disabled,
1906 	 * or the HCD has noted it is disabled, ignore this request
1907 	 */
1908 	if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) ||
1909 	    le32_to_cpu(ctrl_ctx->drop_flags) &
1910 	    xhci_get_endpoint_flag(&ep->desc)) {
1911 		/* Do not warn when called after a usb_device_reset */
1912 		if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)
1913 			xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
1914 				  __func__, ep);
1915 		return 0;
1916 	}
1917 
1918 	ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
1919 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1920 
1921 	ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
1922 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1923 
1924 	xhci_debugfs_remove_endpoint(xhci, xhci->devs[udev->slot_id], ep_index);
1925 
1926 	xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);
1927 
1928 	xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1929 			(unsigned int) ep->desc.bEndpointAddress,
1930 			udev->slot_id,
1931 			(unsigned int) new_drop_flags,
1932 			(unsigned int) new_add_flags);
1933 	return 0;
1934 }
1935 EXPORT_SYMBOL_GPL(xhci_drop_endpoint);
1936 
1937 /* Add an endpoint to a new possible bandwidth configuration for this device.
1938  * Only one call to this function is allowed per endpoint before
1939  * check_bandwidth() or reset_bandwidth() must be called.
1940  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1941  * add the endpoint to the schedule with possibly new parameters denoted by a
1942  * different endpoint descriptor in usb_host_endpoint.
1943  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1944  * not allowed.
1945  *
1946  * The USB core will not allow URBs to be queued to an endpoint until the
1947  * configuration or alt setting is installed in the device, so there's no need
1948  * for mutual exclusion to protect the xhci->devs[slot_id] structure.
1949  */
1950 int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1951 		      struct usb_host_endpoint *ep)
1952 {
1953 	struct xhci_hcd *xhci;
1954 	struct xhci_container_ctx *in_ctx;
1955 	unsigned int ep_index;
1956 	struct xhci_input_control_ctx *ctrl_ctx;
1957 	struct xhci_ep_ctx *ep_ctx;
1958 	u32 added_ctxs;
1959 	u32 new_add_flags, new_drop_flags;
1960 	struct xhci_virt_device *virt_dev;
1961 	int ret = 0;
1962 
1963 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1964 	if (ret <= 0) {
1965 		/* So we won't queue a reset ep command for a root hub */
1966 		ep->hcpriv = NULL;
1967 		return ret;
1968 	}
1969 	xhci = hcd_to_xhci(hcd);
1970 	if (xhci->xhc_state & XHCI_STATE_DYING)
1971 		return -ENODEV;
1972 
1973 	added_ctxs = xhci_get_endpoint_flag(&ep->desc);
1974 	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
1975 		/* FIXME when we have to issue an evaluate endpoint command to
1976 		 * deal with ep0 max packet size changing once we get the
1977 		 * descriptors
1978 		 */
1979 		xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
1980 				__func__, added_ctxs);
1981 		return 0;
1982 	}
1983 
1984 	virt_dev = xhci->devs[udev->slot_id];
1985 	in_ctx = virt_dev->in_ctx;
1986 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1987 	if (!ctrl_ctx) {
1988 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1989 				__func__);
1990 		return 0;
1991 	}
1992 
1993 	ep_index = xhci_get_endpoint_index(&ep->desc);
1994 	/* If this endpoint is already in use, and the upper layers are trying
1995 	 * to add it again without dropping it, reject the addition.
1996 	 */
1997 	if (virt_dev->eps[ep_index].ring &&
1998 			!(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {
1999 		xhci_warn(xhci, "Trying to add endpoint 0x%x "
2000 				"without dropping it.\n",
2001 				(unsigned int) ep->desc.bEndpointAddress);
2002 		return -EINVAL;
2003 	}
2004 
2005 	/* If the HCD has already noted the endpoint is enabled,
2006 	 * ignore this request.
2007 	 */
2008 	if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {
2009 		xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
2010 				__func__, ep);
2011 		return 0;
2012 	}
2013 
2014 	/*
2015 	 * Configuration and alternate setting changes must be done in
2016 	 * process context, not interrupt context (or so documenation
2017 	 * for usb_set_interface() and usb_set_configuration() claim).
2018 	 */
2019 	if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
2020 		dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
2021 				__func__, ep->desc.bEndpointAddress);
2022 		return -ENOMEM;
2023 	}
2024 
2025 	ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);
2026 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
2027 
2028 	/* If xhci_endpoint_disable() was called for this endpoint, but the
2029 	 * xHC hasn't been notified yet through the check_bandwidth() call,
2030 	 * this re-adds a new state for the endpoint from the new endpoint
2031 	 * descriptors.  We must drop and re-add this endpoint, so we leave the
2032 	 * drop flags alone.
2033 	 */
2034 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
2035 
2036 	/* Store the usb_device pointer for later use */
2037 	ep->hcpriv = udev;
2038 
2039 	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
2040 	trace_xhci_add_endpoint(ep_ctx);
2041 
2042 	xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
2043 			(unsigned int) ep->desc.bEndpointAddress,
2044 			udev->slot_id,
2045 			(unsigned int) new_drop_flags,
2046 			(unsigned int) new_add_flags);
2047 	return 0;
2048 }
2049 EXPORT_SYMBOL_GPL(xhci_add_endpoint);
2050 
2051 static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
2052 {
2053 	struct xhci_input_control_ctx *ctrl_ctx;
2054 	struct xhci_ep_ctx *ep_ctx;
2055 	struct xhci_slot_ctx *slot_ctx;
2056 	int i;
2057 
2058 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
2059 	if (!ctrl_ctx) {
2060 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2061 				__func__);
2062 		return;
2063 	}
2064 
2065 	/* When a device's add flag and drop flag are zero, any subsequent
2066 	 * configure endpoint command will leave that endpoint's state
2067 	 * untouched.  Make sure we don't leave any old state in the input
2068 	 * endpoint contexts.
2069 	 */
2070 	ctrl_ctx->drop_flags = 0;
2071 	ctrl_ctx->add_flags = 0;
2072 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2073 	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
2074 	/* Endpoint 0 is always valid */
2075 	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
2076 	for (i = 1; i < 31; i++) {
2077 		ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
2078 		ep_ctx->ep_info = 0;
2079 		ep_ctx->ep_info2 = 0;
2080 		ep_ctx->deq = 0;
2081 		ep_ctx->tx_info = 0;
2082 	}
2083 }
2084 
2085 static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
2086 		struct usb_device *udev, u32 *cmd_status)
2087 {
2088 	int ret;
2089 
2090 	switch (*cmd_status) {
2091 	case COMP_COMMAND_ABORTED:
2092 	case COMP_COMMAND_RING_STOPPED:
2093 		xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n");
2094 		ret = -ETIME;
2095 		break;
2096 	case COMP_RESOURCE_ERROR:
2097 		dev_warn(&udev->dev,
2098 			 "Not enough host controller resources for new device state.\n");
2099 		ret = -ENOMEM;
2100 		/* FIXME: can we allocate more resources for the HC? */
2101 		break;
2102 	case COMP_BANDWIDTH_ERROR:
2103 	case COMP_SECONDARY_BANDWIDTH_ERROR:
2104 		dev_warn(&udev->dev,
2105 			 "Not enough bandwidth for new device state.\n");
2106 		ret = -ENOSPC;
2107 		/* FIXME: can we go back to the old state? */
2108 		break;
2109 	case COMP_TRB_ERROR:
2110 		/* the HCD set up something wrong */
2111 		dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
2112 				"add flag = 1, "
2113 				"and endpoint is not disabled.\n");
2114 		ret = -EINVAL;
2115 		break;
2116 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
2117 		dev_warn(&udev->dev,
2118 			 "ERROR: Incompatible device for endpoint configure command.\n");
2119 		ret = -ENODEV;
2120 		break;
2121 	case COMP_SUCCESS:
2122 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
2123 				"Successful Endpoint Configure command");
2124 		ret = 0;
2125 		break;
2126 	default:
2127 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
2128 				*cmd_status);
2129 		ret = -EINVAL;
2130 		break;
2131 	}
2132 	return ret;
2133 }
2134 
2135 static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
2136 		struct usb_device *udev, u32 *cmd_status)
2137 {
2138 	int ret;
2139 
2140 	switch (*cmd_status) {
2141 	case COMP_COMMAND_ABORTED:
2142 	case COMP_COMMAND_RING_STOPPED:
2143 		xhci_warn(xhci, "Timeout while waiting for evaluate context command\n");
2144 		ret = -ETIME;
2145 		break;
2146 	case COMP_PARAMETER_ERROR:
2147 		dev_warn(&udev->dev,
2148 			 "WARN: xHCI driver setup invalid evaluate context command.\n");
2149 		ret = -EINVAL;
2150 		break;
2151 	case COMP_SLOT_NOT_ENABLED_ERROR:
2152 		dev_warn(&udev->dev,
2153 			"WARN: slot not enabled for evaluate context command.\n");
2154 		ret = -EINVAL;
2155 		break;
2156 	case COMP_CONTEXT_STATE_ERROR:
2157 		dev_warn(&udev->dev,
2158 			"WARN: invalid context state for evaluate context command.\n");
2159 		ret = -EINVAL;
2160 		break;
2161 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
2162 		dev_warn(&udev->dev,
2163 			"ERROR: Incompatible device for evaluate context command.\n");
2164 		ret = -ENODEV;
2165 		break;
2166 	case COMP_MAX_EXIT_LATENCY_TOO_LARGE_ERROR:
2167 		/* Max Exit Latency too large error */
2168 		dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");
2169 		ret = -EINVAL;
2170 		break;
2171 	case COMP_SUCCESS:
2172 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
2173 				"Successful evaluate context command");
2174 		ret = 0;
2175 		break;
2176 	default:
2177 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
2178 			*cmd_status);
2179 		ret = -EINVAL;
2180 		break;
2181 	}
2182 	return ret;
2183 }
2184 
2185 static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
2186 		struct xhci_input_control_ctx *ctrl_ctx)
2187 {
2188 	u32 valid_add_flags;
2189 	u32 valid_drop_flags;
2190 
2191 	/* Ignore the slot flag (bit 0), and the default control endpoint flag
2192 	 * (bit 1).  The default control endpoint is added during the Address
2193 	 * Device command and is never removed until the slot is disabled.
2194 	 */
2195 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2196 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2197 
2198 	/* Use hweight32 to count the number of ones in the add flags, or
2199 	 * number of endpoints added.  Don't count endpoints that are changed
2200 	 * (both added and dropped).
2201 	 */
2202 	return hweight32(valid_add_flags) -
2203 		hweight32(valid_add_flags & valid_drop_flags);
2204 }
2205 
2206 static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
2207 		struct xhci_input_control_ctx *ctrl_ctx)
2208 {
2209 	u32 valid_add_flags;
2210 	u32 valid_drop_flags;
2211 
2212 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
2213 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
2214 
2215 	return hweight32(valid_drop_flags) -
2216 		hweight32(valid_add_flags & valid_drop_flags);
2217 }
2218 
2219 /*
2220  * We need to reserve the new number of endpoints before the configure endpoint
2221  * command completes.  We can't subtract the dropped endpoints from the number
2222  * of active endpoints until the command completes because we can oversubscribe
2223  * the host in this case:
2224  *
2225  *  - the first configure endpoint command drops more endpoints than it adds
2226  *  - a second configure endpoint command that adds more endpoints is queued
2227  *  - the first configure endpoint command fails, so the config is unchanged
2228  *  - the second command may succeed, even though there isn't enough resources
2229  *
2230  * Must be called with xhci->lock held.
2231  */
2232 static int xhci_reserve_host_resources(struct xhci_hcd *xhci,
2233 		struct xhci_input_control_ctx *ctrl_ctx)
2234 {
2235 	u32 added_eps;
2236 
2237 	added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2238 	if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {
2239 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2240 				"Not enough ep ctxs: "
2241 				"%u active, need to add %u, limit is %u.",
2242 				xhci->num_active_eps, added_eps,
2243 				xhci->limit_active_eps);
2244 		return -ENOMEM;
2245 	}
2246 	xhci->num_active_eps += added_eps;
2247 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2248 			"Adding %u ep ctxs, %u now active.", added_eps,
2249 			xhci->num_active_eps);
2250 	return 0;
2251 }
2252 
2253 /*
2254  * The configure endpoint was failed by the xHC for some other reason, so we
2255  * need to revert the resources that failed configuration would have used.
2256  *
2257  * Must be called with xhci->lock held.
2258  */
2259 static void xhci_free_host_resources(struct xhci_hcd *xhci,
2260 		struct xhci_input_control_ctx *ctrl_ctx)
2261 {
2262 	u32 num_failed_eps;
2263 
2264 	num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2265 	xhci->num_active_eps -= num_failed_eps;
2266 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2267 			"Removing %u failed ep ctxs, %u now active.",
2268 			num_failed_eps,
2269 			xhci->num_active_eps);
2270 }
2271 
2272 /*
2273  * Now that the command has completed, clean up the active endpoint count by
2274  * subtracting out the endpoints that were dropped (but not changed).
2275  *
2276  * Must be called with xhci->lock held.
2277  */
2278 static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,
2279 		struct xhci_input_control_ctx *ctrl_ctx)
2280 {
2281 	u32 num_dropped_eps;
2282 
2283 	num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx);
2284 	xhci->num_active_eps -= num_dropped_eps;
2285 	if (num_dropped_eps)
2286 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2287 				"Removing %u dropped ep ctxs, %u now active.",
2288 				num_dropped_eps,
2289 				xhci->num_active_eps);
2290 }
2291 
2292 static unsigned int xhci_get_block_size(struct usb_device *udev)
2293 {
2294 	switch (udev->speed) {
2295 	case USB_SPEED_LOW:
2296 	case USB_SPEED_FULL:
2297 		return FS_BLOCK;
2298 	case USB_SPEED_HIGH:
2299 		return HS_BLOCK;
2300 	case USB_SPEED_SUPER:
2301 	case USB_SPEED_SUPER_PLUS:
2302 		return SS_BLOCK;
2303 	case USB_SPEED_UNKNOWN:
2304 	case USB_SPEED_WIRELESS:
2305 	default:
2306 		/* Should never happen */
2307 		return 1;
2308 	}
2309 }
2310 
2311 static unsigned int
2312 xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw)
2313 {
2314 	if (interval_bw->overhead[LS_OVERHEAD_TYPE])
2315 		return LS_OVERHEAD;
2316 	if (interval_bw->overhead[FS_OVERHEAD_TYPE])
2317 		return FS_OVERHEAD;
2318 	return HS_OVERHEAD;
2319 }
2320 
2321 /* If we are changing a LS/FS device under a HS hub,
2322  * make sure (if we are activating a new TT) that the HS bus has enough
2323  * bandwidth for this new TT.
2324  */
2325 static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,
2326 		struct xhci_virt_device *virt_dev,
2327 		int old_active_eps)
2328 {
2329 	struct xhci_interval_bw_table *bw_table;
2330 	struct xhci_tt_bw_info *tt_info;
2331 
2332 	/* Find the bandwidth table for the root port this TT is attached to. */
2333 	bw_table = &xhci->rh_bw[virt_dev->real_port - 1].bw_table;
2334 	tt_info = virt_dev->tt_info;
2335 	/* If this TT already had active endpoints, the bandwidth for this TT
2336 	 * has already been added.  Removing all periodic endpoints (and thus
2337 	 * making the TT enactive) will only decrease the bandwidth used.
2338 	 */
2339 	if (old_active_eps)
2340 		return 0;
2341 	if (old_active_eps == 0 && tt_info->active_eps != 0) {
2342 		if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)
2343 			return -ENOMEM;
2344 		return 0;
2345 	}
2346 	/* Not sure why we would have no new active endpoints...
2347 	 *
2348 	 * Maybe because of an Evaluate Context change for a hub update or a
2349 	 * control endpoint 0 max packet size change?
2350 	 * FIXME: skip the bandwidth calculation in that case.
2351 	 */
2352 	return 0;
2353 }
2354 
2355 static int xhci_check_ss_bw(struct xhci_hcd *xhci,
2356 		struct xhci_virt_device *virt_dev)
2357 {
2358 	unsigned int bw_reserved;
2359 
2360 	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100);
2361 	if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))
2362 		return -ENOMEM;
2363 
2364 	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100);
2365 	if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))
2366 		return -ENOMEM;
2367 
2368 	return 0;
2369 }
2370 
2371 /*
2372  * This algorithm is a very conservative estimate of the worst-case scheduling
2373  * scenario for any one interval.  The hardware dynamically schedules the
2374  * packets, so we can't tell which microframe could be the limiting factor in
2375  * the bandwidth scheduling.  This only takes into account periodic endpoints.
2376  *
2377  * Obviously, we can't solve an NP complete problem to find the minimum worst
2378  * case scenario.  Instead, we come up with an estimate that is no less than
2379  * the worst case bandwidth used for any one microframe, but may be an
2380  * over-estimate.
2381  *
2382  * We walk the requirements for each endpoint by interval, starting with the
2383  * smallest interval, and place packets in the schedule where there is only one
2384  * possible way to schedule packets for that interval.  In order to simplify
2385  * this algorithm, we record the largest max packet size for each interval, and
2386  * assume all packets will be that size.
2387  *
2388  * For interval 0, we obviously must schedule all packets for each interval.
2389  * The bandwidth for interval 0 is just the amount of data to be transmitted
2390  * (the sum of all max ESIT payload sizes, plus any overhead per packet times
2391  * the number of packets).
2392  *
2393  * For interval 1, we have two possible microframes to schedule those packets
2394  * in.  For this algorithm, if we can schedule the same number of packets for
2395  * each possible scheduling opportunity (each microframe), we will do so.  The
2396  * remaining number of packets will be saved to be transmitted in the gaps in
2397  * the next interval's scheduling sequence.
2398  *
2399  * As we move those remaining packets to be scheduled with interval 2 packets,
2400  * we have to double the number of remaining packets to transmit.  This is
2401  * because the intervals are actually powers of 2, and we would be transmitting
2402  * the previous interval's packets twice in this interval.  We also have to be
2403  * sure that when we look at the largest max packet size for this interval, we
2404  * also look at the largest max packet size for the remaining packets and take
2405  * the greater of the two.
2406  *
2407  * The algorithm continues to evenly distribute packets in each scheduling
2408  * opportunity, and push the remaining packets out, until we get to the last
2409  * interval.  Then those packets and their associated overhead are just added
2410  * to the bandwidth used.
2411  */
2412 static int xhci_check_bw_table(struct xhci_hcd *xhci,
2413 		struct xhci_virt_device *virt_dev,
2414 		int old_active_eps)
2415 {
2416 	unsigned int bw_reserved;
2417 	unsigned int max_bandwidth;
2418 	unsigned int bw_used;
2419 	unsigned int block_size;
2420 	struct xhci_interval_bw_table *bw_table;
2421 	unsigned int packet_size = 0;
2422 	unsigned int overhead = 0;
2423 	unsigned int packets_transmitted = 0;
2424 	unsigned int packets_remaining = 0;
2425 	unsigned int i;
2426 
2427 	if (virt_dev->udev->speed >= USB_SPEED_SUPER)
2428 		return xhci_check_ss_bw(xhci, virt_dev);
2429 
2430 	if (virt_dev->udev->speed == USB_SPEED_HIGH) {
2431 		max_bandwidth = HS_BW_LIMIT;
2432 		/* Convert percent of bus BW reserved to blocks reserved */
2433 		bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100);
2434 	} else {
2435 		max_bandwidth = FS_BW_LIMIT;
2436 		bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100);
2437 	}
2438 
2439 	bw_table = virt_dev->bw_table;
2440 	/* We need to translate the max packet size and max ESIT payloads into
2441 	 * the units the hardware uses.
2442 	 */
2443 	block_size = xhci_get_block_size(virt_dev->udev);
2444 
2445 	/* If we are manipulating a LS/FS device under a HS hub, double check
2446 	 * that the HS bus has enough bandwidth if we are activing a new TT.
2447 	 */
2448 	if (virt_dev->tt_info) {
2449 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2450 				"Recalculating BW for rootport %u",
2451 				virt_dev->real_port);
2452 		if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {
2453 			xhci_warn(xhci, "Not enough bandwidth on HS bus for "
2454 					"newly activated TT.\n");
2455 			return -ENOMEM;
2456 		}
2457 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2458 				"Recalculating BW for TT slot %u port %u",
2459 				virt_dev->tt_info->slot_id,
2460 				virt_dev->tt_info->ttport);
2461 	} else {
2462 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2463 				"Recalculating BW for rootport %u",
2464 				virt_dev->real_port);
2465 	}
2466 
2467 	/* Add in how much bandwidth will be used for interval zero, or the
2468 	 * rounded max ESIT payload + number of packets * largest overhead.
2469 	 */
2470 	bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +
2471 		bw_table->interval_bw[0].num_packets *
2472 		xhci_get_largest_overhead(&bw_table->interval_bw[0]);
2473 
2474 	for (i = 1; i < XHCI_MAX_INTERVAL; i++) {
2475 		unsigned int bw_added;
2476 		unsigned int largest_mps;
2477 		unsigned int interval_overhead;
2478 
2479 		/*
2480 		 * How many packets could we transmit in this interval?
2481 		 * If packets didn't fit in the previous interval, we will need
2482 		 * to transmit that many packets twice within this interval.
2483 		 */
2484 		packets_remaining = 2 * packets_remaining +
2485 			bw_table->interval_bw[i].num_packets;
2486 
2487 		/* Find the largest max packet size of this or the previous
2488 		 * interval.
2489 		 */
2490 		if (list_empty(&bw_table->interval_bw[i].endpoints))
2491 			largest_mps = 0;
2492 		else {
2493 			struct xhci_virt_ep *virt_ep;
2494 			struct list_head *ep_entry;
2495 
2496 			ep_entry = bw_table->interval_bw[i].endpoints.next;
2497 			virt_ep = list_entry(ep_entry,
2498 					struct xhci_virt_ep, bw_endpoint_list);
2499 			/* Convert to blocks, rounding up */
2500 			largest_mps = DIV_ROUND_UP(
2501 					virt_ep->bw_info.max_packet_size,
2502 					block_size);
2503 		}
2504 		if (largest_mps > packet_size)
2505 			packet_size = largest_mps;
2506 
2507 		/* Use the larger overhead of this or the previous interval. */
2508 		interval_overhead = xhci_get_largest_overhead(
2509 				&bw_table->interval_bw[i]);
2510 		if (interval_overhead > overhead)
2511 			overhead = interval_overhead;
2512 
2513 		/* How many packets can we evenly distribute across
2514 		 * (1 << (i + 1)) possible scheduling opportunities?
2515 		 */
2516 		packets_transmitted = packets_remaining >> (i + 1);
2517 
2518 		/* Add in the bandwidth used for those scheduled packets */
2519 		bw_added = packets_transmitted * (overhead + packet_size);
2520 
2521 		/* How many packets do we have remaining to transmit? */
2522 		packets_remaining = packets_remaining % (1 << (i + 1));
2523 
2524 		/* What largest max packet size should those packets have? */
2525 		/* If we've transmitted all packets, don't carry over the
2526 		 * largest packet size.
2527 		 */
2528 		if (packets_remaining == 0) {
2529 			packet_size = 0;
2530 			overhead = 0;
2531 		} else if (packets_transmitted > 0) {
2532 			/* Otherwise if we do have remaining packets, and we've
2533 			 * scheduled some packets in this interval, take the
2534 			 * largest max packet size from endpoints with this
2535 			 * interval.
2536 			 */
2537 			packet_size = largest_mps;
2538 			overhead = interval_overhead;
2539 		}
2540 		/* Otherwise carry over packet_size and overhead from the last
2541 		 * time we had a remainder.
2542 		 */
2543 		bw_used += bw_added;
2544 		if (bw_used > max_bandwidth) {
2545 			xhci_warn(xhci, "Not enough bandwidth. "
2546 					"Proposed: %u, Max: %u\n",
2547 				bw_used, max_bandwidth);
2548 			return -ENOMEM;
2549 		}
2550 	}
2551 	/*
2552 	 * Ok, we know we have some packets left over after even-handedly
2553 	 * scheduling interval 15.  We don't know which microframes they will
2554 	 * fit into, so we over-schedule and say they will be scheduled every
2555 	 * microframe.
2556 	 */
2557 	if (packets_remaining > 0)
2558 		bw_used += overhead + packet_size;
2559 
2560 	if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {
2561 		unsigned int port_index = virt_dev->real_port - 1;
2562 
2563 		/* OK, we're manipulating a HS device attached to a
2564 		 * root port bandwidth domain.  Include the number of active TTs
2565 		 * in the bandwidth used.
2566 		 */
2567 		bw_used += TT_HS_OVERHEAD *
2568 			xhci->rh_bw[port_index].num_active_tts;
2569 	}
2570 
2571 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2572 		"Final bandwidth: %u, Limit: %u, Reserved: %u, "
2573 		"Available: %u " "percent",
2574 		bw_used, max_bandwidth, bw_reserved,
2575 		(max_bandwidth - bw_used - bw_reserved) * 100 /
2576 		max_bandwidth);
2577 
2578 	bw_used += bw_reserved;
2579 	if (bw_used > max_bandwidth) {
2580 		xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",
2581 				bw_used, max_bandwidth);
2582 		return -ENOMEM;
2583 	}
2584 
2585 	bw_table->bw_used = bw_used;
2586 	return 0;
2587 }
2588 
2589 static bool xhci_is_async_ep(unsigned int ep_type)
2590 {
2591 	return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
2592 					ep_type != ISOC_IN_EP &&
2593 					ep_type != INT_IN_EP);
2594 }
2595 
2596 static bool xhci_is_sync_in_ep(unsigned int ep_type)
2597 {
2598 	return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP);
2599 }
2600 
2601 static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw)
2602 {
2603 	unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);
2604 
2605 	if (ep_bw->ep_interval == 0)
2606 		return SS_OVERHEAD_BURST +
2607 			(ep_bw->mult * ep_bw->num_packets *
2608 					(SS_OVERHEAD + mps));
2609 	return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *
2610 				(SS_OVERHEAD + mps + SS_OVERHEAD_BURST),
2611 				1 << ep_bw->ep_interval);
2612 
2613 }
2614 
2615 static void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,
2616 		struct xhci_bw_info *ep_bw,
2617 		struct xhci_interval_bw_table *bw_table,
2618 		struct usb_device *udev,
2619 		struct xhci_virt_ep *virt_ep,
2620 		struct xhci_tt_bw_info *tt_info)
2621 {
2622 	struct xhci_interval_bw	*interval_bw;
2623 	int normalized_interval;
2624 
2625 	if (xhci_is_async_ep(ep_bw->type))
2626 		return;
2627 
2628 	if (udev->speed >= USB_SPEED_SUPER) {
2629 		if (xhci_is_sync_in_ep(ep_bw->type))
2630 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=
2631 				xhci_get_ss_bw_consumed(ep_bw);
2632 		else
2633 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=
2634 				xhci_get_ss_bw_consumed(ep_bw);
2635 		return;
2636 	}
2637 
2638 	/* SuperSpeed endpoints never get added to intervals in the table, so
2639 	 * this check is only valid for HS/FS/LS devices.
2640 	 */
2641 	if (list_empty(&virt_ep->bw_endpoint_list))
2642 		return;
2643 	/* For LS/FS devices, we need to translate the interval expressed in
2644 	 * microframes to frames.
2645 	 */
2646 	if (udev->speed == USB_SPEED_HIGH)
2647 		normalized_interval = ep_bw->ep_interval;
2648 	else
2649 		normalized_interval = ep_bw->ep_interval - 3;
2650 
2651 	if (normalized_interval == 0)
2652 		bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;
2653 	interval_bw = &bw_table->interval_bw[normalized_interval];
2654 	interval_bw->num_packets -= ep_bw->num_packets;
2655 	switch (udev->speed) {
2656 	case USB_SPEED_LOW:
2657 		interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;
2658 		break;
2659 	case USB_SPEED_FULL:
2660 		interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;
2661 		break;
2662 	case USB_SPEED_HIGH:
2663 		interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;
2664 		break;
2665 	case USB_SPEED_SUPER:
2666 	case USB_SPEED_SUPER_PLUS:
2667 	case USB_SPEED_UNKNOWN:
2668 	case USB_SPEED_WIRELESS:
2669 		/* Should never happen because only LS/FS/HS endpoints will get
2670 		 * added to the endpoint list.
2671 		 */
2672 		return;
2673 	}
2674 	if (tt_info)
2675 		tt_info->active_eps -= 1;
2676 	list_del_init(&virt_ep->bw_endpoint_list);
2677 }
2678 
2679 static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,
2680 		struct xhci_bw_info *ep_bw,
2681 		struct xhci_interval_bw_table *bw_table,
2682 		struct usb_device *udev,
2683 		struct xhci_virt_ep *virt_ep,
2684 		struct xhci_tt_bw_info *tt_info)
2685 {
2686 	struct xhci_interval_bw	*interval_bw;
2687 	struct xhci_virt_ep *smaller_ep;
2688 	int normalized_interval;
2689 
2690 	if (xhci_is_async_ep(ep_bw->type))
2691 		return;
2692 
2693 	if (udev->speed == USB_SPEED_SUPER) {
2694 		if (xhci_is_sync_in_ep(ep_bw->type))
2695 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=
2696 				xhci_get_ss_bw_consumed(ep_bw);
2697 		else
2698 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=
2699 				xhci_get_ss_bw_consumed(ep_bw);
2700 		return;
2701 	}
2702 
2703 	/* For LS/FS devices, we need to translate the interval expressed in
2704 	 * microframes to frames.
2705 	 */
2706 	if (udev->speed == USB_SPEED_HIGH)
2707 		normalized_interval = ep_bw->ep_interval;
2708 	else
2709 		normalized_interval = ep_bw->ep_interval - 3;
2710 
2711 	if (normalized_interval == 0)
2712 		bw_table->interval0_esit_payload += ep_bw->max_esit_payload;
2713 	interval_bw = &bw_table->interval_bw[normalized_interval];
2714 	interval_bw->num_packets += ep_bw->num_packets;
2715 	switch (udev->speed) {
2716 	case USB_SPEED_LOW:
2717 		interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;
2718 		break;
2719 	case USB_SPEED_FULL:
2720 		interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;
2721 		break;
2722 	case USB_SPEED_HIGH:
2723 		interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;
2724 		break;
2725 	case USB_SPEED_SUPER:
2726 	case USB_SPEED_SUPER_PLUS:
2727 	case USB_SPEED_UNKNOWN:
2728 	case USB_SPEED_WIRELESS:
2729 		/* Should never happen because only LS/FS/HS endpoints will get
2730 		 * added to the endpoint list.
2731 		 */
2732 		return;
2733 	}
2734 
2735 	if (tt_info)
2736 		tt_info->active_eps += 1;
2737 	/* Insert the endpoint into the list, largest max packet size first. */
2738 	list_for_each_entry(smaller_ep, &interval_bw->endpoints,
2739 			bw_endpoint_list) {
2740 		if (ep_bw->max_packet_size >=
2741 				smaller_ep->bw_info.max_packet_size) {
2742 			/* Add the new ep before the smaller endpoint */
2743 			list_add_tail(&virt_ep->bw_endpoint_list,
2744 					&smaller_ep->bw_endpoint_list);
2745 			return;
2746 		}
2747 	}
2748 	/* Add the new endpoint at the end of the list. */
2749 	list_add_tail(&virt_ep->bw_endpoint_list,
2750 			&interval_bw->endpoints);
2751 }
2752 
2753 void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
2754 		struct xhci_virt_device *virt_dev,
2755 		int old_active_eps)
2756 {
2757 	struct xhci_root_port_bw_info *rh_bw_info;
2758 	if (!virt_dev->tt_info)
2759 		return;
2760 
2761 	rh_bw_info = &xhci->rh_bw[virt_dev->real_port - 1];
2762 	if (old_active_eps == 0 &&
2763 				virt_dev->tt_info->active_eps != 0) {
2764 		rh_bw_info->num_active_tts += 1;
2765 		rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;
2766 	} else if (old_active_eps != 0 &&
2767 				virt_dev->tt_info->active_eps == 0) {
2768 		rh_bw_info->num_active_tts -= 1;
2769 		rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;
2770 	}
2771 }
2772 
2773 static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
2774 		struct xhci_virt_device *virt_dev,
2775 		struct xhci_container_ctx *in_ctx)
2776 {
2777 	struct xhci_bw_info ep_bw_info[31];
2778 	int i;
2779 	struct xhci_input_control_ctx *ctrl_ctx;
2780 	int old_active_eps = 0;
2781 
2782 	if (virt_dev->tt_info)
2783 		old_active_eps = virt_dev->tt_info->active_eps;
2784 
2785 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2786 	if (!ctrl_ctx) {
2787 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2788 				__func__);
2789 		return -ENOMEM;
2790 	}
2791 
2792 	for (i = 0; i < 31; i++) {
2793 		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2794 			continue;
2795 
2796 		/* Make a copy of the BW info in case we need to revert this */
2797 		memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
2798 				sizeof(ep_bw_info[i]));
2799 		/* Drop the endpoint from the interval table if the endpoint is
2800 		 * being dropped or changed.
2801 		 */
2802 		if (EP_IS_DROPPED(ctrl_ctx, i))
2803 			xhci_drop_ep_from_interval_table(xhci,
2804 					&virt_dev->eps[i].bw_info,
2805 					virt_dev->bw_table,
2806 					virt_dev->udev,
2807 					&virt_dev->eps[i],
2808 					virt_dev->tt_info);
2809 	}
2810 	/* Overwrite the information stored in the endpoints' bw_info */
2811 	xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
2812 	for (i = 0; i < 31; i++) {
2813 		/* Add any changed or added endpoints to the interval table */
2814 		if (EP_IS_ADDED(ctrl_ctx, i))
2815 			xhci_add_ep_to_interval_table(xhci,
2816 					&virt_dev->eps[i].bw_info,
2817 					virt_dev->bw_table,
2818 					virt_dev->udev,
2819 					&virt_dev->eps[i],
2820 					virt_dev->tt_info);
2821 	}
2822 
2823 	if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
2824 		/* Ok, this fits in the bandwidth we have.
2825 		 * Update the number of active TTs.
2826 		 */
2827 		xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
2828 		return 0;
2829 	}
2830 
2831 	/* We don't have enough bandwidth for this, revert the stored info. */
2832 	for (i = 0; i < 31; i++) {
2833 		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2834 			continue;
2835 
2836 		/* Drop the new copies of any added or changed endpoints from
2837 		 * the interval table.
2838 		 */
2839 		if (EP_IS_ADDED(ctrl_ctx, i)) {
2840 			xhci_drop_ep_from_interval_table(xhci,
2841 					&virt_dev->eps[i].bw_info,
2842 					virt_dev->bw_table,
2843 					virt_dev->udev,
2844 					&virt_dev->eps[i],
2845 					virt_dev->tt_info);
2846 		}
2847 		/* Revert the endpoint back to its old information */
2848 		memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
2849 				sizeof(ep_bw_info[i]));
2850 		/* Add any changed or dropped endpoints back into the table */
2851 		if (EP_IS_DROPPED(ctrl_ctx, i))
2852 			xhci_add_ep_to_interval_table(xhci,
2853 					&virt_dev->eps[i].bw_info,
2854 					virt_dev->bw_table,
2855 					virt_dev->udev,
2856 					&virt_dev->eps[i],
2857 					virt_dev->tt_info);
2858 	}
2859 	return -ENOMEM;
2860 }
2861 
2862 
2863 /* Issue a configure endpoint command or evaluate context command
2864  * and wait for it to finish.
2865  */
2866 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
2867 		struct usb_device *udev,
2868 		struct xhci_command *command,
2869 		bool ctx_change, bool must_succeed)
2870 {
2871 	int ret;
2872 	unsigned long flags;
2873 	struct xhci_input_control_ctx *ctrl_ctx;
2874 	struct xhci_virt_device *virt_dev;
2875 	struct xhci_slot_ctx *slot_ctx;
2876 
2877 	if (!command)
2878 		return -EINVAL;
2879 
2880 	spin_lock_irqsave(&xhci->lock, flags);
2881 
2882 	if (xhci->xhc_state & XHCI_STATE_DYING) {
2883 		spin_unlock_irqrestore(&xhci->lock, flags);
2884 		return -ESHUTDOWN;
2885 	}
2886 
2887 	virt_dev = xhci->devs[udev->slot_id];
2888 
2889 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2890 	if (!ctrl_ctx) {
2891 		spin_unlock_irqrestore(&xhci->lock, flags);
2892 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2893 				__func__);
2894 		return -ENOMEM;
2895 	}
2896 
2897 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&
2898 			xhci_reserve_host_resources(xhci, ctrl_ctx)) {
2899 		spin_unlock_irqrestore(&xhci->lock, flags);
2900 		xhci_warn(xhci, "Not enough host resources, "
2901 				"active endpoint contexts = %u\n",
2902 				xhci->num_active_eps);
2903 		return -ENOMEM;
2904 	}
2905 	if ((xhci->quirks & XHCI_SW_BW_CHECKING) &&
2906 	    xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) {
2907 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2908 			xhci_free_host_resources(xhci, ctrl_ctx);
2909 		spin_unlock_irqrestore(&xhci->lock, flags);
2910 		xhci_warn(xhci, "Not enough bandwidth\n");
2911 		return -ENOMEM;
2912 	}
2913 
2914 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
2915 
2916 	trace_xhci_configure_endpoint_ctrl_ctx(ctrl_ctx);
2917 	trace_xhci_configure_endpoint(slot_ctx);
2918 
2919 	if (!ctx_change)
2920 		ret = xhci_queue_configure_endpoint(xhci, command,
2921 				command->in_ctx->dma,
2922 				udev->slot_id, must_succeed);
2923 	else
2924 		ret = xhci_queue_evaluate_context(xhci, command,
2925 				command->in_ctx->dma,
2926 				udev->slot_id, must_succeed);
2927 	if (ret < 0) {
2928 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2929 			xhci_free_host_resources(xhci, ctrl_ctx);
2930 		spin_unlock_irqrestore(&xhci->lock, flags);
2931 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
2932 				"FIXME allocate a new ring segment");
2933 		return -ENOMEM;
2934 	}
2935 	xhci_ring_cmd_db(xhci);
2936 	spin_unlock_irqrestore(&xhci->lock, flags);
2937 
2938 	/* Wait for the configure endpoint command to complete */
2939 	wait_for_completion(command->completion);
2940 
2941 	if (!ctx_change)
2942 		ret = xhci_configure_endpoint_result(xhci, udev,
2943 						     &command->status);
2944 	else
2945 		ret = xhci_evaluate_context_result(xhci, udev,
2946 						   &command->status);
2947 
2948 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
2949 		spin_lock_irqsave(&xhci->lock, flags);
2950 		/* If the command failed, remove the reserved resources.
2951 		 * Otherwise, clean up the estimate to include dropped eps.
2952 		 */
2953 		if (ret)
2954 			xhci_free_host_resources(xhci, ctrl_ctx);
2955 		else
2956 			xhci_finish_resource_reservation(xhci, ctrl_ctx);
2957 		spin_unlock_irqrestore(&xhci->lock, flags);
2958 	}
2959 	return ret;
2960 }
2961 
2962 static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci,
2963 	struct xhci_virt_device *vdev, int i)
2964 {
2965 	struct xhci_virt_ep *ep = &vdev->eps[i];
2966 
2967 	if (ep->ep_state & EP_HAS_STREAMS) {
2968 		xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n",
2969 				xhci_get_endpoint_address(i));
2970 		xhci_free_stream_info(xhci, ep->stream_info);
2971 		ep->stream_info = NULL;
2972 		ep->ep_state &= ~EP_HAS_STREAMS;
2973 	}
2974 }
2975 
2976 /* Called after one or more calls to xhci_add_endpoint() or
2977  * xhci_drop_endpoint().  If this call fails, the USB core is expected
2978  * to call xhci_reset_bandwidth().
2979  *
2980  * Since we are in the middle of changing either configuration or
2981  * installing a new alt setting, the USB core won't allow URBs to be
2982  * enqueued for any endpoint on the old config or interface.  Nothing
2983  * else should be touching the xhci->devs[slot_id] structure, so we
2984  * don't need to take the xhci->lock for manipulating that.
2985  */
2986 int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2987 {
2988 	int i;
2989 	int ret = 0;
2990 	struct xhci_hcd *xhci;
2991 	struct xhci_virt_device	*virt_dev;
2992 	struct xhci_input_control_ctx *ctrl_ctx;
2993 	struct xhci_slot_ctx *slot_ctx;
2994 	struct xhci_command *command;
2995 
2996 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2997 	if (ret <= 0)
2998 		return ret;
2999 	xhci = hcd_to_xhci(hcd);
3000 	if ((xhci->xhc_state & XHCI_STATE_DYING) ||
3001 		(xhci->xhc_state & XHCI_STATE_REMOVING))
3002 		return -ENODEV;
3003 
3004 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
3005 	virt_dev = xhci->devs[udev->slot_id];
3006 
3007 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
3008 	if (!command)
3009 		return -ENOMEM;
3010 
3011 	command->in_ctx = virt_dev->in_ctx;
3012 
3013 	/* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */
3014 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
3015 	if (!ctrl_ctx) {
3016 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3017 				__func__);
3018 		ret = -ENOMEM;
3019 		goto command_cleanup;
3020 	}
3021 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
3022 	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
3023 	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
3024 
3025 	/* Don't issue the command if there's no endpoints to update. */
3026 	if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
3027 	    ctrl_ctx->drop_flags == 0) {
3028 		ret = 0;
3029 		goto command_cleanup;
3030 	}
3031 	/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
3032 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
3033 	for (i = 31; i >= 1; i--) {
3034 		__le32 le32 = cpu_to_le32(BIT(i));
3035 
3036 		if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))
3037 		    || (ctrl_ctx->add_flags & le32) || i == 1) {
3038 			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
3039 			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
3040 			break;
3041 		}
3042 	}
3043 
3044 	ret = xhci_configure_endpoint(xhci, udev, command,
3045 			false, false);
3046 	if (ret)
3047 		/* Callee should call reset_bandwidth() */
3048 		goto command_cleanup;
3049 
3050 	/* Free any rings that were dropped, but not changed. */
3051 	for (i = 1; i < 31; i++) {
3052 		if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
3053 		    !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {
3054 			xhci_free_endpoint_ring(xhci, virt_dev, i);
3055 			xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
3056 		}
3057 	}
3058 	xhci_zero_in_ctx(xhci, virt_dev);
3059 	/*
3060 	 * Install any rings for completely new endpoints or changed endpoints,
3061 	 * and free any old rings from changed endpoints.
3062 	 */
3063 	for (i = 1; i < 31; i++) {
3064 		if (!virt_dev->eps[i].new_ring)
3065 			continue;
3066 		/* Only free the old ring if it exists.
3067 		 * It may not if this is the first add of an endpoint.
3068 		 */
3069 		if (virt_dev->eps[i].ring) {
3070 			xhci_free_endpoint_ring(xhci, virt_dev, i);
3071 		}
3072 		xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
3073 		virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
3074 		virt_dev->eps[i].new_ring = NULL;
3075 		xhci_debugfs_create_endpoint(xhci, virt_dev, i);
3076 	}
3077 command_cleanup:
3078 	kfree(command->completion);
3079 	kfree(command);
3080 
3081 	return ret;
3082 }
3083 EXPORT_SYMBOL_GPL(xhci_check_bandwidth);
3084 
3085 void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
3086 {
3087 	struct xhci_hcd *xhci;
3088 	struct xhci_virt_device	*virt_dev;
3089 	int i, ret;
3090 
3091 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3092 	if (ret <= 0)
3093 		return;
3094 	xhci = hcd_to_xhci(hcd);
3095 
3096 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
3097 	virt_dev = xhci->devs[udev->slot_id];
3098 	/* Free any rings allocated for added endpoints */
3099 	for (i = 0; i < 31; i++) {
3100 		if (virt_dev->eps[i].new_ring) {
3101 			xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
3102 			xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
3103 			virt_dev->eps[i].new_ring = NULL;
3104 		}
3105 	}
3106 	xhci_zero_in_ctx(xhci, virt_dev);
3107 }
3108 EXPORT_SYMBOL_GPL(xhci_reset_bandwidth);
3109 
3110 static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
3111 		struct xhci_container_ctx *in_ctx,
3112 		struct xhci_container_ctx *out_ctx,
3113 		struct xhci_input_control_ctx *ctrl_ctx,
3114 		u32 add_flags, u32 drop_flags)
3115 {
3116 	ctrl_ctx->add_flags = cpu_to_le32(add_flags);
3117 	ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
3118 	xhci_slot_copy(xhci, in_ctx, out_ctx);
3119 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
3120 }
3121 
3122 static void xhci_endpoint_disable(struct usb_hcd *hcd,
3123 				  struct usb_host_endpoint *host_ep)
3124 {
3125 	struct xhci_hcd		*xhci;
3126 	struct xhci_virt_device	*vdev;
3127 	struct xhci_virt_ep	*ep;
3128 	struct usb_device	*udev;
3129 	unsigned long		flags;
3130 	unsigned int		ep_index;
3131 
3132 	xhci = hcd_to_xhci(hcd);
3133 rescan:
3134 	spin_lock_irqsave(&xhci->lock, flags);
3135 
3136 	udev = (struct usb_device *)host_ep->hcpriv;
3137 	if (!udev || !udev->slot_id)
3138 		goto done;
3139 
3140 	vdev = xhci->devs[udev->slot_id];
3141 	if (!vdev)
3142 		goto done;
3143 
3144 	ep_index = xhci_get_endpoint_index(&host_ep->desc);
3145 	ep = &vdev->eps[ep_index];
3146 	if (!ep)
3147 		goto done;
3148 
3149 	/* wait for hub_tt_work to finish clearing hub TT */
3150 	if (ep->ep_state & EP_CLEARING_TT) {
3151 		spin_unlock_irqrestore(&xhci->lock, flags);
3152 		schedule_timeout_uninterruptible(1);
3153 		goto rescan;
3154 	}
3155 
3156 	if (ep->ep_state)
3157 		xhci_dbg(xhci, "endpoint disable with ep_state 0x%x\n",
3158 			 ep->ep_state);
3159 done:
3160 	host_ep->hcpriv = NULL;
3161 	spin_unlock_irqrestore(&xhci->lock, flags);
3162 }
3163 
3164 /*
3165  * Called after usb core issues a clear halt control message.
3166  * The host side of the halt should already be cleared by a reset endpoint
3167  * command issued when the STALL event was received.
3168  *
3169  * The reset endpoint command may only be issued to endpoints in the halted
3170  * state. For software that wishes to reset the data toggle or sequence number
3171  * of an endpoint that isn't in the halted state this function will issue a
3172  * configure endpoint command with the Drop and Add bits set for the target
3173  * endpoint. Refer to the additional note in xhci spcification section 4.6.8.
3174  */
3175 
3176 static void xhci_endpoint_reset(struct usb_hcd *hcd,
3177 		struct usb_host_endpoint *host_ep)
3178 {
3179 	struct xhci_hcd *xhci;
3180 	struct usb_device *udev;
3181 	struct xhci_virt_device *vdev;
3182 	struct xhci_virt_ep *ep;
3183 	struct xhci_input_control_ctx *ctrl_ctx;
3184 	struct xhci_command *stop_cmd, *cfg_cmd;
3185 	unsigned int ep_index;
3186 	unsigned long flags;
3187 	u32 ep_flag;
3188 	int err;
3189 
3190 	xhci = hcd_to_xhci(hcd);
3191 	if (!host_ep->hcpriv)
3192 		return;
3193 	udev = (struct usb_device *) host_ep->hcpriv;
3194 	vdev = xhci->devs[udev->slot_id];
3195 
3196 	/*
3197 	 * vdev may be lost due to xHC restore error and re-initialization
3198 	 * during S3/S4 resume. A new vdev will be allocated later by
3199 	 * xhci_discover_or_reset_device()
3200 	 */
3201 	if (!udev->slot_id || !vdev)
3202 		return;
3203 	ep_index = xhci_get_endpoint_index(&host_ep->desc);
3204 	ep = &vdev->eps[ep_index];
3205 	if (!ep)
3206 		return;
3207 
3208 	/* Bail out if toggle is already being cleared by a endpoint reset */
3209 	if (ep->ep_state & EP_HARD_CLEAR_TOGGLE) {
3210 		ep->ep_state &= ~EP_HARD_CLEAR_TOGGLE;
3211 		return;
3212 	}
3213 	/* Only interrupt and bulk ep's use data toggle, USB2 spec 5.5.4-> */
3214 	if (usb_endpoint_xfer_control(&host_ep->desc) ||
3215 	    usb_endpoint_xfer_isoc(&host_ep->desc))
3216 		return;
3217 
3218 	ep_flag = xhci_get_endpoint_flag(&host_ep->desc);
3219 
3220 	if (ep_flag == SLOT_FLAG || ep_flag == EP0_FLAG)
3221 		return;
3222 
3223 	stop_cmd = xhci_alloc_command(xhci, true, GFP_NOWAIT);
3224 	if (!stop_cmd)
3225 		return;
3226 
3227 	cfg_cmd = xhci_alloc_command_with_ctx(xhci, true, GFP_NOWAIT);
3228 	if (!cfg_cmd)
3229 		goto cleanup;
3230 
3231 	spin_lock_irqsave(&xhci->lock, flags);
3232 
3233 	/* block queuing new trbs and ringing ep doorbell */
3234 	ep->ep_state |= EP_SOFT_CLEAR_TOGGLE;
3235 
3236 	/*
3237 	 * Make sure endpoint ring is empty before resetting the toggle/seq.
3238 	 * Driver is required to synchronously cancel all transfer request.
3239 	 * Stop the endpoint to force xHC to update the output context
3240 	 */
3241 
3242 	if (!list_empty(&ep->ring->td_list)) {
3243 		dev_err(&udev->dev, "EP not empty, refuse reset\n");
3244 		spin_unlock_irqrestore(&xhci->lock, flags);
3245 		xhci_free_command(xhci, cfg_cmd);
3246 		goto cleanup;
3247 	}
3248 
3249 	err = xhci_queue_stop_endpoint(xhci, stop_cmd, udev->slot_id,
3250 					ep_index, 0);
3251 	if (err < 0) {
3252 		spin_unlock_irqrestore(&xhci->lock, flags);
3253 		xhci_free_command(xhci, cfg_cmd);
3254 		xhci_dbg(xhci, "%s: Failed to queue stop ep command, %d ",
3255 				__func__, err);
3256 		goto cleanup;
3257 	}
3258 
3259 	xhci_ring_cmd_db(xhci);
3260 	spin_unlock_irqrestore(&xhci->lock, flags);
3261 
3262 	wait_for_completion(stop_cmd->completion);
3263 
3264 	spin_lock_irqsave(&xhci->lock, flags);
3265 
3266 	/* config ep command clears toggle if add and drop ep flags are set */
3267 	ctrl_ctx = xhci_get_input_control_ctx(cfg_cmd->in_ctx);
3268 	if (!ctrl_ctx) {
3269 		spin_unlock_irqrestore(&xhci->lock, flags);
3270 		xhci_free_command(xhci, cfg_cmd);
3271 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3272 				__func__);
3273 		goto cleanup;
3274 	}
3275 
3276 	xhci_setup_input_ctx_for_config_ep(xhci, cfg_cmd->in_ctx, vdev->out_ctx,
3277 					   ctrl_ctx, ep_flag, ep_flag);
3278 	xhci_endpoint_copy(xhci, cfg_cmd->in_ctx, vdev->out_ctx, ep_index);
3279 
3280 	err = xhci_queue_configure_endpoint(xhci, cfg_cmd, cfg_cmd->in_ctx->dma,
3281 				      udev->slot_id, false);
3282 	if (err < 0) {
3283 		spin_unlock_irqrestore(&xhci->lock, flags);
3284 		xhci_free_command(xhci, cfg_cmd);
3285 		xhci_dbg(xhci, "%s: Failed to queue config ep command, %d ",
3286 				__func__, err);
3287 		goto cleanup;
3288 	}
3289 
3290 	xhci_ring_cmd_db(xhci);
3291 	spin_unlock_irqrestore(&xhci->lock, flags);
3292 
3293 	wait_for_completion(cfg_cmd->completion);
3294 
3295 	xhci_free_command(xhci, cfg_cmd);
3296 cleanup:
3297 	xhci_free_command(xhci, stop_cmd);
3298 	if (ep->ep_state & EP_SOFT_CLEAR_TOGGLE)
3299 		ep->ep_state &= ~EP_SOFT_CLEAR_TOGGLE;
3300 }
3301 
3302 static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
3303 		struct usb_device *udev, struct usb_host_endpoint *ep,
3304 		unsigned int slot_id)
3305 {
3306 	int ret;
3307 	unsigned int ep_index;
3308 	unsigned int ep_state;
3309 
3310 	if (!ep)
3311 		return -EINVAL;
3312 	ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
3313 	if (ret <= 0)
3314 		return -EINVAL;
3315 	if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {
3316 		xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
3317 				" descriptor for ep 0x%x does not support streams\n",
3318 				ep->desc.bEndpointAddress);
3319 		return -EINVAL;
3320 	}
3321 
3322 	ep_index = xhci_get_endpoint_index(&ep->desc);
3323 	ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3324 	if (ep_state & EP_HAS_STREAMS ||
3325 			ep_state & EP_GETTING_STREAMS) {
3326 		xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
3327 				"already has streams set up.\n",
3328 				ep->desc.bEndpointAddress);
3329 		xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
3330 				"dynamic stream context array reallocation.\n");
3331 		return -EINVAL;
3332 	}
3333 	if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
3334 		xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
3335 				"endpoint 0x%x; URBs are pending.\n",
3336 				ep->desc.bEndpointAddress);
3337 		return -EINVAL;
3338 	}
3339 	return 0;
3340 }
3341 
3342 static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
3343 		unsigned int *num_streams, unsigned int *num_stream_ctxs)
3344 {
3345 	unsigned int max_streams;
3346 
3347 	/* The stream context array size must be a power of two */
3348 	*num_stream_ctxs = roundup_pow_of_two(*num_streams);
3349 	/*
3350 	 * Find out how many primary stream array entries the host controller
3351 	 * supports.  Later we may use secondary stream arrays (similar to 2nd
3352 	 * level page entries), but that's an optional feature for xHCI host
3353 	 * controllers. xHCs must support at least 4 stream IDs.
3354 	 */
3355 	max_streams = HCC_MAX_PSA(xhci->hcc_params);
3356 	if (*num_stream_ctxs > max_streams) {
3357 		xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
3358 				max_streams);
3359 		*num_stream_ctxs = max_streams;
3360 		*num_streams = max_streams;
3361 	}
3362 }
3363 
3364 /* Returns an error code if one of the endpoint already has streams.
3365  * This does not change any data structures, it only checks and gathers
3366  * information.
3367  */
3368 static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
3369 		struct usb_device *udev,
3370 		struct usb_host_endpoint **eps, unsigned int num_eps,
3371 		unsigned int *num_streams, u32 *changed_ep_bitmask)
3372 {
3373 	unsigned int max_streams;
3374 	unsigned int endpoint_flag;
3375 	int i;
3376 	int ret;
3377 
3378 	for (i = 0; i < num_eps; i++) {
3379 		ret = xhci_check_streams_endpoint(xhci, udev,
3380 				eps[i], udev->slot_id);
3381 		if (ret < 0)
3382 			return ret;
3383 
3384 		max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);
3385 		if (max_streams < (*num_streams - 1)) {
3386 			xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
3387 					eps[i]->desc.bEndpointAddress,
3388 					max_streams);
3389 			*num_streams = max_streams+1;
3390 		}
3391 
3392 		endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
3393 		if (*changed_ep_bitmask & endpoint_flag)
3394 			return -EINVAL;
3395 		*changed_ep_bitmask |= endpoint_flag;
3396 	}
3397 	return 0;
3398 }
3399 
3400 static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
3401 		struct usb_device *udev,
3402 		struct usb_host_endpoint **eps, unsigned int num_eps)
3403 {
3404 	u32 changed_ep_bitmask = 0;
3405 	unsigned int slot_id;
3406 	unsigned int ep_index;
3407 	unsigned int ep_state;
3408 	int i;
3409 
3410 	slot_id = udev->slot_id;
3411 	if (!xhci->devs[slot_id])
3412 		return 0;
3413 
3414 	for (i = 0; i < num_eps; i++) {
3415 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3416 		ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3417 		/* Are streams already being freed for the endpoint? */
3418 		if (ep_state & EP_GETTING_NO_STREAMS) {
3419 			xhci_warn(xhci, "WARN Can't disable streams for "
3420 					"endpoint 0x%x, "
3421 					"streams are being disabled already\n",
3422 					eps[i]->desc.bEndpointAddress);
3423 			return 0;
3424 		}
3425 		/* Are there actually any streams to free? */
3426 		if (!(ep_state & EP_HAS_STREAMS) &&
3427 				!(ep_state & EP_GETTING_STREAMS)) {
3428 			xhci_warn(xhci, "WARN Can't disable streams for "
3429 					"endpoint 0x%x, "
3430 					"streams are already disabled!\n",
3431 					eps[i]->desc.bEndpointAddress);
3432 			xhci_warn(xhci, "WARN xhci_free_streams() called "
3433 					"with non-streams endpoint\n");
3434 			return 0;
3435 		}
3436 		changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
3437 	}
3438 	return changed_ep_bitmask;
3439 }
3440 
3441 /*
3442  * The USB device drivers use this function (through the HCD interface in USB
3443  * core) to prepare a set of bulk endpoints to use streams.  Streams are used to
3444  * coordinate mass storage command queueing across multiple endpoints (basically
3445  * a stream ID == a task ID).
3446  *
3447  * Setting up streams involves allocating the same size stream context array
3448  * for each endpoint and issuing a configure endpoint command for all endpoints.
3449  *
3450  * Don't allow the call to succeed if one endpoint only supports one stream
3451  * (which means it doesn't support streams at all).
3452  *
3453  * Drivers may get less stream IDs than they asked for, if the host controller
3454  * hardware or endpoints claim they can't support the number of requested
3455  * stream IDs.
3456  */
3457 static int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
3458 		struct usb_host_endpoint **eps, unsigned int num_eps,
3459 		unsigned int num_streams, gfp_t mem_flags)
3460 {
3461 	int i, ret;
3462 	struct xhci_hcd *xhci;
3463 	struct xhci_virt_device *vdev;
3464 	struct xhci_command *config_cmd;
3465 	struct xhci_input_control_ctx *ctrl_ctx;
3466 	unsigned int ep_index;
3467 	unsigned int num_stream_ctxs;
3468 	unsigned int max_packet;
3469 	unsigned long flags;
3470 	u32 changed_ep_bitmask = 0;
3471 
3472 	if (!eps)
3473 		return -EINVAL;
3474 
3475 	/* Add one to the number of streams requested to account for
3476 	 * stream 0 that is reserved for xHCI usage.
3477 	 */
3478 	num_streams += 1;
3479 	xhci = hcd_to_xhci(hcd);
3480 	xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
3481 			num_streams);
3482 
3483 	/* MaxPSASize value 0 (2 streams) means streams are not supported */
3484 	if ((xhci->quirks & XHCI_BROKEN_STREAMS) ||
3485 			HCC_MAX_PSA(xhci->hcc_params) < 4) {
3486 		xhci_dbg(xhci, "xHCI controller does not support streams.\n");
3487 		return -ENOSYS;
3488 	}
3489 
3490 	config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
3491 	if (!config_cmd)
3492 		return -ENOMEM;
3493 
3494 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
3495 	if (!ctrl_ctx) {
3496 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3497 				__func__);
3498 		xhci_free_command(xhci, config_cmd);
3499 		return -ENOMEM;
3500 	}
3501 
3502 	/* Check to make sure all endpoints are not already configured for
3503 	 * streams.  While we're at it, find the maximum number of streams that
3504 	 * all the endpoints will support and check for duplicate endpoints.
3505 	 */
3506 	spin_lock_irqsave(&xhci->lock, flags);
3507 	ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
3508 			num_eps, &num_streams, &changed_ep_bitmask);
3509 	if (ret < 0) {
3510 		xhci_free_command(xhci, config_cmd);
3511 		spin_unlock_irqrestore(&xhci->lock, flags);
3512 		return ret;
3513 	}
3514 	if (num_streams <= 1) {
3515 		xhci_warn(xhci, "WARN: endpoints can't handle "
3516 				"more than one stream.\n");
3517 		xhci_free_command(xhci, config_cmd);
3518 		spin_unlock_irqrestore(&xhci->lock, flags);
3519 		return -EINVAL;
3520 	}
3521 	vdev = xhci->devs[udev->slot_id];
3522 	/* Mark each endpoint as being in transition, so
3523 	 * xhci_urb_enqueue() will reject all URBs.
3524 	 */
3525 	for (i = 0; i < num_eps; i++) {
3526 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3527 		vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
3528 	}
3529 	spin_unlock_irqrestore(&xhci->lock, flags);
3530 
3531 	/* Setup internal data structures and allocate HW data structures for
3532 	 * streams (but don't install the HW structures in the input context
3533 	 * until we're sure all memory allocation succeeded).
3534 	 */
3535 	xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
3536 	xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
3537 			num_stream_ctxs, num_streams);
3538 
3539 	for (i = 0; i < num_eps; i++) {
3540 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3541 		max_packet = usb_endpoint_maxp(&eps[i]->desc);
3542 		vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
3543 				num_stream_ctxs,
3544 				num_streams,
3545 				max_packet, mem_flags);
3546 		if (!vdev->eps[ep_index].stream_info)
3547 			goto cleanup;
3548 		/* Set maxPstreams in endpoint context and update deq ptr to
3549 		 * point to stream context array. FIXME
3550 		 */
3551 	}
3552 
3553 	/* Set up the input context for a configure endpoint command. */
3554 	for (i = 0; i < num_eps; i++) {
3555 		struct xhci_ep_ctx *ep_ctx;
3556 
3557 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3558 		ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);
3559 
3560 		xhci_endpoint_copy(xhci, config_cmd->in_ctx,
3561 				vdev->out_ctx, ep_index);
3562 		xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
3563 				vdev->eps[ep_index].stream_info);
3564 	}
3565 	/* Tell the HW to drop its old copy of the endpoint context info
3566 	 * and add the updated copy from the input context.
3567 	 */
3568 	xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
3569 			vdev->out_ctx, ctrl_ctx,
3570 			changed_ep_bitmask, changed_ep_bitmask);
3571 
3572 	/* Issue and wait for the configure endpoint command */
3573 	ret = xhci_configure_endpoint(xhci, udev, config_cmd,
3574 			false, false);
3575 
3576 	/* xHC rejected the configure endpoint command for some reason, so we
3577 	 * leave the old ring intact and free our internal streams data
3578 	 * structure.
3579 	 */
3580 	if (ret < 0)
3581 		goto cleanup;
3582 
3583 	spin_lock_irqsave(&xhci->lock, flags);
3584 	for (i = 0; i < num_eps; i++) {
3585 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3586 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3587 		xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
3588 			 udev->slot_id, ep_index);
3589 		vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
3590 	}
3591 	xhci_free_command(xhci, config_cmd);
3592 	spin_unlock_irqrestore(&xhci->lock, flags);
3593 
3594 	for (i = 0; i < num_eps; i++) {
3595 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3596 		xhci_debugfs_create_stream_files(xhci, vdev, ep_index);
3597 	}
3598 	/* Subtract 1 for stream 0, which drivers can't use */
3599 	return num_streams - 1;
3600 
3601 cleanup:
3602 	/* If it didn't work, free the streams! */
3603 	for (i = 0; i < num_eps; i++) {
3604 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3605 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3606 		vdev->eps[ep_index].stream_info = NULL;
3607 		/* FIXME Unset maxPstreams in endpoint context and
3608 		 * update deq ptr to point to normal string ring.
3609 		 */
3610 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3611 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3612 		xhci_endpoint_zero(xhci, vdev, eps[i]);
3613 	}
3614 	xhci_free_command(xhci, config_cmd);
3615 	return -ENOMEM;
3616 }
3617 
3618 /* Transition the endpoint from using streams to being a "normal" endpoint
3619  * without streams.
3620  *
3621  * Modify the endpoint context state, submit a configure endpoint command,
3622  * and free all endpoint rings for streams if that completes successfully.
3623  */
3624 static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
3625 		struct usb_host_endpoint **eps, unsigned int num_eps,
3626 		gfp_t mem_flags)
3627 {
3628 	int i, ret;
3629 	struct xhci_hcd *xhci;
3630 	struct xhci_virt_device *vdev;
3631 	struct xhci_command *command;
3632 	struct xhci_input_control_ctx *ctrl_ctx;
3633 	unsigned int ep_index;
3634 	unsigned long flags;
3635 	u32 changed_ep_bitmask;
3636 
3637 	xhci = hcd_to_xhci(hcd);
3638 	vdev = xhci->devs[udev->slot_id];
3639 
3640 	/* Set up a configure endpoint command to remove the streams rings */
3641 	spin_lock_irqsave(&xhci->lock, flags);
3642 	changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
3643 			udev, eps, num_eps);
3644 	if (changed_ep_bitmask == 0) {
3645 		spin_unlock_irqrestore(&xhci->lock, flags);
3646 		return -EINVAL;
3647 	}
3648 
3649 	/* Use the xhci_command structure from the first endpoint.  We may have
3650 	 * allocated too many, but the driver may call xhci_free_streams() for
3651 	 * each endpoint it grouped into one call to xhci_alloc_streams().
3652 	 */
3653 	ep_index = xhci_get_endpoint_index(&eps[0]->desc);
3654 	command = vdev->eps[ep_index].stream_info->free_streams_command;
3655 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
3656 	if (!ctrl_ctx) {
3657 		spin_unlock_irqrestore(&xhci->lock, flags);
3658 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3659 				__func__);
3660 		return -EINVAL;
3661 	}
3662 
3663 	for (i = 0; i < num_eps; i++) {
3664 		struct xhci_ep_ctx *ep_ctx;
3665 
3666 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3667 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
3668 		xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
3669 			EP_GETTING_NO_STREAMS;
3670 
3671 		xhci_endpoint_copy(xhci, command->in_ctx,
3672 				vdev->out_ctx, ep_index);
3673 		xhci_setup_no_streams_ep_input_ctx(ep_ctx,
3674 				&vdev->eps[ep_index]);
3675 	}
3676 	xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
3677 			vdev->out_ctx, ctrl_ctx,
3678 			changed_ep_bitmask, changed_ep_bitmask);
3679 	spin_unlock_irqrestore(&xhci->lock, flags);
3680 
3681 	/* Issue and wait for the configure endpoint command,
3682 	 * which must succeed.
3683 	 */
3684 	ret = xhci_configure_endpoint(xhci, udev, command,
3685 			false, true);
3686 
3687 	/* xHC rejected the configure endpoint command for some reason, so we
3688 	 * leave the streams rings intact.
3689 	 */
3690 	if (ret < 0)
3691 		return ret;
3692 
3693 	spin_lock_irqsave(&xhci->lock, flags);
3694 	for (i = 0; i < num_eps; i++) {
3695 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3696 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3697 		vdev->eps[ep_index].stream_info = NULL;
3698 		/* FIXME Unset maxPstreams in endpoint context and
3699 		 * update deq ptr to point to normal string ring.
3700 		 */
3701 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
3702 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3703 	}
3704 	spin_unlock_irqrestore(&xhci->lock, flags);
3705 
3706 	return 0;
3707 }
3708 
3709 /*
3710  * Deletes endpoint resources for endpoints that were active before a Reset
3711  * Device command, or a Disable Slot command.  The Reset Device command leaves
3712  * the control endpoint intact, whereas the Disable Slot command deletes it.
3713  *
3714  * Must be called with xhci->lock held.
3715  */
3716 void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
3717 	struct xhci_virt_device *virt_dev, bool drop_control_ep)
3718 {
3719 	int i;
3720 	unsigned int num_dropped_eps = 0;
3721 	unsigned int drop_flags = 0;
3722 
3723 	for (i = (drop_control_ep ? 0 : 1); i < 31; i++) {
3724 		if (virt_dev->eps[i].ring) {
3725 			drop_flags |= 1 << i;
3726 			num_dropped_eps++;
3727 		}
3728 	}
3729 	xhci->num_active_eps -= num_dropped_eps;
3730 	if (num_dropped_eps)
3731 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3732 				"Dropped %u ep ctxs, flags = 0x%x, "
3733 				"%u now active.",
3734 				num_dropped_eps, drop_flags,
3735 				xhci->num_active_eps);
3736 }
3737 
3738 /*
3739  * This submits a Reset Device Command, which will set the device state to 0,
3740  * set the device address to 0, and disable all the endpoints except the default
3741  * control endpoint.  The USB core should come back and call
3742  * xhci_address_device(), and then re-set up the configuration.  If this is
3743  * called because of a usb_reset_and_verify_device(), then the old alternate
3744  * settings will be re-installed through the normal bandwidth allocation
3745  * functions.
3746  *
3747  * Wait for the Reset Device command to finish.  Remove all structures
3748  * associated with the endpoints that were disabled.  Clear the input device
3749  * structure? Reset the control endpoint 0 max packet size?
3750  *
3751  * If the virt_dev to be reset does not exist or does not match the udev,
3752  * it means the device is lost, possibly due to the xHC restore error and
3753  * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
3754  * re-allocate the device.
3755  */
3756 static int xhci_discover_or_reset_device(struct usb_hcd *hcd,
3757 		struct usb_device *udev)
3758 {
3759 	int ret, i;
3760 	unsigned long flags;
3761 	struct xhci_hcd *xhci;
3762 	unsigned int slot_id;
3763 	struct xhci_virt_device *virt_dev;
3764 	struct xhci_command *reset_device_cmd;
3765 	struct xhci_slot_ctx *slot_ctx;
3766 	int old_active_eps = 0;
3767 
3768 	ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
3769 	if (ret <= 0)
3770 		return ret;
3771 	xhci = hcd_to_xhci(hcd);
3772 	slot_id = udev->slot_id;
3773 	virt_dev = xhci->devs[slot_id];
3774 	if (!virt_dev) {
3775 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3776 				"not exist. Re-allocate the device\n", slot_id);
3777 		ret = xhci_alloc_dev(hcd, udev);
3778 		if (ret == 1)
3779 			return 0;
3780 		else
3781 			return -EINVAL;
3782 	}
3783 
3784 	if (virt_dev->tt_info)
3785 		old_active_eps = virt_dev->tt_info->active_eps;
3786 
3787 	if (virt_dev->udev != udev) {
3788 		/* If the virt_dev and the udev does not match, this virt_dev
3789 		 * may belong to another udev.
3790 		 * Re-allocate the device.
3791 		 */
3792 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3793 				"not match the udev. Re-allocate the device\n",
3794 				slot_id);
3795 		ret = xhci_alloc_dev(hcd, udev);
3796 		if (ret == 1)
3797 			return 0;
3798 		else
3799 			return -EINVAL;
3800 	}
3801 
3802 	/* If device is not setup, there is no point in resetting it */
3803 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3804 	if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
3805 						SLOT_STATE_DISABLED)
3806 		return 0;
3807 
3808 	trace_xhci_discover_or_reset_device(slot_ctx);
3809 
3810 	xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
3811 	/* Allocate the command structure that holds the struct completion.
3812 	 * Assume we're in process context, since the normal device reset
3813 	 * process has to wait for the device anyway.  Storage devices are
3814 	 * reset as part of error handling, so use GFP_NOIO instead of
3815 	 * GFP_KERNEL.
3816 	 */
3817 	reset_device_cmd = xhci_alloc_command(xhci, true, GFP_NOIO);
3818 	if (!reset_device_cmd) {
3819 		xhci_dbg(xhci, "Couldn't allocate command structure.\n");
3820 		return -ENOMEM;
3821 	}
3822 
3823 	/* Attempt to submit the Reset Device command to the command ring */
3824 	spin_lock_irqsave(&xhci->lock, flags);
3825 
3826 	ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id);
3827 	if (ret) {
3828 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3829 		spin_unlock_irqrestore(&xhci->lock, flags);
3830 		goto command_cleanup;
3831 	}
3832 	xhci_ring_cmd_db(xhci);
3833 	spin_unlock_irqrestore(&xhci->lock, flags);
3834 
3835 	/* Wait for the Reset Device command to finish */
3836 	wait_for_completion(reset_device_cmd->completion);
3837 
3838 	/* The Reset Device command can't fail, according to the 0.95/0.96 spec,
3839 	 * unless we tried to reset a slot ID that wasn't enabled,
3840 	 * or the device wasn't in the addressed or configured state.
3841 	 */
3842 	ret = reset_device_cmd->status;
3843 	switch (ret) {
3844 	case COMP_COMMAND_ABORTED:
3845 	case COMP_COMMAND_RING_STOPPED:
3846 		xhci_warn(xhci, "Timeout waiting for reset device command\n");
3847 		ret = -ETIME;
3848 		goto command_cleanup;
3849 	case COMP_SLOT_NOT_ENABLED_ERROR: /* 0.95 completion for bad slot ID */
3850 	case COMP_CONTEXT_STATE_ERROR: /* 0.96 completion code for same thing */
3851 		xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n",
3852 				slot_id,
3853 				xhci_get_slot_state(xhci, virt_dev->out_ctx));
3854 		xhci_dbg(xhci, "Not freeing device rings.\n");
3855 		/* Don't treat this as an error.  May change my mind later. */
3856 		ret = 0;
3857 		goto command_cleanup;
3858 	case COMP_SUCCESS:
3859 		xhci_dbg(xhci, "Successful reset device command.\n");
3860 		break;
3861 	default:
3862 		if (xhci_is_vendor_info_code(xhci, ret))
3863 			break;
3864 		xhci_warn(xhci, "Unknown completion code %u for "
3865 				"reset device command.\n", ret);
3866 		ret = -EINVAL;
3867 		goto command_cleanup;
3868 	}
3869 
3870 	/* Free up host controller endpoint resources */
3871 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
3872 		spin_lock_irqsave(&xhci->lock, flags);
3873 		/* Don't delete the default control endpoint resources */
3874 		xhci_free_device_endpoint_resources(xhci, virt_dev, false);
3875 		spin_unlock_irqrestore(&xhci->lock, flags);
3876 	}
3877 
3878 	/* Everything but endpoint 0 is disabled, so free the rings. */
3879 	for (i = 1; i < 31; i++) {
3880 		struct xhci_virt_ep *ep = &virt_dev->eps[i];
3881 
3882 		if (ep->ep_state & EP_HAS_STREAMS) {
3883 			xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n",
3884 					xhci_get_endpoint_address(i));
3885 			xhci_free_stream_info(xhci, ep->stream_info);
3886 			ep->stream_info = NULL;
3887 			ep->ep_state &= ~EP_HAS_STREAMS;
3888 		}
3889 
3890 		if (ep->ring) {
3891 			xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
3892 			xhci_free_endpoint_ring(xhci, virt_dev, i);
3893 		}
3894 		if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))
3895 			xhci_drop_ep_from_interval_table(xhci,
3896 					&virt_dev->eps[i].bw_info,
3897 					virt_dev->bw_table,
3898 					udev,
3899 					&virt_dev->eps[i],
3900 					virt_dev->tt_info);
3901 		xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);
3902 	}
3903 	/* If necessary, update the number of active TTs on this root port */
3904 	xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
3905 	virt_dev->flags = 0;
3906 	ret = 0;
3907 
3908 command_cleanup:
3909 	xhci_free_command(xhci, reset_device_cmd);
3910 	return ret;
3911 }
3912 
3913 /*
3914  * At this point, the struct usb_device is about to go away, the device has
3915  * disconnected, and all traffic has been stopped and the endpoints have been
3916  * disabled.  Free any HC data structures associated with that device.
3917  */
3918 static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
3919 {
3920 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3921 	struct xhci_virt_device *virt_dev;
3922 	struct xhci_slot_ctx *slot_ctx;
3923 	int i, ret;
3924 
3925 #ifndef CONFIG_USB_DEFAULT_PERSIST
3926 	/*
3927 	 * We called pm_runtime_get_noresume when the device was attached.
3928 	 * Decrement the counter here to allow controller to runtime suspend
3929 	 * if no devices remain.
3930 	 */
3931 	if (xhci->quirks & XHCI_RESET_ON_RESUME)
3932 		pm_runtime_put_noidle(hcd->self.controller);
3933 #endif
3934 
3935 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3936 	/* If the host is halted due to driver unload, we still need to free the
3937 	 * device.
3938 	 */
3939 	if (ret <= 0 && ret != -ENODEV)
3940 		return;
3941 
3942 	virt_dev = xhci->devs[udev->slot_id];
3943 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3944 	trace_xhci_free_dev(slot_ctx);
3945 
3946 	/* Stop any wayward timer functions (which may grab the lock) */
3947 	for (i = 0; i < 31; i++) {
3948 		virt_dev->eps[i].ep_state &= ~EP_STOP_CMD_PENDING;
3949 		del_timer_sync(&virt_dev->eps[i].stop_cmd_timer);
3950 	}
3951 	virt_dev->udev = NULL;
3952 	ret = xhci_disable_slot(xhci, udev->slot_id);
3953 	if (ret)
3954 		xhci_free_virt_device(xhci, udev->slot_id);
3955 }
3956 
3957 int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)
3958 {
3959 	struct xhci_command *command;
3960 	unsigned long flags;
3961 	u32 state;
3962 	int ret = 0;
3963 
3964 	command = xhci_alloc_command(xhci, false, GFP_KERNEL);
3965 	if (!command)
3966 		return -ENOMEM;
3967 
3968 	xhci_debugfs_remove_slot(xhci, slot_id);
3969 
3970 	spin_lock_irqsave(&xhci->lock, flags);
3971 	/* Don't disable the slot if the host controller is dead. */
3972 	state = readl(&xhci->op_regs->status);
3973 	if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
3974 			(xhci->xhc_state & XHCI_STATE_HALTED)) {
3975 		spin_unlock_irqrestore(&xhci->lock, flags);
3976 		kfree(command);
3977 		return -ENODEV;
3978 	}
3979 
3980 	ret = xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
3981 				slot_id);
3982 	if (ret) {
3983 		spin_unlock_irqrestore(&xhci->lock, flags);
3984 		kfree(command);
3985 		return ret;
3986 	}
3987 	xhci_ring_cmd_db(xhci);
3988 	spin_unlock_irqrestore(&xhci->lock, flags);
3989 	return ret;
3990 }
3991 
3992 /*
3993  * Checks if we have enough host controller resources for the default control
3994  * endpoint.
3995  *
3996  * Must be called with xhci->lock held.
3997  */
3998 static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci)
3999 {
4000 	if (xhci->num_active_eps + 1 > xhci->limit_active_eps) {
4001 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
4002 				"Not enough ep ctxs: "
4003 				"%u active, need to add 1, limit is %u.",
4004 				xhci->num_active_eps, xhci->limit_active_eps);
4005 		return -ENOMEM;
4006 	}
4007 	xhci->num_active_eps += 1;
4008 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
4009 			"Adding 1 ep ctx, %u now active.",
4010 			xhci->num_active_eps);
4011 	return 0;
4012 }
4013 
4014 
4015 /*
4016  * Returns 0 if the xHC ran out of device slots, the Enable Slot command
4017  * timed out, or allocating memory failed.  Returns 1 on success.
4018  */
4019 int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
4020 {
4021 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4022 	struct xhci_virt_device *vdev;
4023 	struct xhci_slot_ctx *slot_ctx;
4024 	unsigned long flags;
4025 	int ret, slot_id;
4026 	struct xhci_command *command;
4027 
4028 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
4029 	if (!command)
4030 		return 0;
4031 
4032 	spin_lock_irqsave(&xhci->lock, flags);
4033 	ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);
4034 	if (ret) {
4035 		spin_unlock_irqrestore(&xhci->lock, flags);
4036 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
4037 		xhci_free_command(xhci, command);
4038 		return 0;
4039 	}
4040 	xhci_ring_cmd_db(xhci);
4041 	spin_unlock_irqrestore(&xhci->lock, flags);
4042 
4043 	wait_for_completion(command->completion);
4044 	slot_id = command->slot_id;
4045 
4046 	if (!slot_id || command->status != COMP_SUCCESS) {
4047 		xhci_err(xhci, "Error while assigning device slot ID\n");
4048 		xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",
4049 				HCS_MAX_SLOTS(
4050 					readl(&xhci->cap_regs->hcs_params1)));
4051 		xhci_free_command(xhci, command);
4052 		return 0;
4053 	}
4054 
4055 	xhci_free_command(xhci, command);
4056 
4057 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
4058 		spin_lock_irqsave(&xhci->lock, flags);
4059 		ret = xhci_reserve_host_control_ep_resources(xhci);
4060 		if (ret) {
4061 			spin_unlock_irqrestore(&xhci->lock, flags);
4062 			xhci_warn(xhci, "Not enough host resources, "
4063 					"active endpoint contexts = %u\n",
4064 					xhci->num_active_eps);
4065 			goto disable_slot;
4066 		}
4067 		spin_unlock_irqrestore(&xhci->lock, flags);
4068 	}
4069 	/* Use GFP_NOIO, since this function can be called from
4070 	 * xhci_discover_or_reset_device(), which may be called as part of
4071 	 * mass storage driver error handling.
4072 	 */
4073 	if (!xhci_alloc_virt_device(xhci, slot_id, udev, GFP_NOIO)) {
4074 		xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");
4075 		goto disable_slot;
4076 	}
4077 	vdev = xhci->devs[slot_id];
4078 	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
4079 	trace_xhci_alloc_dev(slot_ctx);
4080 
4081 	udev->slot_id = slot_id;
4082 
4083 	xhci_debugfs_create_slot(xhci, slot_id);
4084 
4085 #ifndef CONFIG_USB_DEFAULT_PERSIST
4086 	/*
4087 	 * If resetting upon resume, we can't put the controller into runtime
4088 	 * suspend if there is a device attached.
4089 	 */
4090 	if (xhci->quirks & XHCI_RESET_ON_RESUME)
4091 		pm_runtime_get_noresume(hcd->self.controller);
4092 #endif
4093 
4094 	/* Is this a LS or FS device under a HS hub? */
4095 	/* Hub or peripherial? */
4096 	return 1;
4097 
4098 disable_slot:
4099 	ret = xhci_disable_slot(xhci, udev->slot_id);
4100 	if (ret)
4101 		xhci_free_virt_device(xhci, udev->slot_id);
4102 
4103 	return 0;
4104 }
4105 
4106 /*
4107  * Issue an Address Device command and optionally send a corresponding
4108  * SetAddress request to the device.
4109  */
4110 static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
4111 			     enum xhci_setup_dev setup)
4112 {
4113 	const char *act = setup == SETUP_CONTEXT_ONLY ? "context" : "address";
4114 	unsigned long flags;
4115 	struct xhci_virt_device *virt_dev;
4116 	int ret = 0;
4117 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4118 	struct xhci_slot_ctx *slot_ctx;
4119 	struct xhci_input_control_ctx *ctrl_ctx;
4120 	u64 temp_64;
4121 	struct xhci_command *command = NULL;
4122 
4123 	mutex_lock(&xhci->mutex);
4124 
4125 	if (xhci->xhc_state) {	/* dying, removing or halted */
4126 		ret = -ESHUTDOWN;
4127 		goto out;
4128 	}
4129 
4130 	if (!udev->slot_id) {
4131 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4132 				"Bad Slot ID %d", udev->slot_id);
4133 		ret = -EINVAL;
4134 		goto out;
4135 	}
4136 
4137 	virt_dev = xhci->devs[udev->slot_id];
4138 
4139 	if (WARN_ON(!virt_dev)) {
4140 		/*
4141 		 * In plug/unplug torture test with an NEC controller,
4142 		 * a zero-dereference was observed once due to virt_dev = 0.
4143 		 * Print useful debug rather than crash if it is observed again!
4144 		 */
4145 		xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n",
4146 			udev->slot_id);
4147 		ret = -EINVAL;
4148 		goto out;
4149 	}
4150 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
4151 	trace_xhci_setup_device_slot(slot_ctx);
4152 
4153 	if (setup == SETUP_CONTEXT_ONLY) {
4154 		if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
4155 		    SLOT_STATE_DEFAULT) {
4156 			xhci_dbg(xhci, "Slot already in default state\n");
4157 			goto out;
4158 		}
4159 	}
4160 
4161 	command = xhci_alloc_command(xhci, true, GFP_KERNEL);
4162 	if (!command) {
4163 		ret = -ENOMEM;
4164 		goto out;
4165 	}
4166 
4167 	command->in_ctx = virt_dev->in_ctx;
4168 
4169 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
4170 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
4171 	if (!ctrl_ctx) {
4172 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4173 				__func__);
4174 		ret = -EINVAL;
4175 		goto out;
4176 	}
4177 	/*
4178 	 * If this is the first Set Address since device plug-in or
4179 	 * virt_device realloaction after a resume with an xHCI power loss,
4180 	 * then set up the slot context.
4181 	 */
4182 	if (!slot_ctx->dev_info)
4183 		xhci_setup_addressable_virt_dev(xhci, udev);
4184 	/* Otherwise, update the control endpoint ring enqueue pointer. */
4185 	else
4186 		xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
4187 	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
4188 	ctrl_ctx->drop_flags = 0;
4189 
4190 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
4191 				le32_to_cpu(slot_ctx->dev_info) >> 27);
4192 
4193 	trace_xhci_address_ctrl_ctx(ctrl_ctx);
4194 	spin_lock_irqsave(&xhci->lock, flags);
4195 	trace_xhci_setup_device(virt_dev);
4196 	ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
4197 					udev->slot_id, setup);
4198 	if (ret) {
4199 		spin_unlock_irqrestore(&xhci->lock, flags);
4200 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4201 				"FIXME: allocate a command ring segment");
4202 		goto out;
4203 	}
4204 	xhci_ring_cmd_db(xhci);
4205 	spin_unlock_irqrestore(&xhci->lock, flags);
4206 
4207 	/* ctrl tx can take up to 5 sec; XXX: need more time for xHC? */
4208 	wait_for_completion(command->completion);
4209 
4210 	/* FIXME: From section 4.3.4: "Software shall be responsible for timing
4211 	 * the SetAddress() "recovery interval" required by USB and aborting the
4212 	 * command on a timeout.
4213 	 */
4214 	switch (command->status) {
4215 	case COMP_COMMAND_ABORTED:
4216 	case COMP_COMMAND_RING_STOPPED:
4217 		xhci_warn(xhci, "Timeout while waiting for setup device command\n");
4218 		ret = -ETIME;
4219 		break;
4220 	case COMP_CONTEXT_STATE_ERROR:
4221 	case COMP_SLOT_NOT_ENABLED_ERROR:
4222 		xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n",
4223 			 act, udev->slot_id);
4224 		ret = -EINVAL;
4225 		break;
4226 	case COMP_USB_TRANSACTION_ERROR:
4227 		dev_warn(&udev->dev, "Device not responding to setup %s.\n", act);
4228 
4229 		mutex_unlock(&xhci->mutex);
4230 		ret = xhci_disable_slot(xhci, udev->slot_id);
4231 		if (!ret)
4232 			xhci_alloc_dev(hcd, udev);
4233 		kfree(command->completion);
4234 		kfree(command);
4235 		return -EPROTO;
4236 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
4237 		dev_warn(&udev->dev,
4238 			 "ERROR: Incompatible device for setup %s command\n", act);
4239 		ret = -ENODEV;
4240 		break;
4241 	case COMP_SUCCESS:
4242 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4243 			       "Successful setup %s command", act);
4244 		break;
4245 	default:
4246 		xhci_err(xhci,
4247 			 "ERROR: unexpected setup %s command completion code 0x%x.\n",
4248 			 act, command->status);
4249 		trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1);
4250 		ret = -EINVAL;
4251 		break;
4252 	}
4253 	if (ret)
4254 		goto out;
4255 	temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
4256 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4257 			"Op regs DCBAA ptr = %#016llx", temp_64);
4258 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4259 		"Slot ID %d dcbaa entry @%p = %#016llx",
4260 		udev->slot_id,
4261 		&xhci->dcbaa->dev_context_ptrs[udev->slot_id],
4262 		(unsigned long long)
4263 		le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));
4264 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4265 			"Output Context DMA address = %#08llx",
4266 			(unsigned long long)virt_dev->out_ctx->dma);
4267 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
4268 				le32_to_cpu(slot_ctx->dev_info) >> 27);
4269 	/*
4270 	 * USB core uses address 1 for the roothubs, so we add one to the
4271 	 * address given back to us by the HC.
4272 	 */
4273 	trace_xhci_address_ctx(xhci, virt_dev->out_ctx,
4274 				le32_to_cpu(slot_ctx->dev_info) >> 27);
4275 	/* Zero the input context control for later use */
4276 	ctrl_ctx->add_flags = 0;
4277 	ctrl_ctx->drop_flags = 0;
4278 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
4279 	udev->devaddr = (u8)(le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
4280 
4281 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
4282 		       "Internal device address = %d",
4283 		       le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
4284 out:
4285 	mutex_unlock(&xhci->mutex);
4286 	if (command) {
4287 		kfree(command->completion);
4288 		kfree(command);
4289 	}
4290 	return ret;
4291 }
4292 
4293 static int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev)
4294 {
4295 	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ADDRESS);
4296 }
4297 
4298 static int xhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev)
4299 {
4300 	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ONLY);
4301 }
4302 
4303 /*
4304  * Transfer the port index into real index in the HW port status
4305  * registers. Caculate offset between the port's PORTSC register
4306  * and port status base. Divide the number of per port register
4307  * to get the real index. The raw port number bases 1.
4308  */
4309 int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1)
4310 {
4311 	struct xhci_hub *rhub;
4312 
4313 	rhub = xhci_get_rhub(hcd);
4314 	return rhub->ports[port1 - 1]->hw_portnum + 1;
4315 }
4316 
4317 /*
4318  * Issue an Evaluate Context command to change the Maximum Exit Latency in the
4319  * slot context.  If that succeeds, store the new MEL in the xhci_virt_device.
4320  */
4321 static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,
4322 			struct usb_device *udev, u16 max_exit_latency)
4323 {
4324 	struct xhci_virt_device *virt_dev;
4325 	struct xhci_command *command;
4326 	struct xhci_input_control_ctx *ctrl_ctx;
4327 	struct xhci_slot_ctx *slot_ctx;
4328 	unsigned long flags;
4329 	int ret;
4330 
4331 	spin_lock_irqsave(&xhci->lock, flags);
4332 
4333 	virt_dev = xhci->devs[udev->slot_id];
4334 
4335 	/*
4336 	 * virt_dev might not exists yet if xHC resumed from hibernate (S4) and
4337 	 * xHC was re-initialized. Exit latency will be set later after
4338 	 * hub_port_finish_reset() is done and xhci->devs[] are re-allocated
4339 	 */
4340 
4341 	if (!virt_dev || max_exit_latency == virt_dev->current_mel) {
4342 		spin_unlock_irqrestore(&xhci->lock, flags);
4343 		return 0;
4344 	}
4345 
4346 	/* Attempt to issue an Evaluate Context command to change the MEL. */
4347 	command = xhci->lpm_command;
4348 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
4349 	if (!ctrl_ctx) {
4350 		spin_unlock_irqrestore(&xhci->lock, flags);
4351 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4352 				__func__);
4353 		return -ENOMEM;
4354 	}
4355 
4356 	xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx);
4357 	spin_unlock_irqrestore(&xhci->lock, flags);
4358 
4359 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
4360 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
4361 	slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));
4362 	slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);
4363 	slot_ctx->dev_state = 0;
4364 
4365 	xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
4366 			"Set up evaluate context for LPM MEL change.");
4367 
4368 	/* Issue and wait for the evaluate context command. */
4369 	ret = xhci_configure_endpoint(xhci, udev, command,
4370 			true, true);
4371 
4372 	if (!ret) {
4373 		spin_lock_irqsave(&xhci->lock, flags);
4374 		virt_dev->current_mel = max_exit_latency;
4375 		spin_unlock_irqrestore(&xhci->lock, flags);
4376 	}
4377 	return ret;
4378 }
4379 
4380 #ifdef CONFIG_PM
4381 
4382 /* BESL to HIRD Encoding array for USB2 LPM */
4383 static int xhci_besl_encoding[16] = {125, 150, 200, 300, 400, 500, 1000, 2000,
4384 	3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};
4385 
4386 /* Calculate HIRD/BESL for USB2 PORTPMSC*/
4387 static int xhci_calculate_hird_besl(struct xhci_hcd *xhci,
4388 					struct usb_device *udev)
4389 {
4390 	int u2del, besl, besl_host;
4391 	int besl_device = 0;
4392 	u32 field;
4393 
4394 	u2del = HCS_U2_LATENCY(xhci->hcs_params3);
4395 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4396 
4397 	if (field & USB_BESL_SUPPORT) {
4398 		for (besl_host = 0; besl_host < 16; besl_host++) {
4399 			if (xhci_besl_encoding[besl_host] >= u2del)
4400 				break;
4401 		}
4402 		/* Use baseline BESL value as default */
4403 		if (field & USB_BESL_BASELINE_VALID)
4404 			besl_device = USB_GET_BESL_BASELINE(field);
4405 		else if (field & USB_BESL_DEEP_VALID)
4406 			besl_device = USB_GET_BESL_DEEP(field);
4407 	} else {
4408 		if (u2del <= 50)
4409 			besl_host = 0;
4410 		else
4411 			besl_host = (u2del - 51) / 75 + 1;
4412 	}
4413 
4414 	besl = besl_host + besl_device;
4415 	if (besl > 15)
4416 		besl = 15;
4417 
4418 	return besl;
4419 }
4420 
4421 /* Calculate BESLD, L1 timeout and HIRDM for USB2 PORTHLPMC */
4422 static int xhci_calculate_usb2_hw_lpm_params(struct usb_device *udev)
4423 {
4424 	u32 field;
4425 	int l1;
4426 	int besld = 0;
4427 	int hirdm = 0;
4428 
4429 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4430 
4431 	/* xHCI l1 is set in steps of 256us, xHCI 1.0 section 5.4.11.2 */
4432 	l1 = udev->l1_params.timeout / 256;
4433 
4434 	/* device has preferred BESLD */
4435 	if (field & USB_BESL_DEEP_VALID) {
4436 		besld = USB_GET_BESL_DEEP(field);
4437 		hirdm = 1;
4438 	}
4439 
4440 	return PORT_BESLD(besld) | PORT_L1_TIMEOUT(l1) | PORT_HIRDM(hirdm);
4441 }
4442 
4443 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
4444 			struct usb_device *udev, int enable)
4445 {
4446 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
4447 	struct xhci_port **ports;
4448 	__le32 __iomem	*pm_addr, *hlpm_addr;
4449 	u32		pm_val, hlpm_val, field;
4450 	unsigned int	port_num;
4451 	unsigned long	flags;
4452 	int		hird, exit_latency;
4453 	int		ret;
4454 
4455 	if (xhci->quirks & XHCI_HW_LPM_DISABLE)
4456 		return -EPERM;
4457 
4458 	if (hcd->speed >= HCD_USB3 || !xhci->hw_lpm_support ||
4459 			!udev->lpm_capable)
4460 		return -EPERM;
4461 
4462 	if (!udev->parent || udev->parent->parent ||
4463 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4464 		return -EPERM;
4465 
4466 	if (udev->usb2_hw_lpm_capable != 1)
4467 		return -EPERM;
4468 
4469 	spin_lock_irqsave(&xhci->lock, flags);
4470 
4471 	ports = xhci->usb2_rhub.ports;
4472 	port_num = udev->portnum - 1;
4473 	pm_addr = ports[port_num]->addr + PORTPMSC;
4474 	pm_val = readl(pm_addr);
4475 	hlpm_addr = ports[port_num]->addr + PORTHLPMC;
4476 
4477 	xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n",
4478 			enable ? "enable" : "disable", port_num + 1);
4479 
4480 	if (enable) {
4481 		/* Host supports BESL timeout instead of HIRD */
4482 		if (udev->usb2_hw_lpm_besl_capable) {
4483 			/* if device doesn't have a preferred BESL value use a
4484 			 * default one which works with mixed HIRD and BESL
4485 			 * systems. See XHCI_DEFAULT_BESL definition in xhci.h
4486 			 */
4487 			field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4488 			if ((field & USB_BESL_SUPPORT) &&
4489 			    (field & USB_BESL_BASELINE_VALID))
4490 				hird = USB_GET_BESL_BASELINE(field);
4491 			else
4492 				hird = udev->l1_params.besl;
4493 
4494 			exit_latency = xhci_besl_encoding[hird];
4495 			spin_unlock_irqrestore(&xhci->lock, flags);
4496 
4497 			/* USB 3.0 code dedicate one xhci->lpm_command->in_ctx
4498 			 * input context for link powermanagement evaluate
4499 			 * context commands. It is protected by hcd->bandwidth
4500 			 * mutex and is shared by all devices. We need to set
4501 			 * the max ext latency in USB 2 BESL LPM as well, so
4502 			 * use the same mutex and xhci_change_max_exit_latency()
4503 			 */
4504 			mutex_lock(hcd->bandwidth_mutex);
4505 			ret = xhci_change_max_exit_latency(xhci, udev,
4506 							   exit_latency);
4507 			mutex_unlock(hcd->bandwidth_mutex);
4508 
4509 			if (ret < 0)
4510 				return ret;
4511 			spin_lock_irqsave(&xhci->lock, flags);
4512 
4513 			hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev);
4514 			writel(hlpm_val, hlpm_addr);
4515 			/* flush write */
4516 			readl(hlpm_addr);
4517 		} else {
4518 			hird = xhci_calculate_hird_besl(xhci, udev);
4519 		}
4520 
4521 		pm_val &= ~PORT_HIRD_MASK;
4522 		pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id);
4523 		writel(pm_val, pm_addr);
4524 		pm_val = readl(pm_addr);
4525 		pm_val |= PORT_HLE;
4526 		writel(pm_val, pm_addr);
4527 		/* flush write */
4528 		readl(pm_addr);
4529 	} else {
4530 		pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK);
4531 		writel(pm_val, pm_addr);
4532 		/* flush write */
4533 		readl(pm_addr);
4534 		if (udev->usb2_hw_lpm_besl_capable) {
4535 			spin_unlock_irqrestore(&xhci->lock, flags);
4536 			mutex_lock(hcd->bandwidth_mutex);
4537 			xhci_change_max_exit_latency(xhci, udev, 0);
4538 			mutex_unlock(hcd->bandwidth_mutex);
4539 			readl_poll_timeout(ports[port_num]->addr, pm_val,
4540 					   (pm_val & PORT_PLS_MASK) == XDEV_U0,
4541 					   100, 10000);
4542 			return 0;
4543 		}
4544 	}
4545 
4546 	spin_unlock_irqrestore(&xhci->lock, flags);
4547 	return 0;
4548 }
4549 
4550 /* check if a usb2 port supports a given extened capability protocol
4551  * only USB2 ports extended protocol capability values are cached.
4552  * Return 1 if capability is supported
4553  */
4554 static int xhci_check_usb2_port_capability(struct xhci_hcd *xhci, int port,
4555 					   unsigned capability)
4556 {
4557 	u32 port_offset, port_count;
4558 	int i;
4559 
4560 	for (i = 0; i < xhci->num_ext_caps; i++) {
4561 		if (xhci->ext_caps[i] & capability) {
4562 			/* port offsets starts at 1 */
4563 			port_offset = XHCI_EXT_PORT_OFF(xhci->ext_caps[i]) - 1;
4564 			port_count = XHCI_EXT_PORT_COUNT(xhci->ext_caps[i]);
4565 			if (port >= port_offset &&
4566 			    port < port_offset + port_count)
4567 				return 1;
4568 		}
4569 	}
4570 	return 0;
4571 }
4572 
4573 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
4574 {
4575 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
4576 	int		portnum = udev->portnum - 1;
4577 
4578 	if (hcd->speed >= HCD_USB3 || !udev->lpm_capable)
4579 		return 0;
4580 
4581 	/* we only support lpm for non-hub device connected to root hub yet */
4582 	if (!udev->parent || udev->parent->parent ||
4583 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4584 		return 0;
4585 
4586 	if (xhci->hw_lpm_support == 1 &&
4587 			xhci_check_usb2_port_capability(
4588 				xhci, portnum, XHCI_HLC)) {
4589 		udev->usb2_hw_lpm_capable = 1;
4590 		udev->l1_params.timeout = XHCI_L1_TIMEOUT;
4591 		udev->l1_params.besl = XHCI_DEFAULT_BESL;
4592 		if (xhci_check_usb2_port_capability(xhci, portnum,
4593 					XHCI_BLC))
4594 			udev->usb2_hw_lpm_besl_capable = 1;
4595 	}
4596 
4597 	return 0;
4598 }
4599 
4600 /*---------------------- USB 3.0 Link PM functions ------------------------*/
4601 
4602 /* Service interval in nanoseconds = 2^(bInterval - 1) * 125us * 1000ns / 1us */
4603 static unsigned long long xhci_service_interval_to_ns(
4604 		struct usb_endpoint_descriptor *desc)
4605 {
4606 	return (1ULL << (desc->bInterval - 1)) * 125 * 1000;
4607 }
4608 
4609 static u16 xhci_get_timeout_no_hub_lpm(struct usb_device *udev,
4610 		enum usb3_link_state state)
4611 {
4612 	unsigned long long sel;
4613 	unsigned long long pel;
4614 	unsigned int max_sel_pel;
4615 	char *state_name;
4616 
4617 	switch (state) {
4618 	case USB3_LPM_U1:
4619 		/* Convert SEL and PEL stored in nanoseconds to microseconds */
4620 		sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
4621 		pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);
4622 		max_sel_pel = USB3_LPM_MAX_U1_SEL_PEL;
4623 		state_name = "U1";
4624 		break;
4625 	case USB3_LPM_U2:
4626 		sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
4627 		pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);
4628 		max_sel_pel = USB3_LPM_MAX_U2_SEL_PEL;
4629 		state_name = "U2";
4630 		break;
4631 	default:
4632 		dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n",
4633 				__func__);
4634 		return USB3_LPM_DISABLED;
4635 	}
4636 
4637 	if (sel <= max_sel_pel && pel <= max_sel_pel)
4638 		return USB3_LPM_DEVICE_INITIATED;
4639 
4640 	if (sel > max_sel_pel)
4641 		dev_dbg(&udev->dev, "Device-initiated %s disabled "
4642 				"due to long SEL %llu ms\n",
4643 				state_name, sel);
4644 	else
4645 		dev_dbg(&udev->dev, "Device-initiated %s disabled "
4646 				"due to long PEL %llu ms\n",
4647 				state_name, pel);
4648 	return USB3_LPM_DISABLED;
4649 }
4650 
4651 /* The U1 timeout should be the maximum of the following values:
4652  *  - For control endpoints, U1 system exit latency (SEL) * 3
4653  *  - For bulk endpoints, U1 SEL * 5
4654  *  - For interrupt endpoints:
4655  *    - Notification EPs, U1 SEL * 3
4656  *    - Periodic EPs, max(105% of bInterval, U1 SEL * 2)
4657  *  - For isochronous endpoints, max(105% of bInterval, U1 SEL * 2)
4658  */
4659 static unsigned long long xhci_calculate_intel_u1_timeout(
4660 		struct usb_device *udev,
4661 		struct usb_endpoint_descriptor *desc)
4662 {
4663 	unsigned long long timeout_ns;
4664 	int ep_type;
4665 	int intr_type;
4666 
4667 	ep_type = usb_endpoint_type(desc);
4668 	switch (ep_type) {
4669 	case USB_ENDPOINT_XFER_CONTROL:
4670 		timeout_ns = udev->u1_params.sel * 3;
4671 		break;
4672 	case USB_ENDPOINT_XFER_BULK:
4673 		timeout_ns = udev->u1_params.sel * 5;
4674 		break;
4675 	case USB_ENDPOINT_XFER_INT:
4676 		intr_type = usb_endpoint_interrupt_type(desc);
4677 		if (intr_type == USB_ENDPOINT_INTR_NOTIFICATION) {
4678 			timeout_ns = udev->u1_params.sel * 3;
4679 			break;
4680 		}
4681 		/* Otherwise the calculation is the same as isoc eps */
4682 		fallthrough;
4683 	case USB_ENDPOINT_XFER_ISOC:
4684 		timeout_ns = xhci_service_interval_to_ns(desc);
4685 		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns * 105, 100);
4686 		if (timeout_ns < udev->u1_params.sel * 2)
4687 			timeout_ns = udev->u1_params.sel * 2;
4688 		break;
4689 	default:
4690 		return 0;
4691 	}
4692 
4693 	return timeout_ns;
4694 }
4695 
4696 /* Returns the hub-encoded U1 timeout value. */
4697 static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,
4698 		struct usb_device *udev,
4699 		struct usb_endpoint_descriptor *desc)
4700 {
4701 	unsigned long long timeout_ns;
4702 
4703 	if (xhci->quirks & XHCI_INTEL_HOST)
4704 		timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc);
4705 	else
4706 		timeout_ns = udev->u1_params.sel;
4707 
4708 	/* Prevent U1 if service interval is shorter than U1 exit latency */
4709 	if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4710 		if (xhci_service_interval_to_ns(desc) <= timeout_ns) {
4711 			dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n");
4712 			return USB3_LPM_DISABLED;
4713 		}
4714 	}
4715 
4716 	/* The U1 timeout is encoded in 1us intervals.
4717 	 * Don't return a timeout of zero, because that's USB3_LPM_DISABLED.
4718 	 */
4719 	if (timeout_ns == USB3_LPM_DISABLED)
4720 		timeout_ns = 1;
4721 	else
4722 		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 1000);
4723 
4724 	/* If the necessary timeout value is bigger than what we can set in the
4725 	 * USB 3.0 hub, we have to disable hub-initiated U1.
4726 	 */
4727 	if (timeout_ns <= USB3_LPM_U1_MAX_TIMEOUT)
4728 		return timeout_ns;
4729 	dev_dbg(&udev->dev, "Hub-initiated U1 disabled "
4730 			"due to long timeout %llu ms\n", timeout_ns);
4731 	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U1);
4732 }
4733 
4734 /* The U2 timeout should be the maximum of:
4735  *  - 10 ms (to avoid the bandwidth impact on the scheduler)
4736  *  - largest bInterval of any active periodic endpoint (to avoid going
4737  *    into lower power link states between intervals).
4738  *  - the U2 Exit Latency of the device
4739  */
4740 static unsigned long long xhci_calculate_intel_u2_timeout(
4741 		struct usb_device *udev,
4742 		struct usb_endpoint_descriptor *desc)
4743 {
4744 	unsigned long long timeout_ns;
4745 	unsigned long long u2_del_ns;
4746 
4747 	timeout_ns = 10 * 1000 * 1000;
4748 
4749 	if ((usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) &&
4750 			(xhci_service_interval_to_ns(desc) > timeout_ns))
4751 		timeout_ns = xhci_service_interval_to_ns(desc);
4752 
4753 	u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL;
4754 	if (u2_del_ns > timeout_ns)
4755 		timeout_ns = u2_del_ns;
4756 
4757 	return timeout_ns;
4758 }
4759 
4760 /* Returns the hub-encoded U2 timeout value. */
4761 static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,
4762 		struct usb_device *udev,
4763 		struct usb_endpoint_descriptor *desc)
4764 {
4765 	unsigned long long timeout_ns;
4766 
4767 	if (xhci->quirks & XHCI_INTEL_HOST)
4768 		timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc);
4769 	else
4770 		timeout_ns = udev->u2_params.sel;
4771 
4772 	/* Prevent U2 if service interval is shorter than U2 exit latency */
4773 	if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
4774 		if (xhci_service_interval_to_ns(desc) <= timeout_ns) {
4775 			dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n");
4776 			return USB3_LPM_DISABLED;
4777 		}
4778 	}
4779 
4780 	/* The U2 timeout is encoded in 256us intervals */
4781 	timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000);
4782 	/* If the necessary timeout value is bigger than what we can set in the
4783 	 * USB 3.0 hub, we have to disable hub-initiated U2.
4784 	 */
4785 	if (timeout_ns <= USB3_LPM_U2_MAX_TIMEOUT)
4786 		return timeout_ns;
4787 	dev_dbg(&udev->dev, "Hub-initiated U2 disabled "
4788 			"due to long timeout %llu ms\n", timeout_ns);
4789 	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U2);
4790 }
4791 
4792 static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4793 		struct usb_device *udev,
4794 		struct usb_endpoint_descriptor *desc,
4795 		enum usb3_link_state state,
4796 		u16 *timeout)
4797 {
4798 	if (state == USB3_LPM_U1)
4799 		return xhci_calculate_u1_timeout(xhci, udev, desc);
4800 	else if (state == USB3_LPM_U2)
4801 		return xhci_calculate_u2_timeout(xhci, udev, desc);
4802 
4803 	return USB3_LPM_DISABLED;
4804 }
4805 
4806 static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4807 		struct usb_device *udev,
4808 		struct usb_endpoint_descriptor *desc,
4809 		enum usb3_link_state state,
4810 		u16 *timeout)
4811 {
4812 	u16 alt_timeout;
4813 
4814 	alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev,
4815 		desc, state, timeout);
4816 
4817 	/* If we found we can't enable hub-initiated LPM, and
4818 	 * the U1 or U2 exit latency was too high to allow
4819 	 * device-initiated LPM as well, then we will disable LPM
4820 	 * for this device, so stop searching any further.
4821 	 */
4822 	if (alt_timeout == USB3_LPM_DISABLED) {
4823 		*timeout = alt_timeout;
4824 		return -E2BIG;
4825 	}
4826 	if (alt_timeout > *timeout)
4827 		*timeout = alt_timeout;
4828 	return 0;
4829 }
4830 
4831 static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci,
4832 		struct usb_device *udev,
4833 		struct usb_host_interface *alt,
4834 		enum usb3_link_state state,
4835 		u16 *timeout)
4836 {
4837 	int j;
4838 
4839 	for (j = 0; j < alt->desc.bNumEndpoints; j++) {
4840 		if (xhci_update_timeout_for_endpoint(xhci, udev,
4841 					&alt->endpoint[j].desc, state, timeout))
4842 			return -E2BIG;
4843 		continue;
4844 	}
4845 	return 0;
4846 }
4847 
4848 static int xhci_check_intel_tier_policy(struct usb_device *udev,
4849 		enum usb3_link_state state)
4850 {
4851 	struct usb_device *parent;
4852 	unsigned int num_hubs;
4853 
4854 	if (state == USB3_LPM_U2)
4855 		return 0;
4856 
4857 	/* Don't enable U1 if the device is on a 2nd tier hub or lower. */
4858 	for (parent = udev->parent, num_hubs = 0; parent->parent;
4859 			parent = parent->parent)
4860 		num_hubs++;
4861 
4862 	if (num_hubs < 2)
4863 		return 0;
4864 
4865 	dev_dbg(&udev->dev, "Disabling U1 link state for device"
4866 			" below second-tier hub.\n");
4867 	dev_dbg(&udev->dev, "Plug device into first-tier hub "
4868 			"to decrease power consumption.\n");
4869 	return -E2BIG;
4870 }
4871 
4872 static int xhci_check_tier_policy(struct xhci_hcd *xhci,
4873 		struct usb_device *udev,
4874 		enum usb3_link_state state)
4875 {
4876 	if (xhci->quirks & XHCI_INTEL_HOST)
4877 		return xhci_check_intel_tier_policy(udev, state);
4878 	else
4879 		return 0;
4880 }
4881 
4882 /* Returns the U1 or U2 timeout that should be enabled.
4883  * If the tier check or timeout setting functions return with a non-zero exit
4884  * code, that means the timeout value has been finalized and we shouldn't look
4885  * at any more endpoints.
4886  */
4887 static u16 xhci_calculate_lpm_timeout(struct usb_hcd *hcd,
4888 			struct usb_device *udev, enum usb3_link_state state)
4889 {
4890 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4891 	struct usb_host_config *config;
4892 	char *state_name;
4893 	int i;
4894 	u16 timeout = USB3_LPM_DISABLED;
4895 
4896 	if (state == USB3_LPM_U1)
4897 		state_name = "U1";
4898 	else if (state == USB3_LPM_U2)
4899 		state_name = "U2";
4900 	else {
4901 		dev_warn(&udev->dev, "Can't enable unknown link state %i\n",
4902 				state);
4903 		return timeout;
4904 	}
4905 
4906 	if (xhci_check_tier_policy(xhci, udev, state) < 0)
4907 		return timeout;
4908 
4909 	/* Gather some information about the currently installed configuration
4910 	 * and alternate interface settings.
4911 	 */
4912 	if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc,
4913 			state, &timeout))
4914 		return timeout;
4915 
4916 	config = udev->actconfig;
4917 	if (!config)
4918 		return timeout;
4919 
4920 	for (i = 0; i < config->desc.bNumInterfaces; i++) {
4921 		struct usb_driver *driver;
4922 		struct usb_interface *intf = config->interface[i];
4923 
4924 		if (!intf)
4925 			continue;
4926 
4927 		/* Check if any currently bound drivers want hub-initiated LPM
4928 		 * disabled.
4929 		 */
4930 		if (intf->dev.driver) {
4931 			driver = to_usb_driver(intf->dev.driver);
4932 			if (driver && driver->disable_hub_initiated_lpm) {
4933 				dev_dbg(&udev->dev, "Hub-initiated %s disabled at request of driver %s\n",
4934 					state_name, driver->name);
4935 				timeout = xhci_get_timeout_no_hub_lpm(udev,
4936 								      state);
4937 				if (timeout == USB3_LPM_DISABLED)
4938 					return timeout;
4939 			}
4940 		}
4941 
4942 		/* Not sure how this could happen... */
4943 		if (!intf->cur_altsetting)
4944 			continue;
4945 
4946 		if (xhci_update_timeout_for_interface(xhci, udev,
4947 					intf->cur_altsetting,
4948 					state, &timeout))
4949 			return timeout;
4950 	}
4951 	return timeout;
4952 }
4953 
4954 static int calculate_max_exit_latency(struct usb_device *udev,
4955 		enum usb3_link_state state_changed,
4956 		u16 hub_encoded_timeout)
4957 {
4958 	unsigned long long u1_mel_us = 0;
4959 	unsigned long long u2_mel_us = 0;
4960 	unsigned long long mel_us = 0;
4961 	bool disabling_u1;
4962 	bool disabling_u2;
4963 	bool enabling_u1;
4964 	bool enabling_u2;
4965 
4966 	disabling_u1 = (state_changed == USB3_LPM_U1 &&
4967 			hub_encoded_timeout == USB3_LPM_DISABLED);
4968 	disabling_u2 = (state_changed == USB3_LPM_U2 &&
4969 			hub_encoded_timeout == USB3_LPM_DISABLED);
4970 
4971 	enabling_u1 = (state_changed == USB3_LPM_U1 &&
4972 			hub_encoded_timeout != USB3_LPM_DISABLED);
4973 	enabling_u2 = (state_changed == USB3_LPM_U2 &&
4974 			hub_encoded_timeout != USB3_LPM_DISABLED);
4975 
4976 	/* If U1 was already enabled and we're not disabling it,
4977 	 * or we're going to enable U1, account for the U1 max exit latency.
4978 	 */
4979 	if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) ||
4980 			enabling_u1)
4981 		u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000);
4982 	if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) ||
4983 			enabling_u2)
4984 		u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000);
4985 
4986 	if (u1_mel_us > u2_mel_us)
4987 		mel_us = u1_mel_us;
4988 	else
4989 		mel_us = u2_mel_us;
4990 	/* xHCI host controller max exit latency field is only 16 bits wide. */
4991 	if (mel_us > MAX_EXIT) {
4992 		dev_warn(&udev->dev, "Link PM max exit latency of %lluus "
4993 				"is too big.\n", mel_us);
4994 		return -E2BIG;
4995 	}
4996 	return mel_us;
4997 }
4998 
4999 /* Returns the USB3 hub-encoded value for the U1/U2 timeout. */
5000 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
5001 			struct usb_device *udev, enum usb3_link_state state)
5002 {
5003 	struct xhci_hcd	*xhci;
5004 	u16 hub_encoded_timeout;
5005 	int mel;
5006 	int ret;
5007 
5008 	xhci = hcd_to_xhci(hcd);
5009 	/* The LPM timeout values are pretty host-controller specific, so don't
5010 	 * enable hub-initiated timeouts unless the vendor has provided
5011 	 * information about their timeout algorithm.
5012 	 */
5013 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
5014 			!xhci->devs[udev->slot_id])
5015 		return USB3_LPM_DISABLED;
5016 
5017 	hub_encoded_timeout = xhci_calculate_lpm_timeout(hcd, udev, state);
5018 	mel = calculate_max_exit_latency(udev, state, hub_encoded_timeout);
5019 	if (mel < 0) {
5020 		/* Max Exit Latency is too big, disable LPM. */
5021 		hub_encoded_timeout = USB3_LPM_DISABLED;
5022 		mel = 0;
5023 	}
5024 
5025 	ret = xhci_change_max_exit_latency(xhci, udev, mel);
5026 	if (ret)
5027 		return ret;
5028 	return hub_encoded_timeout;
5029 }
5030 
5031 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
5032 			struct usb_device *udev, enum usb3_link_state state)
5033 {
5034 	struct xhci_hcd	*xhci;
5035 	u16 mel;
5036 
5037 	xhci = hcd_to_xhci(hcd);
5038 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
5039 			!xhci->devs[udev->slot_id])
5040 		return 0;
5041 
5042 	mel = calculate_max_exit_latency(udev, state, USB3_LPM_DISABLED);
5043 	return xhci_change_max_exit_latency(xhci, udev, mel);
5044 }
5045 #else /* CONFIG_PM */
5046 
5047 static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
5048 				struct usb_device *udev, int enable)
5049 {
5050 	return 0;
5051 }
5052 
5053 static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
5054 {
5055 	return 0;
5056 }
5057 
5058 static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
5059 			struct usb_device *udev, enum usb3_link_state state)
5060 {
5061 	return USB3_LPM_DISABLED;
5062 }
5063 
5064 static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
5065 			struct usb_device *udev, enum usb3_link_state state)
5066 {
5067 	return 0;
5068 }
5069 #endif	/* CONFIG_PM */
5070 
5071 /*-------------------------------------------------------------------------*/
5072 
5073 /* Once a hub descriptor is fetched for a device, we need to update the xHC's
5074  * internal data structures for the device.
5075  */
5076 static int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
5077 			struct usb_tt *tt, gfp_t mem_flags)
5078 {
5079 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
5080 	struct xhci_virt_device *vdev;
5081 	struct xhci_command *config_cmd;
5082 	struct xhci_input_control_ctx *ctrl_ctx;
5083 	struct xhci_slot_ctx *slot_ctx;
5084 	unsigned long flags;
5085 	unsigned think_time;
5086 	int ret;
5087 
5088 	/* Ignore root hubs */
5089 	if (!hdev->parent)
5090 		return 0;
5091 
5092 	vdev = xhci->devs[hdev->slot_id];
5093 	if (!vdev) {
5094 		xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");
5095 		return -EINVAL;
5096 	}
5097 
5098 	config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
5099 	if (!config_cmd)
5100 		return -ENOMEM;
5101 
5102 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
5103 	if (!ctrl_ctx) {
5104 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
5105 				__func__);
5106 		xhci_free_command(xhci, config_cmd);
5107 		return -ENOMEM;
5108 	}
5109 
5110 	spin_lock_irqsave(&xhci->lock, flags);
5111 	if (hdev->speed == USB_SPEED_HIGH &&
5112 			xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) {
5113 		xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n");
5114 		xhci_free_command(xhci, config_cmd);
5115 		spin_unlock_irqrestore(&xhci->lock, flags);
5116 		return -ENOMEM;
5117 	}
5118 
5119 	xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
5120 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
5121 	slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
5122 	slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);
5123 	/*
5124 	 * refer to section 6.2.2: MTT should be 0 for full speed hub,
5125 	 * but it may be already set to 1 when setup an xHCI virtual
5126 	 * device, so clear it anyway.
5127 	 */
5128 	if (tt->multi)
5129 		slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
5130 	else if (hdev->speed == USB_SPEED_FULL)
5131 		slot_ctx->dev_info &= cpu_to_le32(~DEV_MTT);
5132 
5133 	if (xhci->hci_version > 0x95) {
5134 		xhci_dbg(xhci, "xHCI version %x needs hub "
5135 				"TT think time and number of ports\n",
5136 				(unsigned int) xhci->hci_version);
5137 		slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild));
5138 		/* Set TT think time - convert from ns to FS bit times.
5139 		 * 0 = 8 FS bit times, 1 = 16 FS bit times,
5140 		 * 2 = 24 FS bit times, 3 = 32 FS bit times.
5141 		 *
5142 		 * xHCI 1.0: this field shall be 0 if the device is not a
5143 		 * High-spped hub.
5144 		 */
5145 		think_time = tt->think_time;
5146 		if (think_time != 0)
5147 			think_time = (think_time / 666) - 1;
5148 		if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH)
5149 			slot_ctx->tt_info |=
5150 				cpu_to_le32(TT_THINK_TIME(think_time));
5151 	} else {
5152 		xhci_dbg(xhci, "xHCI version %x doesn't need hub "
5153 				"TT think time or number of ports\n",
5154 				(unsigned int) xhci->hci_version);
5155 	}
5156 	slot_ctx->dev_state = 0;
5157 	spin_unlock_irqrestore(&xhci->lock, flags);
5158 
5159 	xhci_dbg(xhci, "Set up %s for hub device.\n",
5160 			(xhci->hci_version > 0x95) ?
5161 			"configure endpoint" : "evaluate context");
5162 
5163 	/* Issue and wait for the configure endpoint or
5164 	 * evaluate context command.
5165 	 */
5166 	if (xhci->hci_version > 0x95)
5167 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
5168 				false, false);
5169 	else
5170 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
5171 				true, false);
5172 
5173 	xhci_free_command(xhci, config_cmd);
5174 	return ret;
5175 }
5176 
5177 static int xhci_get_frame(struct usb_hcd *hcd)
5178 {
5179 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
5180 	/* EHCI mods by the periodic size.  Why? */
5181 	return readl(&xhci->run_regs->microframe_index) >> 3;
5182 }
5183 
5184 int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
5185 {
5186 	struct xhci_hcd		*xhci;
5187 	/*
5188 	 * TODO: Check with DWC3 clients for sysdev according to
5189 	 * quirks
5190 	 */
5191 	struct device		*dev = hcd->self.sysdev;
5192 	unsigned int		minor_rev;
5193 	int			retval;
5194 
5195 	/* Accept arbitrarily long scatter-gather lists */
5196 	hcd->self.sg_tablesize = ~0;
5197 
5198 	/* support to build packet from discontinuous buffers */
5199 	hcd->self.no_sg_constraint = 1;
5200 
5201 	/* XHCI controllers don't stop the ep queue on short packets :| */
5202 	hcd->self.no_stop_on_short = 1;
5203 
5204 	xhci = hcd_to_xhci(hcd);
5205 
5206 	if (usb_hcd_is_primary_hcd(hcd)) {
5207 		xhci->main_hcd = hcd;
5208 		xhci->usb2_rhub.hcd = hcd;
5209 		/* Mark the first roothub as being USB 2.0.
5210 		 * The xHCI driver will register the USB 3.0 roothub.
5211 		 */
5212 		hcd->speed = HCD_USB2;
5213 		hcd->self.root_hub->speed = USB_SPEED_HIGH;
5214 		/*
5215 		 * USB 2.0 roothub under xHCI has an integrated TT,
5216 		 * (rate matching hub) as opposed to having an OHCI/UHCI
5217 		 * companion controller.
5218 		 */
5219 		hcd->has_tt = 1;
5220 	} else {
5221 		/*
5222 		 * Early xHCI 1.1 spec did not mention USB 3.1 capable hosts
5223 		 * should return 0x31 for sbrn, or that the minor revision
5224 		 * is a two digit BCD containig minor and sub-minor numbers.
5225 		 * This was later clarified in xHCI 1.2.
5226 		 *
5227 		 * Some USB 3.1 capable hosts therefore have sbrn 0x30, and
5228 		 * minor revision set to 0x1 instead of 0x10.
5229 		 */
5230 		if (xhci->usb3_rhub.min_rev == 0x1)
5231 			minor_rev = 1;
5232 		else
5233 			minor_rev = xhci->usb3_rhub.min_rev / 0x10;
5234 
5235 		switch (minor_rev) {
5236 		case 2:
5237 			hcd->speed = HCD_USB32;
5238 			hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
5239 			hcd->self.root_hub->rx_lanes = 2;
5240 			hcd->self.root_hub->tx_lanes = 2;
5241 			hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x2;
5242 			break;
5243 		case 1:
5244 			hcd->speed = HCD_USB31;
5245 			hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
5246 			hcd->self.root_hub->ssp_rate = USB_SSP_GEN_2x1;
5247 			break;
5248 		}
5249 		xhci_info(xhci, "Host supports USB 3.%x %sSuperSpeed\n",
5250 			  minor_rev,
5251 			  minor_rev ? "Enhanced " : "");
5252 
5253 		xhci->usb3_rhub.hcd = hcd;
5254 		/* xHCI private pointer was set in xhci_pci_probe for the second
5255 		 * registered roothub.
5256 		 */
5257 		return 0;
5258 	}
5259 
5260 	mutex_init(&xhci->mutex);
5261 	xhci->cap_regs = hcd->regs;
5262 	xhci->op_regs = hcd->regs +
5263 		HC_LENGTH(readl(&xhci->cap_regs->hc_capbase));
5264 	xhci->run_regs = hcd->regs +
5265 		(readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK);
5266 	/* Cache read-only capability registers */
5267 	xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1);
5268 	xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2);
5269 	xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3);
5270 	xhci->hcc_params = readl(&xhci->cap_regs->hc_capbase);
5271 	xhci->hci_version = HC_VERSION(xhci->hcc_params);
5272 	xhci->hcc_params = readl(&xhci->cap_regs->hcc_params);
5273 	if (xhci->hci_version > 0x100)
5274 		xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
5275 
5276 	xhci->quirks |= quirks;
5277 
5278 	get_quirks(dev, xhci);
5279 
5280 	/* In xhci controllers which follow xhci 1.0 spec gives a spurious
5281 	 * success event after a short transfer. This quirk will ignore such
5282 	 * spurious event.
5283 	 */
5284 	if (xhci->hci_version > 0x96)
5285 		xhci->quirks |= XHCI_SPURIOUS_SUCCESS;
5286 
5287 	/* Make sure the HC is halted. */
5288 	retval = xhci_halt(xhci);
5289 	if (retval)
5290 		return retval;
5291 
5292 	xhci_zero_64b_regs(xhci);
5293 
5294 	xhci_dbg(xhci, "Resetting HCD\n");
5295 	/* Reset the internal HC memory state and registers. */
5296 	retval = xhci_reset(xhci);
5297 	if (retval)
5298 		return retval;
5299 	xhci_dbg(xhci, "Reset complete\n");
5300 
5301 	/*
5302 	 * On some xHCI controllers (e.g. R-Car SoCs), the AC64 bit (bit 0)
5303 	 * of HCCPARAMS1 is set to 1. However, the xHCs don't support 64-bit
5304 	 * address memory pointers actually. So, this driver clears the AC64
5305 	 * bit of xhci->hcc_params to call dma_set_coherent_mask(dev,
5306 	 * DMA_BIT_MASK(32)) in this xhci_gen_setup().
5307 	 */
5308 	if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)
5309 		xhci->hcc_params &= ~BIT(0);
5310 
5311 	/* Set dma_mask and coherent_dma_mask to 64-bits,
5312 	 * if xHC supports 64-bit addressing */
5313 	if (HCC_64BIT_ADDR(xhci->hcc_params) &&
5314 			!dma_set_mask(dev, DMA_BIT_MASK(64))) {
5315 		xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
5316 		dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
5317 	} else {
5318 		/*
5319 		 * This is to avoid error in cases where a 32-bit USB
5320 		 * controller is used on a 64-bit capable system.
5321 		 */
5322 		retval = dma_set_mask(dev, DMA_BIT_MASK(32));
5323 		if (retval)
5324 			return retval;
5325 		xhci_dbg(xhci, "Enabling 32-bit DMA addresses.\n");
5326 		dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
5327 	}
5328 
5329 	xhci_dbg(xhci, "Calling HCD init\n");
5330 	/* Initialize HCD and host controller data structures. */
5331 	retval = xhci_init(hcd);
5332 	if (retval)
5333 		return retval;
5334 	xhci_dbg(xhci, "Called HCD init\n");
5335 
5336 	xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%016llx\n",
5337 		  xhci->hcc_params, xhci->hci_version, xhci->quirks);
5338 
5339 	return 0;
5340 }
5341 EXPORT_SYMBOL_GPL(xhci_gen_setup);
5342 
5343 static void xhci_clear_tt_buffer_complete(struct usb_hcd *hcd,
5344 		struct usb_host_endpoint *ep)
5345 {
5346 	struct xhci_hcd *xhci;
5347 	struct usb_device *udev;
5348 	unsigned int slot_id;
5349 	unsigned int ep_index;
5350 	unsigned long flags;
5351 
5352 	xhci = hcd_to_xhci(hcd);
5353 
5354 	spin_lock_irqsave(&xhci->lock, flags);
5355 	udev = (struct usb_device *)ep->hcpriv;
5356 	slot_id = udev->slot_id;
5357 	ep_index = xhci_get_endpoint_index(&ep->desc);
5358 
5359 	xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_CLEARING_TT;
5360 	xhci_ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
5361 	spin_unlock_irqrestore(&xhci->lock, flags);
5362 }
5363 
5364 static const struct hc_driver xhci_hc_driver = {
5365 	.description =		"xhci-hcd",
5366 	.product_desc =		"xHCI Host Controller",
5367 	.hcd_priv_size =	sizeof(struct xhci_hcd),
5368 
5369 	/*
5370 	 * generic hardware linkage
5371 	 */
5372 	.irq =			xhci_irq,
5373 	.flags =		HCD_MEMORY | HCD_DMA | HCD_USB3 | HCD_SHARED |
5374 				HCD_BH,
5375 
5376 	/*
5377 	 * basic lifecycle operations
5378 	 */
5379 	.reset =		NULL, /* set in xhci_init_driver() */
5380 	.start =		xhci_run,
5381 	.stop =			xhci_stop,
5382 	.shutdown =		xhci_shutdown,
5383 
5384 	/*
5385 	 * managing i/o requests and associated device resources
5386 	 */
5387 	.map_urb_for_dma =      xhci_map_urb_for_dma,
5388 	.unmap_urb_for_dma =    xhci_unmap_urb_for_dma,
5389 	.urb_enqueue =		xhci_urb_enqueue,
5390 	.urb_dequeue =		xhci_urb_dequeue,
5391 	.alloc_dev =		xhci_alloc_dev,
5392 	.free_dev =		xhci_free_dev,
5393 	.alloc_streams =	xhci_alloc_streams,
5394 	.free_streams =		xhci_free_streams,
5395 	.add_endpoint =		xhci_add_endpoint,
5396 	.drop_endpoint =	xhci_drop_endpoint,
5397 	.endpoint_disable =	xhci_endpoint_disable,
5398 	.endpoint_reset =	xhci_endpoint_reset,
5399 	.check_bandwidth =	xhci_check_bandwidth,
5400 	.reset_bandwidth =	xhci_reset_bandwidth,
5401 	.address_device =	xhci_address_device,
5402 	.enable_device =	xhci_enable_device,
5403 	.update_hub_device =	xhci_update_hub_device,
5404 	.reset_device =		xhci_discover_or_reset_device,
5405 
5406 	/*
5407 	 * scheduling support
5408 	 */
5409 	.get_frame_number =	xhci_get_frame,
5410 
5411 	/*
5412 	 * root hub support
5413 	 */
5414 	.hub_control =		xhci_hub_control,
5415 	.hub_status_data =	xhci_hub_status_data,
5416 	.bus_suspend =		xhci_bus_suspend,
5417 	.bus_resume =		xhci_bus_resume,
5418 	.get_resuming_ports =	xhci_get_resuming_ports,
5419 
5420 	/*
5421 	 * call back when device connected and addressed
5422 	 */
5423 	.update_device =        xhci_update_device,
5424 	.set_usb2_hw_lpm =	xhci_set_usb2_hardware_lpm,
5425 	.enable_usb3_lpm_timeout =	xhci_enable_usb3_lpm_timeout,
5426 	.disable_usb3_lpm_timeout =	xhci_disable_usb3_lpm_timeout,
5427 	.find_raw_port_number =	xhci_find_raw_port_number,
5428 	.clear_tt_buffer_complete = xhci_clear_tt_buffer_complete,
5429 };
5430 
5431 void xhci_init_driver(struct hc_driver *drv,
5432 		      const struct xhci_driver_overrides *over)
5433 {
5434 	BUG_ON(!over);
5435 
5436 	/* Copy the generic table to drv then apply the overrides */
5437 	*drv = xhci_hc_driver;
5438 
5439 	if (over) {
5440 		drv->hcd_priv_size += over->extra_priv_size;
5441 		if (over->reset)
5442 			drv->reset = over->reset;
5443 		if (over->start)
5444 			drv->start = over->start;
5445 		if (over->add_endpoint)
5446 			drv->add_endpoint = over->add_endpoint;
5447 		if (over->drop_endpoint)
5448 			drv->drop_endpoint = over->drop_endpoint;
5449 		if (over->check_bandwidth)
5450 			drv->check_bandwidth = over->check_bandwidth;
5451 		if (over->reset_bandwidth)
5452 			drv->reset_bandwidth = over->reset_bandwidth;
5453 	}
5454 }
5455 EXPORT_SYMBOL_GPL(xhci_init_driver);
5456 
5457 MODULE_DESCRIPTION(DRIVER_DESC);
5458 MODULE_AUTHOR(DRIVER_AUTHOR);
5459 MODULE_LICENSE("GPL");
5460 
5461 static int __init xhci_hcd_init(void)
5462 {
5463 	/*
5464 	 * Check the compiler generated sizes of structures that must be laid
5465 	 * out in specific ways for hardware access.
5466 	 */
5467 	BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
5468 	BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8);
5469 	BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8);
5470 	/* xhci_device_control has eight fields, and also
5471 	 * embeds one xhci_slot_ctx and 31 xhci_ep_ctx
5472 	 */
5473 	BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8);
5474 	BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8);
5475 	BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8);
5476 	BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 8*32/8);
5477 	BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8);
5478 	/* xhci_run_regs has eight fields and embeds 128 xhci_intr_regs */
5479 	BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8);
5480 
5481 	if (usb_disabled())
5482 		return -ENODEV;
5483 
5484 	xhci_debugfs_create_root();
5485 
5486 	return 0;
5487 }
5488 
5489 /*
5490  * If an init function is provided, an exit function must also be provided
5491  * to allow module unload.
5492  */
5493 static void __exit xhci_hcd_fini(void)
5494 {
5495 	xhci_debugfs_remove_root();
5496 }
5497 
5498 module_init(xhci_hcd_init);
5499 module_exit(xhci_hcd_fini);
5500