xref: /openbmc/linux/drivers/usb/host/xhci.c (revision d7a3d85e)
1 /*
2  * xHCI host controller driver
3  *
4  * Copyright (C) 2008 Intel Corp.
5  *
6  * Author: Sarah Sharp
7  * Some code borrowed from the Linux EHCI driver.
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License version 2 as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
16  * for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software Foundation,
20  * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21  */
22 
23 #include <linux/pci.h>
24 #include <linux/irq.h>
25 #include <linux/log2.h>
26 #include <linux/module.h>
27 #include <linux/moduleparam.h>
28 #include <linux/slab.h>
29 #include <linux/dmi.h>
30 #include <linux/dma-mapping.h>
31 
32 #include "xhci.h"
33 #include "xhci-trace.h"
34 
35 #define DRIVER_AUTHOR "Sarah Sharp"
36 #define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"
37 
38 #define	PORT_WAKE_BITS	(PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E)
39 
40 /* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */
41 static int link_quirk;
42 module_param(link_quirk, int, S_IRUGO | S_IWUSR);
43 MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");
44 
45 static unsigned int quirks;
46 module_param(quirks, uint, S_IRUGO);
47 MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default");
48 
49 /* TODO: copied from ehci-hcd.c - can this be refactored? */
50 /*
51  * xhci_handshake - spin reading hc until handshake completes or fails
52  * @ptr: address of hc register to be read
53  * @mask: bits to look at in result of read
54  * @done: value of those bits when handshake succeeds
55  * @usec: timeout in microseconds
56  *
57  * Returns negative errno, or zero on success
58  *
59  * Success happens when the "mask" bits have the specified value (hardware
60  * handshake done).  There are two failure modes:  "usec" have passed (major
61  * hardware flakeout), or the register reads as all-ones (hardware removed).
62  */
63 int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec)
64 {
65 	u32	result;
66 
67 	do {
68 		result = readl(ptr);
69 		if (result == ~(u32)0)		/* card removed */
70 			return -ENODEV;
71 		result &= mask;
72 		if (result == done)
73 			return 0;
74 		udelay(1);
75 		usec--;
76 	} while (usec > 0);
77 	return -ETIMEDOUT;
78 }
79 
80 /*
81  * Disable interrupts and begin the xHCI halting process.
82  */
83 void xhci_quiesce(struct xhci_hcd *xhci)
84 {
85 	u32 halted;
86 	u32 cmd;
87 	u32 mask;
88 
89 	mask = ~(XHCI_IRQS);
90 	halted = readl(&xhci->op_regs->status) & STS_HALT;
91 	if (!halted)
92 		mask &= ~CMD_RUN;
93 
94 	cmd = readl(&xhci->op_regs->command);
95 	cmd &= mask;
96 	writel(cmd, &xhci->op_regs->command);
97 }
98 
99 /*
100  * Force HC into halt state.
101  *
102  * Disable any IRQs and clear the run/stop bit.
103  * HC will complete any current and actively pipelined transactions, and
104  * should halt within 16 ms of the run/stop bit being cleared.
105  * Read HC Halted bit in the status register to see when the HC is finished.
106  */
107 int xhci_halt(struct xhci_hcd *xhci)
108 {
109 	int ret;
110 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC");
111 	xhci_quiesce(xhci);
112 
113 	ret = xhci_handshake(&xhci->op_regs->status,
114 			STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
115 	if (!ret) {
116 		xhci->xhc_state |= XHCI_STATE_HALTED;
117 		xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
118 	} else
119 		xhci_warn(xhci, "Host not halted after %u microseconds.\n",
120 				XHCI_MAX_HALT_USEC);
121 	return ret;
122 }
123 
124 /*
125  * Set the run bit and wait for the host to be running.
126  */
127 static int xhci_start(struct xhci_hcd *xhci)
128 {
129 	u32 temp;
130 	int ret;
131 
132 	temp = readl(&xhci->op_regs->command);
133 	temp |= (CMD_RUN);
134 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.",
135 			temp);
136 	writel(temp, &xhci->op_regs->command);
137 
138 	/*
139 	 * Wait for the HCHalted Status bit to be 0 to indicate the host is
140 	 * running.
141 	 */
142 	ret = xhci_handshake(&xhci->op_regs->status,
143 			STS_HALT, 0, XHCI_MAX_HALT_USEC);
144 	if (ret == -ETIMEDOUT)
145 		xhci_err(xhci, "Host took too long to start, "
146 				"waited %u microseconds.\n",
147 				XHCI_MAX_HALT_USEC);
148 	if (!ret)
149 		xhci->xhc_state &= ~XHCI_STATE_HALTED;
150 	return ret;
151 }
152 
153 /*
154  * Reset a halted HC.
155  *
156  * This resets pipelines, timers, counters, state machines, etc.
157  * Transactions will be terminated immediately, and operational registers
158  * will be set to their defaults.
159  */
160 int xhci_reset(struct xhci_hcd *xhci)
161 {
162 	u32 command;
163 	u32 state;
164 	int ret, i;
165 
166 	state = readl(&xhci->op_regs->status);
167 	if ((state & STS_HALT) == 0) {
168 		xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
169 		return 0;
170 	}
171 
172 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC");
173 	command = readl(&xhci->op_regs->command);
174 	command |= CMD_RESET;
175 	writel(command, &xhci->op_regs->command);
176 
177 	ret = xhci_handshake(&xhci->op_regs->command,
178 			CMD_RESET, 0, 10 * 1000 * 1000);
179 	if (ret)
180 		return ret;
181 
182 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
183 			 "Wait for controller to be ready for doorbell rings");
184 	/*
185 	 * xHCI cannot write to any doorbells or operational registers other
186 	 * than status until the "Controller Not Ready" flag is cleared.
187 	 */
188 	ret = xhci_handshake(&xhci->op_regs->status,
189 			STS_CNR, 0, 10 * 1000 * 1000);
190 
191 	for (i = 0; i < 2; ++i) {
192 		xhci->bus_state[i].port_c_suspend = 0;
193 		xhci->bus_state[i].suspended_ports = 0;
194 		xhci->bus_state[i].resuming_ports = 0;
195 	}
196 
197 	return ret;
198 }
199 
200 #ifdef CONFIG_PCI
201 static int xhci_free_msi(struct xhci_hcd *xhci)
202 {
203 	int i;
204 
205 	if (!xhci->msix_entries)
206 		return -EINVAL;
207 
208 	for (i = 0; i < xhci->msix_count; i++)
209 		if (xhci->msix_entries[i].vector)
210 			free_irq(xhci->msix_entries[i].vector,
211 					xhci_to_hcd(xhci));
212 	return 0;
213 }
214 
215 /*
216  * Set up MSI
217  */
218 static int xhci_setup_msi(struct xhci_hcd *xhci)
219 {
220 	int ret;
221 	struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
222 
223 	ret = pci_enable_msi(pdev);
224 	if (ret) {
225 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
226 				"failed to allocate MSI entry");
227 		return ret;
228 	}
229 
230 	ret = request_irq(pdev->irq, xhci_msi_irq,
231 				0, "xhci_hcd", xhci_to_hcd(xhci));
232 	if (ret) {
233 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
234 				"disable MSI interrupt");
235 		pci_disable_msi(pdev);
236 	}
237 
238 	return ret;
239 }
240 
241 /*
242  * Free IRQs
243  * free all IRQs request
244  */
245 static void xhci_free_irq(struct xhci_hcd *xhci)
246 {
247 	struct pci_dev *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
248 	int ret;
249 
250 	/* return if using legacy interrupt */
251 	if (xhci_to_hcd(xhci)->irq > 0)
252 		return;
253 
254 	ret = xhci_free_msi(xhci);
255 	if (!ret)
256 		return;
257 	if (pdev->irq > 0)
258 		free_irq(pdev->irq, xhci_to_hcd(xhci));
259 
260 	return;
261 }
262 
263 /*
264  * Set up MSI-X
265  */
266 static int xhci_setup_msix(struct xhci_hcd *xhci)
267 {
268 	int i, ret = 0;
269 	struct usb_hcd *hcd = xhci_to_hcd(xhci);
270 	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
271 
272 	/*
273 	 * calculate number of msi-x vectors supported.
274 	 * - HCS_MAX_INTRS: the max number of interrupts the host can handle,
275 	 *   with max number of interrupters based on the xhci HCSPARAMS1.
276 	 * - num_online_cpus: maximum msi-x vectors per CPUs core.
277 	 *   Add additional 1 vector to ensure always available interrupt.
278 	 */
279 	xhci->msix_count = min(num_online_cpus() + 1,
280 				HCS_MAX_INTRS(xhci->hcs_params1));
281 
282 	xhci->msix_entries =
283 		kmalloc((sizeof(struct msix_entry))*xhci->msix_count,
284 				GFP_KERNEL);
285 	if (!xhci->msix_entries) {
286 		xhci_err(xhci, "Failed to allocate MSI-X entries\n");
287 		return -ENOMEM;
288 	}
289 
290 	for (i = 0; i < xhci->msix_count; i++) {
291 		xhci->msix_entries[i].entry = i;
292 		xhci->msix_entries[i].vector = 0;
293 	}
294 
295 	ret = pci_enable_msix_exact(pdev, xhci->msix_entries, xhci->msix_count);
296 	if (ret) {
297 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
298 				"Failed to enable MSI-X");
299 		goto free_entries;
300 	}
301 
302 	for (i = 0; i < xhci->msix_count; i++) {
303 		ret = request_irq(xhci->msix_entries[i].vector,
304 				xhci_msi_irq,
305 				0, "xhci_hcd", xhci_to_hcd(xhci));
306 		if (ret)
307 			goto disable_msix;
308 	}
309 
310 	hcd->msix_enabled = 1;
311 	return ret;
312 
313 disable_msix:
314 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "disable MSI-X interrupt");
315 	xhci_free_irq(xhci);
316 	pci_disable_msix(pdev);
317 free_entries:
318 	kfree(xhci->msix_entries);
319 	xhci->msix_entries = NULL;
320 	return ret;
321 }
322 
323 /* Free any IRQs and disable MSI-X */
324 static void xhci_cleanup_msix(struct xhci_hcd *xhci)
325 {
326 	struct usb_hcd *hcd = xhci_to_hcd(xhci);
327 	struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
328 
329 	if (xhci->quirks & XHCI_PLAT)
330 		return;
331 
332 	xhci_free_irq(xhci);
333 
334 	if (xhci->msix_entries) {
335 		pci_disable_msix(pdev);
336 		kfree(xhci->msix_entries);
337 		xhci->msix_entries = NULL;
338 	} else {
339 		pci_disable_msi(pdev);
340 	}
341 
342 	hcd->msix_enabled = 0;
343 	return;
344 }
345 
346 static void __maybe_unused xhci_msix_sync_irqs(struct xhci_hcd *xhci)
347 {
348 	int i;
349 
350 	if (xhci->msix_entries) {
351 		for (i = 0; i < xhci->msix_count; i++)
352 			synchronize_irq(xhci->msix_entries[i].vector);
353 	}
354 }
355 
356 static int xhci_try_enable_msi(struct usb_hcd *hcd)
357 {
358 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
359 	struct pci_dev  *pdev;
360 	int ret;
361 
362 	/* The xhci platform device has set up IRQs through usb_add_hcd. */
363 	if (xhci->quirks & XHCI_PLAT)
364 		return 0;
365 
366 	pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
367 	/*
368 	 * Some Fresco Logic host controllers advertise MSI, but fail to
369 	 * generate interrupts.  Don't even try to enable MSI.
370 	 */
371 	if (xhci->quirks & XHCI_BROKEN_MSI)
372 		goto legacy_irq;
373 
374 	/* unregister the legacy interrupt */
375 	if (hcd->irq)
376 		free_irq(hcd->irq, hcd);
377 	hcd->irq = 0;
378 
379 	ret = xhci_setup_msix(xhci);
380 	if (ret)
381 		/* fall back to msi*/
382 		ret = xhci_setup_msi(xhci);
383 
384 	if (!ret)
385 		/* hcd->irq is 0, we have MSI */
386 		return 0;
387 
388 	if (!pdev->irq) {
389 		xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");
390 		return -EINVAL;
391 	}
392 
393  legacy_irq:
394 	if (!strlen(hcd->irq_descr))
395 		snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",
396 			 hcd->driver->description, hcd->self.busnum);
397 
398 	/* fall back to legacy interrupt*/
399 	ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
400 			hcd->irq_descr, hcd);
401 	if (ret) {
402 		xhci_err(xhci, "request interrupt %d failed\n",
403 				pdev->irq);
404 		return ret;
405 	}
406 	hcd->irq = pdev->irq;
407 	return 0;
408 }
409 
410 #else
411 
412 static inline int xhci_try_enable_msi(struct usb_hcd *hcd)
413 {
414 	return 0;
415 }
416 
417 static inline void xhci_cleanup_msix(struct xhci_hcd *xhci)
418 {
419 }
420 
421 static inline void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
422 {
423 }
424 
425 #endif
426 
427 static void compliance_mode_recovery(unsigned long arg)
428 {
429 	struct xhci_hcd *xhci;
430 	struct usb_hcd *hcd;
431 	u32 temp;
432 	int i;
433 
434 	xhci = (struct xhci_hcd *)arg;
435 
436 	for (i = 0; i < xhci->num_usb3_ports; i++) {
437 		temp = readl(xhci->usb3_ports[i]);
438 		if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) {
439 			/*
440 			 * Compliance Mode Detected. Letting USB Core
441 			 * handle the Warm Reset
442 			 */
443 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
444 					"Compliance mode detected->port %d",
445 					i + 1);
446 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
447 					"Attempting compliance mode recovery");
448 			hcd = xhci->shared_hcd;
449 
450 			if (hcd->state == HC_STATE_SUSPENDED)
451 				usb_hcd_resume_root_hub(hcd);
452 
453 			usb_hcd_poll_rh_status(hcd);
454 		}
455 	}
456 
457 	if (xhci->port_status_u0 != ((1 << xhci->num_usb3_ports)-1))
458 		mod_timer(&xhci->comp_mode_recovery_timer,
459 			jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
460 }
461 
462 /*
463  * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver
464  * that causes ports behind that hardware to enter compliance mode sometimes.
465  * The quirk creates a timer that polls every 2 seconds the link state of
466  * each host controller's port and recovers it by issuing a Warm reset
467  * if Compliance mode is detected, otherwise the port will become "dead" (no
468  * device connections or disconnections will be detected anymore). Becasue no
469  * status event is generated when entering compliance mode (per xhci spec),
470  * this quirk is needed on systems that have the failing hardware installed.
471  */
472 static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)
473 {
474 	xhci->port_status_u0 = 0;
475 	setup_timer(&xhci->comp_mode_recovery_timer,
476 		    compliance_mode_recovery, (unsigned long)xhci);
477 	xhci->comp_mode_recovery_timer.expires = jiffies +
478 			msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);
479 
480 	set_timer_slack(&xhci->comp_mode_recovery_timer,
481 			msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
482 	add_timer(&xhci->comp_mode_recovery_timer);
483 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
484 			"Compliance mode recovery timer initialized");
485 }
486 
487 /*
488  * This function identifies the systems that have installed the SN65LVPE502CP
489  * USB3.0 re-driver and that need the Compliance Mode Quirk.
490  * Systems:
491  * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820
492  */
493 static bool xhci_compliance_mode_recovery_timer_quirk_check(void)
494 {
495 	const char *dmi_product_name, *dmi_sys_vendor;
496 
497 	dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
498 	dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
499 	if (!dmi_product_name || !dmi_sys_vendor)
500 		return false;
501 
502 	if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))
503 		return false;
504 
505 	if (strstr(dmi_product_name, "Z420") ||
506 			strstr(dmi_product_name, "Z620") ||
507 			strstr(dmi_product_name, "Z820") ||
508 			strstr(dmi_product_name, "Z1 Workstation"))
509 		return true;
510 
511 	return false;
512 }
513 
514 static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)
515 {
516 	return (xhci->port_status_u0 == ((1 << xhci->num_usb3_ports)-1));
517 }
518 
519 
520 /*
521  * Initialize memory for HCD and xHC (one-time init).
522  *
523  * Program the PAGESIZE register, initialize the device context array, create
524  * device contexts (?), set up a command ring segment (or two?), create event
525  * ring (one for now).
526  */
527 int xhci_init(struct usb_hcd *hcd)
528 {
529 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
530 	int retval = 0;
531 
532 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init");
533 	spin_lock_init(&xhci->lock);
534 	if (xhci->hci_version == 0x95 && link_quirk) {
535 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
536 				"QUIRK: Not clearing Link TRB chain bits.");
537 		xhci->quirks |= XHCI_LINK_TRB_QUIRK;
538 	} else {
539 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
540 				"xHCI doesn't need link TRB QUIRK");
541 	}
542 	retval = xhci_mem_init(xhci, GFP_KERNEL);
543 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init");
544 
545 	/* Initializing Compliance Mode Recovery Data If Needed */
546 	if (xhci_compliance_mode_recovery_timer_quirk_check()) {
547 		xhci->quirks |= XHCI_COMP_MODE_QUIRK;
548 		compliance_mode_recovery_timer_init(xhci);
549 	}
550 
551 	return retval;
552 }
553 
554 /*-------------------------------------------------------------------------*/
555 
556 
557 static int xhci_run_finished(struct xhci_hcd *xhci)
558 {
559 	if (xhci_start(xhci)) {
560 		xhci_halt(xhci);
561 		return -ENODEV;
562 	}
563 	xhci->shared_hcd->state = HC_STATE_RUNNING;
564 	xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
565 
566 	if (xhci->quirks & XHCI_NEC_HOST)
567 		xhci_ring_cmd_db(xhci);
568 
569 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
570 			"Finished xhci_run for USB3 roothub");
571 	return 0;
572 }
573 
574 /*
575  * Start the HC after it was halted.
576  *
577  * This function is called by the USB core when the HC driver is added.
578  * Its opposite is xhci_stop().
579  *
580  * xhci_init() must be called once before this function can be called.
581  * Reset the HC, enable device slot contexts, program DCBAAP, and
582  * set command ring pointer and event ring pointer.
583  *
584  * Setup MSI-X vectors and enable interrupts.
585  */
586 int xhci_run(struct usb_hcd *hcd)
587 {
588 	u32 temp;
589 	u64 temp_64;
590 	int ret;
591 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
592 
593 	/* Start the xHCI host controller running only after the USB 2.0 roothub
594 	 * is setup.
595 	 */
596 
597 	hcd->uses_new_polling = 1;
598 	if (!usb_hcd_is_primary_hcd(hcd))
599 		return xhci_run_finished(xhci);
600 
601 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");
602 
603 	ret = xhci_try_enable_msi(hcd);
604 	if (ret)
605 		return ret;
606 
607 	xhci_dbg(xhci, "Command ring memory map follows:\n");
608 	xhci_debug_ring(xhci, xhci->cmd_ring);
609 	xhci_dbg_ring_ptrs(xhci, xhci->cmd_ring);
610 	xhci_dbg_cmd_ptrs(xhci);
611 
612 	xhci_dbg(xhci, "ERST memory map follows:\n");
613 	xhci_dbg_erst(xhci, &xhci->erst);
614 	xhci_dbg(xhci, "Event ring:\n");
615 	xhci_debug_ring(xhci, xhci->event_ring);
616 	xhci_dbg_ring_ptrs(xhci, xhci->event_ring);
617 	temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
618 	temp_64 &= ~ERST_PTR_MASK;
619 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
620 			"ERST deq = 64'h%0lx", (long unsigned int) temp_64);
621 
622 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
623 			"// Set the interrupt modulation register");
624 	temp = readl(&xhci->ir_set->irq_control);
625 	temp &= ~ER_IRQ_INTERVAL_MASK;
626 	temp |= (u32) 160;
627 	writel(temp, &xhci->ir_set->irq_control);
628 
629 	/* Set the HCD state before we enable the irqs */
630 	temp = readl(&xhci->op_regs->command);
631 	temp |= (CMD_EIE);
632 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
633 			"// Enable interrupts, cmd = 0x%x.", temp);
634 	writel(temp, &xhci->op_regs->command);
635 
636 	temp = readl(&xhci->ir_set->irq_pending);
637 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
638 			"// Enabling event ring interrupter %p by writing 0x%x to irq_pending",
639 			xhci->ir_set, (unsigned int) ER_IRQ_ENABLE(temp));
640 	writel(ER_IRQ_ENABLE(temp), &xhci->ir_set->irq_pending);
641 	xhci_print_ir_set(xhci, 0);
642 
643 	if (xhci->quirks & XHCI_NEC_HOST) {
644 		struct xhci_command *command;
645 		command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
646 		if (!command)
647 			return -ENOMEM;
648 		xhci_queue_vendor_command(xhci, command, 0, 0, 0,
649 				TRB_TYPE(TRB_NEC_GET_FW));
650 	}
651 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
652 			"Finished xhci_run for USB2 roothub");
653 	return 0;
654 }
655 EXPORT_SYMBOL_GPL(xhci_run);
656 
657 static void xhci_only_stop_hcd(struct usb_hcd *hcd)
658 {
659 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
660 
661 	spin_lock_irq(&xhci->lock);
662 	xhci_halt(xhci);
663 
664 	/* The shared_hcd is going to be deallocated shortly (the USB core only
665 	 * calls this function when allocation fails in usb_add_hcd(), or
666 	 * usb_remove_hcd() is called).  So we need to unset xHCI's pointer.
667 	 */
668 	xhci->shared_hcd = NULL;
669 	spin_unlock_irq(&xhci->lock);
670 }
671 
672 /*
673  * Stop xHCI driver.
674  *
675  * This function is called by the USB core when the HC driver is removed.
676  * Its opposite is xhci_run().
677  *
678  * Disable device contexts, disable IRQs, and quiesce the HC.
679  * Reset the HC, finish any completed transactions, and cleanup memory.
680  */
681 void xhci_stop(struct usb_hcd *hcd)
682 {
683 	u32 temp;
684 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
685 
686 	if (!usb_hcd_is_primary_hcd(hcd)) {
687 		xhci_only_stop_hcd(xhci->shared_hcd);
688 		return;
689 	}
690 
691 	spin_lock_irq(&xhci->lock);
692 	/* Make sure the xHC is halted for a USB3 roothub
693 	 * (xhci_stop() could be called as part of failed init).
694 	 */
695 	xhci_halt(xhci);
696 	xhci_reset(xhci);
697 	spin_unlock_irq(&xhci->lock);
698 
699 	xhci_cleanup_msix(xhci);
700 
701 	/* Deleting Compliance Mode Recovery Timer */
702 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
703 			(!(xhci_all_ports_seen_u0(xhci)))) {
704 		del_timer_sync(&xhci->comp_mode_recovery_timer);
705 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
706 				"%s: compliance mode recovery timer deleted",
707 				__func__);
708 	}
709 
710 	if (xhci->quirks & XHCI_AMD_PLL_FIX)
711 		usb_amd_dev_put();
712 
713 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
714 			"// Disabling event ring interrupts");
715 	temp = readl(&xhci->op_regs->status);
716 	writel(temp & ~STS_EINT, &xhci->op_regs->status);
717 	temp = readl(&xhci->ir_set->irq_pending);
718 	writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
719 	xhci_print_ir_set(xhci, 0);
720 
721 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory");
722 	xhci_mem_cleanup(xhci);
723 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
724 			"xhci_stop completed - status = %x",
725 			readl(&xhci->op_regs->status));
726 }
727 
728 /*
729  * Shutdown HC (not bus-specific)
730  *
731  * This is called when the machine is rebooting or halting.  We assume that the
732  * machine will be powered off, and the HC's internal state will be reset.
733  * Don't bother to free memory.
734  *
735  * This will only ever be called with the main usb_hcd (the USB3 roothub).
736  */
737 void xhci_shutdown(struct usb_hcd *hcd)
738 {
739 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
740 
741 	if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
742 		usb_disable_xhci_ports(to_pci_dev(hcd->self.controller));
743 
744 	spin_lock_irq(&xhci->lock);
745 	xhci_halt(xhci);
746 	/* Workaround for spurious wakeups at shutdown with HSW */
747 	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
748 		xhci_reset(xhci);
749 	spin_unlock_irq(&xhci->lock);
750 
751 	xhci_cleanup_msix(xhci);
752 
753 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
754 			"xhci_shutdown completed - status = %x",
755 			readl(&xhci->op_regs->status));
756 
757 	/* Yet another workaround for spurious wakeups at shutdown with HSW */
758 	if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
759 		pci_set_power_state(to_pci_dev(hcd->self.controller), PCI_D3hot);
760 }
761 
762 #ifdef CONFIG_PM
763 static void xhci_save_registers(struct xhci_hcd *xhci)
764 {
765 	xhci->s3.command = readl(&xhci->op_regs->command);
766 	xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification);
767 	xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
768 	xhci->s3.config_reg = readl(&xhci->op_regs->config_reg);
769 	xhci->s3.erst_size = readl(&xhci->ir_set->erst_size);
770 	xhci->s3.erst_base = xhci_read_64(xhci, &xhci->ir_set->erst_base);
771 	xhci->s3.erst_dequeue = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
772 	xhci->s3.irq_pending = readl(&xhci->ir_set->irq_pending);
773 	xhci->s3.irq_control = readl(&xhci->ir_set->irq_control);
774 }
775 
776 static void xhci_restore_registers(struct xhci_hcd *xhci)
777 {
778 	writel(xhci->s3.command, &xhci->op_regs->command);
779 	writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
780 	xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
781 	writel(xhci->s3.config_reg, &xhci->op_regs->config_reg);
782 	writel(xhci->s3.erst_size, &xhci->ir_set->erst_size);
783 	xhci_write_64(xhci, xhci->s3.erst_base, &xhci->ir_set->erst_base);
784 	xhci_write_64(xhci, xhci->s3.erst_dequeue, &xhci->ir_set->erst_dequeue);
785 	writel(xhci->s3.irq_pending, &xhci->ir_set->irq_pending);
786 	writel(xhci->s3.irq_control, &xhci->ir_set->irq_control);
787 }
788 
789 static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
790 {
791 	u64	val_64;
792 
793 	/* step 2: initialize command ring buffer */
794 	val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
795 	val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
796 		(xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
797 				      xhci->cmd_ring->dequeue) &
798 		 (u64) ~CMD_RING_RSVD_BITS) |
799 		xhci->cmd_ring->cycle_state;
800 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
801 			"// Setting command ring address to 0x%llx",
802 			(long unsigned long) val_64);
803 	xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
804 }
805 
806 /*
807  * The whole command ring must be cleared to zero when we suspend the host.
808  *
809  * The host doesn't save the command ring pointer in the suspend well, so we
810  * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
811  * aligned, because of the reserved bits in the command ring dequeue pointer
812  * register.  Therefore, we can't just set the dequeue pointer back in the
813  * middle of the ring (TRBs are 16-byte aligned).
814  */
815 static void xhci_clear_command_ring(struct xhci_hcd *xhci)
816 {
817 	struct xhci_ring *ring;
818 	struct xhci_segment *seg;
819 
820 	ring = xhci->cmd_ring;
821 	seg = ring->deq_seg;
822 	do {
823 		memset(seg->trbs, 0,
824 			sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
825 		seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
826 			cpu_to_le32(~TRB_CYCLE);
827 		seg = seg->next;
828 	} while (seg != ring->deq_seg);
829 
830 	/* Reset the software enqueue and dequeue pointers */
831 	ring->deq_seg = ring->first_seg;
832 	ring->dequeue = ring->first_seg->trbs;
833 	ring->enq_seg = ring->deq_seg;
834 	ring->enqueue = ring->dequeue;
835 
836 	ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
837 	/*
838 	 * Ring is now zeroed, so the HW should look for change of ownership
839 	 * when the cycle bit is set to 1.
840 	 */
841 	ring->cycle_state = 1;
842 
843 	/*
844 	 * Reset the hardware dequeue pointer.
845 	 * Yes, this will need to be re-written after resume, but we're paranoid
846 	 * and want to make sure the hardware doesn't access bogus memory
847 	 * because, say, the BIOS or an SMI started the host without changing
848 	 * the command ring pointers.
849 	 */
850 	xhci_set_cmd_ring_deq(xhci);
851 }
852 
853 static void xhci_disable_port_wake_on_bits(struct xhci_hcd *xhci)
854 {
855 	int port_index;
856 	__le32 __iomem **port_array;
857 	unsigned long flags;
858 	u32 t1, t2;
859 
860 	spin_lock_irqsave(&xhci->lock, flags);
861 
862 	/* disble usb3 ports Wake bits*/
863 	port_index = xhci->num_usb3_ports;
864 	port_array = xhci->usb3_ports;
865 	while (port_index--) {
866 		t1 = readl(port_array[port_index]);
867 		t1 = xhci_port_state_to_neutral(t1);
868 		t2 = t1 & ~PORT_WAKE_BITS;
869 		if (t1 != t2)
870 			writel(t2, port_array[port_index]);
871 	}
872 
873 	/* disble usb2 ports Wake bits*/
874 	port_index = xhci->num_usb2_ports;
875 	port_array = xhci->usb2_ports;
876 	while (port_index--) {
877 		t1 = readl(port_array[port_index]);
878 		t1 = xhci_port_state_to_neutral(t1);
879 		t2 = t1 & ~PORT_WAKE_BITS;
880 		if (t1 != t2)
881 			writel(t2, port_array[port_index]);
882 	}
883 
884 	spin_unlock_irqrestore(&xhci->lock, flags);
885 }
886 
887 /*
888  * Stop HC (not bus-specific)
889  *
890  * This is called when the machine transition into S3/S4 mode.
891  *
892  */
893 int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)
894 {
895 	int			rc = 0;
896 	unsigned int		delay = XHCI_MAX_HALT_USEC;
897 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
898 	u32			command;
899 
900 	if (hcd->state != HC_STATE_SUSPENDED ||
901 			xhci->shared_hcd->state != HC_STATE_SUSPENDED)
902 		return -EINVAL;
903 
904 	/* Clear root port wake on bits if wakeup not allowed. */
905 	if (!do_wakeup)
906 		xhci_disable_port_wake_on_bits(xhci);
907 
908 	/* Don't poll the roothubs on bus suspend. */
909 	xhci_dbg(xhci, "%s: stopping port polling.\n", __func__);
910 	clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
911 	del_timer_sync(&hcd->rh_timer);
912 	clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
913 	del_timer_sync(&xhci->shared_hcd->rh_timer);
914 
915 	spin_lock_irq(&xhci->lock);
916 	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
917 	clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
918 	/* step 1: stop endpoint */
919 	/* skipped assuming that port suspend has done */
920 
921 	/* step 2: clear Run/Stop bit */
922 	command = readl(&xhci->op_regs->command);
923 	command &= ~CMD_RUN;
924 	writel(command, &xhci->op_regs->command);
925 
926 	/* Some chips from Fresco Logic need an extraordinary delay */
927 	delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;
928 
929 	if (xhci_handshake(&xhci->op_regs->status,
930 		      STS_HALT, STS_HALT, delay)) {
931 		xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
932 		spin_unlock_irq(&xhci->lock);
933 		return -ETIMEDOUT;
934 	}
935 	xhci_clear_command_ring(xhci);
936 
937 	/* step 3: save registers */
938 	xhci_save_registers(xhci);
939 
940 	/* step 4: set CSS flag */
941 	command = readl(&xhci->op_regs->command);
942 	command |= CMD_CSS;
943 	writel(command, &xhci->op_regs->command);
944 	if (xhci_handshake(&xhci->op_regs->status,
945 				STS_SAVE, 0, 10 * 1000)) {
946 		xhci_warn(xhci, "WARN: xHC save state timeout\n");
947 		spin_unlock_irq(&xhci->lock);
948 		return -ETIMEDOUT;
949 	}
950 	spin_unlock_irq(&xhci->lock);
951 
952 	/*
953 	 * Deleting Compliance Mode Recovery Timer because the xHCI Host
954 	 * is about to be suspended.
955 	 */
956 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
957 			(!(xhci_all_ports_seen_u0(xhci)))) {
958 		del_timer_sync(&xhci->comp_mode_recovery_timer);
959 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
960 				"%s: compliance mode recovery timer deleted",
961 				__func__);
962 	}
963 
964 	/* step 5: remove core well power */
965 	/* synchronize irq when using MSI-X */
966 	xhci_msix_sync_irqs(xhci);
967 
968 	return rc;
969 }
970 EXPORT_SYMBOL_GPL(xhci_suspend);
971 
972 /*
973  * start xHC (not bus-specific)
974  *
975  * This is called when the machine transition from S3/S4 mode.
976  *
977  */
978 int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
979 {
980 	u32			command, temp = 0, status;
981 	struct usb_hcd		*hcd = xhci_to_hcd(xhci);
982 	struct usb_hcd		*secondary_hcd;
983 	int			retval = 0;
984 	bool			comp_timer_running = false;
985 
986 	/* Wait a bit if either of the roothubs need to settle from the
987 	 * transition into bus suspend.
988 	 */
989 	if (time_before(jiffies, xhci->bus_state[0].next_statechange) ||
990 			time_before(jiffies,
991 				xhci->bus_state[1].next_statechange))
992 		msleep(100);
993 
994 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
995 	set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
996 
997 	spin_lock_irq(&xhci->lock);
998 	if (xhci->quirks & XHCI_RESET_ON_RESUME)
999 		hibernated = true;
1000 
1001 	if (!hibernated) {
1002 		/* step 1: restore register */
1003 		xhci_restore_registers(xhci);
1004 		/* step 2: initialize command ring buffer */
1005 		xhci_set_cmd_ring_deq(xhci);
1006 		/* step 3: restore state and start state*/
1007 		/* step 3: set CRS flag */
1008 		command = readl(&xhci->op_regs->command);
1009 		command |= CMD_CRS;
1010 		writel(command, &xhci->op_regs->command);
1011 		if (xhci_handshake(&xhci->op_regs->status,
1012 			      STS_RESTORE, 0, 10 * 1000)) {
1013 			xhci_warn(xhci, "WARN: xHC restore state timeout\n");
1014 			spin_unlock_irq(&xhci->lock);
1015 			return -ETIMEDOUT;
1016 		}
1017 		temp = readl(&xhci->op_regs->status);
1018 	}
1019 
1020 	/* If restore operation fails, re-initialize the HC during resume */
1021 	if ((temp & STS_SRE) || hibernated) {
1022 
1023 		if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
1024 				!(xhci_all_ports_seen_u0(xhci))) {
1025 			del_timer_sync(&xhci->comp_mode_recovery_timer);
1026 			xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1027 				"Compliance Mode Recovery Timer deleted!");
1028 		}
1029 
1030 		/* Let the USB core know _both_ roothubs lost power. */
1031 		usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
1032 		usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
1033 
1034 		xhci_dbg(xhci, "Stop HCD\n");
1035 		xhci_halt(xhci);
1036 		xhci_reset(xhci);
1037 		spin_unlock_irq(&xhci->lock);
1038 		xhci_cleanup_msix(xhci);
1039 
1040 		xhci_dbg(xhci, "// Disabling event ring interrupts\n");
1041 		temp = readl(&xhci->op_regs->status);
1042 		writel(temp & ~STS_EINT, &xhci->op_regs->status);
1043 		temp = readl(&xhci->ir_set->irq_pending);
1044 		writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
1045 		xhci_print_ir_set(xhci, 0);
1046 
1047 		xhci_dbg(xhci, "cleaning up memory\n");
1048 		xhci_mem_cleanup(xhci);
1049 		xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
1050 			    readl(&xhci->op_regs->status));
1051 
1052 		/* USB core calls the PCI reinit and start functions twice:
1053 		 * first with the primary HCD, and then with the secondary HCD.
1054 		 * If we don't do the same, the host will never be started.
1055 		 */
1056 		if (!usb_hcd_is_primary_hcd(hcd))
1057 			secondary_hcd = hcd;
1058 		else
1059 			secondary_hcd = xhci->shared_hcd;
1060 
1061 		xhci_dbg(xhci, "Initialize the xhci_hcd\n");
1062 		retval = xhci_init(hcd->primary_hcd);
1063 		if (retval)
1064 			return retval;
1065 		comp_timer_running = true;
1066 
1067 		xhci_dbg(xhci, "Start the primary HCD\n");
1068 		retval = xhci_run(hcd->primary_hcd);
1069 		if (!retval) {
1070 			xhci_dbg(xhci, "Start the secondary HCD\n");
1071 			retval = xhci_run(secondary_hcd);
1072 		}
1073 		hcd->state = HC_STATE_SUSPENDED;
1074 		xhci->shared_hcd->state = HC_STATE_SUSPENDED;
1075 		goto done;
1076 	}
1077 
1078 	/* step 4: set Run/Stop bit */
1079 	command = readl(&xhci->op_regs->command);
1080 	command |= CMD_RUN;
1081 	writel(command, &xhci->op_regs->command);
1082 	xhci_handshake(&xhci->op_regs->status, STS_HALT,
1083 		  0, 250 * 1000);
1084 
1085 	/* step 5: walk topology and initialize portsc,
1086 	 * portpmsc and portli
1087 	 */
1088 	/* this is done in bus_resume */
1089 
1090 	/* step 6: restart each of the previously
1091 	 * Running endpoints by ringing their doorbells
1092 	 */
1093 
1094 	spin_unlock_irq(&xhci->lock);
1095 
1096  done:
1097 	if (retval == 0) {
1098 		/* Resume root hubs only when have pending events. */
1099 		status = readl(&xhci->op_regs->status);
1100 		if (status & STS_EINT) {
1101 			usb_hcd_resume_root_hub(hcd);
1102 			usb_hcd_resume_root_hub(xhci->shared_hcd);
1103 		}
1104 	}
1105 
1106 	/*
1107 	 * If system is subject to the Quirk, Compliance Mode Timer needs to
1108 	 * be re-initialized Always after a system resume. Ports are subject
1109 	 * to suffer the Compliance Mode issue again. It doesn't matter if
1110 	 * ports have entered previously to U0 before system's suspension.
1111 	 */
1112 	if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)
1113 		compliance_mode_recovery_timer_init(xhci);
1114 
1115 	/* Re-enable port polling. */
1116 	xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1117 	set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1118 	usb_hcd_poll_rh_status(hcd);
1119 	set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
1120 	usb_hcd_poll_rh_status(xhci->shared_hcd);
1121 
1122 	return retval;
1123 }
1124 EXPORT_SYMBOL_GPL(xhci_resume);
1125 #endif	/* CONFIG_PM */
1126 
1127 /*-------------------------------------------------------------------------*/
1128 
1129 /**
1130  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
1131  * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
1132  * value to right shift 1 for the bitmask.
1133  *
1134  * Index  = (epnum * 2) + direction - 1,
1135  * where direction = 0 for OUT, 1 for IN.
1136  * For control endpoints, the IN index is used (OUT index is unused), so
1137  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
1138  */
1139 unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
1140 {
1141 	unsigned int index;
1142 	if (usb_endpoint_xfer_control(desc))
1143 		index = (unsigned int) (usb_endpoint_num(desc)*2);
1144 	else
1145 		index = (unsigned int) (usb_endpoint_num(desc)*2) +
1146 			(usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
1147 	return index;
1148 }
1149 
1150 /* The reverse operation to xhci_get_endpoint_index. Calculate the USB endpoint
1151  * address from the XHCI endpoint index.
1152  */
1153 unsigned int xhci_get_endpoint_address(unsigned int ep_index)
1154 {
1155 	unsigned int number = DIV_ROUND_UP(ep_index, 2);
1156 	unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN;
1157 	return direction | number;
1158 }
1159 
1160 /* Find the flag for this endpoint (for use in the control context).  Use the
1161  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
1162  * bit 1, etc.
1163  */
1164 unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
1165 {
1166 	return 1 << (xhci_get_endpoint_index(desc) + 1);
1167 }
1168 
1169 /* Find the flag for this endpoint (for use in the control context).  Use the
1170  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
1171  * bit 1, etc.
1172  */
1173 unsigned int xhci_get_endpoint_flag_from_index(unsigned int ep_index)
1174 {
1175 	return 1 << (ep_index + 1);
1176 }
1177 
1178 /* Compute the last valid endpoint context index.  Basically, this is the
1179  * endpoint index plus one.  For slot contexts with more than valid endpoint,
1180  * we find the most significant bit set in the added contexts flags.
1181  * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000
1182  * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one.
1183  */
1184 unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
1185 {
1186 	return fls(added_ctxs) - 1;
1187 }
1188 
1189 /* Returns 1 if the arguments are OK;
1190  * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
1191  */
1192 static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
1193 		struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
1194 		const char *func) {
1195 	struct xhci_hcd	*xhci;
1196 	struct xhci_virt_device	*virt_dev;
1197 
1198 	if (!hcd || (check_ep && !ep) || !udev) {
1199 		pr_debug("xHCI %s called with invalid args\n", func);
1200 		return -EINVAL;
1201 	}
1202 	if (!udev->parent) {
1203 		pr_debug("xHCI %s called for root hub\n", func);
1204 		return 0;
1205 	}
1206 
1207 	xhci = hcd_to_xhci(hcd);
1208 	if (check_virt_dev) {
1209 		if (!udev->slot_id || !xhci->devs[udev->slot_id]) {
1210 			xhci_dbg(xhci, "xHCI %s called with unaddressed device\n",
1211 					func);
1212 			return -EINVAL;
1213 		}
1214 
1215 		virt_dev = xhci->devs[udev->slot_id];
1216 		if (virt_dev->udev != udev) {
1217 			xhci_dbg(xhci, "xHCI %s called with udev and "
1218 					  "virt_dev does not match\n", func);
1219 			return -EINVAL;
1220 		}
1221 	}
1222 
1223 	if (xhci->xhc_state & XHCI_STATE_HALTED)
1224 		return -ENODEV;
1225 
1226 	return 1;
1227 }
1228 
1229 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
1230 		struct usb_device *udev, struct xhci_command *command,
1231 		bool ctx_change, bool must_succeed);
1232 
1233 /*
1234  * Full speed devices may have a max packet size greater than 8 bytes, but the
1235  * USB core doesn't know that until it reads the first 8 bytes of the
1236  * descriptor.  If the usb_device's max packet size changes after that point,
1237  * we need to issue an evaluate context command and wait on it.
1238  */
1239 static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
1240 		unsigned int ep_index, struct urb *urb)
1241 {
1242 	struct xhci_container_ctx *out_ctx;
1243 	struct xhci_input_control_ctx *ctrl_ctx;
1244 	struct xhci_ep_ctx *ep_ctx;
1245 	struct xhci_command *command;
1246 	int max_packet_size;
1247 	int hw_max_packet_size;
1248 	int ret = 0;
1249 
1250 	out_ctx = xhci->devs[slot_id]->out_ctx;
1251 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1252 	hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
1253 	max_packet_size = usb_endpoint_maxp(&urb->dev->ep0.desc);
1254 	if (hw_max_packet_size != max_packet_size) {
1255 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1256 				"Max Packet Size for ep 0 changed.");
1257 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1258 				"Max packet size in usb_device = %d",
1259 				max_packet_size);
1260 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1261 				"Max packet size in xHCI HW = %d",
1262 				hw_max_packet_size);
1263 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
1264 				"Issuing evaluate context command.");
1265 
1266 		/* Set up the input context flags for the command */
1267 		/* FIXME: This won't work if a non-default control endpoint
1268 		 * changes max packet sizes.
1269 		 */
1270 
1271 		command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
1272 		if (!command)
1273 			return -ENOMEM;
1274 
1275 		command->in_ctx = xhci->devs[slot_id]->in_ctx;
1276 		ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
1277 		if (!ctrl_ctx) {
1278 			xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1279 					__func__);
1280 			ret = -ENOMEM;
1281 			goto command_cleanup;
1282 		}
1283 		/* Set up the modified control endpoint 0 */
1284 		xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
1285 				xhci->devs[slot_id]->out_ctx, ep_index);
1286 
1287 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
1288 		ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
1289 		ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
1290 
1291 		ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
1292 		ctrl_ctx->drop_flags = 0;
1293 
1294 		xhci_dbg(xhci, "Slot %d input context\n", slot_id);
1295 		xhci_dbg_ctx(xhci, command->in_ctx, ep_index);
1296 		xhci_dbg(xhci, "Slot %d output context\n", slot_id);
1297 		xhci_dbg_ctx(xhci, out_ctx, ep_index);
1298 
1299 		ret = xhci_configure_endpoint(xhci, urb->dev, command,
1300 				true, false);
1301 
1302 		/* Clean up the input context for later use by bandwidth
1303 		 * functions.
1304 		 */
1305 		ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
1306 command_cleanup:
1307 		kfree(command->completion);
1308 		kfree(command);
1309 	}
1310 	return ret;
1311 }
1312 
1313 /*
1314  * non-error returns are a promise to giveback() the urb later
1315  * we drop ownership so next owner (or urb unlink) can get it
1316  */
1317 int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
1318 {
1319 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
1320 	struct xhci_td *buffer;
1321 	unsigned long flags;
1322 	int ret = 0;
1323 	unsigned int slot_id, ep_index;
1324 	struct urb_priv	*urb_priv;
1325 	int size, i;
1326 
1327 	if (!urb || xhci_check_args(hcd, urb->dev, urb->ep,
1328 					true, true, __func__) <= 0)
1329 		return -EINVAL;
1330 
1331 	slot_id = urb->dev->slot_id;
1332 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1333 
1334 	if (!HCD_HW_ACCESSIBLE(hcd)) {
1335 		if (!in_interrupt())
1336 			xhci_dbg(xhci, "urb submitted during PCI suspend\n");
1337 		ret = -ESHUTDOWN;
1338 		goto exit;
1339 	}
1340 
1341 	if (usb_endpoint_xfer_isoc(&urb->ep->desc))
1342 		size = urb->number_of_packets;
1343 	else
1344 		size = 1;
1345 
1346 	urb_priv = kzalloc(sizeof(struct urb_priv) +
1347 				  size * sizeof(struct xhci_td *), mem_flags);
1348 	if (!urb_priv)
1349 		return -ENOMEM;
1350 
1351 	buffer = kzalloc(size * sizeof(struct xhci_td), mem_flags);
1352 	if (!buffer) {
1353 		kfree(urb_priv);
1354 		return -ENOMEM;
1355 	}
1356 
1357 	for (i = 0; i < size; i++) {
1358 		urb_priv->td[i] = buffer;
1359 		buffer++;
1360 	}
1361 
1362 	urb_priv->length = size;
1363 	urb_priv->td_cnt = 0;
1364 	urb->hcpriv = urb_priv;
1365 
1366 	if (usb_endpoint_xfer_control(&urb->ep->desc)) {
1367 		/* Check to see if the max packet size for the default control
1368 		 * endpoint changed during FS device enumeration
1369 		 */
1370 		if (urb->dev->speed == USB_SPEED_FULL) {
1371 			ret = xhci_check_maxpacket(xhci, slot_id,
1372 					ep_index, urb);
1373 			if (ret < 0) {
1374 				xhci_urb_free_priv(urb_priv);
1375 				urb->hcpriv = NULL;
1376 				return ret;
1377 			}
1378 		}
1379 
1380 		/* We have a spinlock and interrupts disabled, so we must pass
1381 		 * atomic context to this function, which may allocate memory.
1382 		 */
1383 		spin_lock_irqsave(&xhci->lock, flags);
1384 		if (xhci->xhc_state & XHCI_STATE_DYING)
1385 			goto dying;
1386 		ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
1387 				slot_id, ep_index);
1388 		if (ret)
1389 			goto free_priv;
1390 		spin_unlock_irqrestore(&xhci->lock, flags);
1391 	} else if (usb_endpoint_xfer_bulk(&urb->ep->desc)) {
1392 		spin_lock_irqsave(&xhci->lock, flags);
1393 		if (xhci->xhc_state & XHCI_STATE_DYING)
1394 			goto dying;
1395 		if (xhci->devs[slot_id]->eps[ep_index].ep_state &
1396 				EP_GETTING_STREAMS) {
1397 			xhci_warn(xhci, "WARN: Can't enqueue URB while bulk ep "
1398 					"is transitioning to using streams.\n");
1399 			ret = -EINVAL;
1400 		} else if (xhci->devs[slot_id]->eps[ep_index].ep_state &
1401 				EP_GETTING_NO_STREAMS) {
1402 			xhci_warn(xhci, "WARN: Can't enqueue URB while bulk ep "
1403 					"is transitioning to "
1404 					"not having streams.\n");
1405 			ret = -EINVAL;
1406 		} else {
1407 			ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
1408 					slot_id, ep_index);
1409 		}
1410 		if (ret)
1411 			goto free_priv;
1412 		spin_unlock_irqrestore(&xhci->lock, flags);
1413 	} else if (usb_endpoint_xfer_int(&urb->ep->desc)) {
1414 		spin_lock_irqsave(&xhci->lock, flags);
1415 		if (xhci->xhc_state & XHCI_STATE_DYING)
1416 			goto dying;
1417 		ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
1418 				slot_id, ep_index);
1419 		if (ret)
1420 			goto free_priv;
1421 		spin_unlock_irqrestore(&xhci->lock, flags);
1422 	} else {
1423 		spin_lock_irqsave(&xhci->lock, flags);
1424 		if (xhci->xhc_state & XHCI_STATE_DYING)
1425 			goto dying;
1426 		ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
1427 				slot_id, ep_index);
1428 		if (ret)
1429 			goto free_priv;
1430 		spin_unlock_irqrestore(&xhci->lock, flags);
1431 	}
1432 exit:
1433 	return ret;
1434 dying:
1435 	xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for "
1436 			"non-responsive xHCI host.\n",
1437 			urb->ep->desc.bEndpointAddress, urb);
1438 	ret = -ESHUTDOWN;
1439 free_priv:
1440 	xhci_urb_free_priv(urb_priv);
1441 	urb->hcpriv = NULL;
1442 	spin_unlock_irqrestore(&xhci->lock, flags);
1443 	return ret;
1444 }
1445 
1446 /* Get the right ring for the given URB.
1447  * If the endpoint supports streams, boundary check the URB's stream ID.
1448  * If the endpoint doesn't support streams, return the singular endpoint ring.
1449  */
1450 static struct xhci_ring *xhci_urb_to_transfer_ring(struct xhci_hcd *xhci,
1451 		struct urb *urb)
1452 {
1453 	unsigned int slot_id;
1454 	unsigned int ep_index;
1455 	unsigned int stream_id;
1456 	struct xhci_virt_ep *ep;
1457 
1458 	slot_id = urb->dev->slot_id;
1459 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1460 	stream_id = urb->stream_id;
1461 	ep = &xhci->devs[slot_id]->eps[ep_index];
1462 	/* Common case: no streams */
1463 	if (!(ep->ep_state & EP_HAS_STREAMS))
1464 		return ep->ring;
1465 
1466 	if (stream_id == 0) {
1467 		xhci_warn(xhci,
1468 				"WARN: Slot ID %u, ep index %u has streams, "
1469 				"but URB has no stream ID.\n",
1470 				slot_id, ep_index);
1471 		return NULL;
1472 	}
1473 
1474 	if (stream_id < ep->stream_info->num_streams)
1475 		return ep->stream_info->stream_rings[stream_id];
1476 
1477 	xhci_warn(xhci,
1478 			"WARN: Slot ID %u, ep index %u has "
1479 			"stream IDs 1 to %u allocated, "
1480 			"but stream ID %u is requested.\n",
1481 			slot_id, ep_index,
1482 			ep->stream_info->num_streams - 1,
1483 			stream_id);
1484 	return NULL;
1485 }
1486 
1487 /*
1488  * Remove the URB's TD from the endpoint ring.  This may cause the HC to stop
1489  * USB transfers, potentially stopping in the middle of a TRB buffer.  The HC
1490  * should pick up where it left off in the TD, unless a Set Transfer Ring
1491  * Dequeue Pointer is issued.
1492  *
1493  * The TRBs that make up the buffers for the canceled URB will be "removed" from
1494  * the ring.  Since the ring is a contiguous structure, they can't be physically
1495  * removed.  Instead, there are two options:
1496  *
1497  *  1) If the HC is in the middle of processing the URB to be canceled, we
1498  *     simply move the ring's dequeue pointer past those TRBs using the Set
1499  *     Transfer Ring Dequeue Pointer command.  This will be the common case,
1500  *     when drivers timeout on the last submitted URB and attempt to cancel.
1501  *
1502  *  2) If the HC is in the middle of a different TD, we turn the TRBs into a
1503  *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
1504  *     HC will need to invalidate the any TRBs it has cached after the stop
1505  *     endpoint command, as noted in the xHCI 0.95 errata.
1506  *
1507  *  3) The TD may have completed by the time the Stop Endpoint Command
1508  *     completes, so software needs to handle that case too.
1509  *
1510  * This function should protect against the TD enqueueing code ringing the
1511  * doorbell while this code is waiting for a Stop Endpoint command to complete.
1512  * It also needs to account for multiple cancellations on happening at the same
1513  * time for the same endpoint.
1514  *
1515  * Note that this function can be called in any context, or so says
1516  * usb_hcd_unlink_urb()
1517  */
1518 int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
1519 {
1520 	unsigned long flags;
1521 	int ret, i;
1522 	u32 temp;
1523 	struct xhci_hcd *xhci;
1524 	struct urb_priv	*urb_priv;
1525 	struct xhci_td *td;
1526 	unsigned int ep_index;
1527 	struct xhci_ring *ep_ring;
1528 	struct xhci_virt_ep *ep;
1529 	struct xhci_command *command;
1530 
1531 	xhci = hcd_to_xhci(hcd);
1532 	spin_lock_irqsave(&xhci->lock, flags);
1533 	/* Make sure the URB hasn't completed or been unlinked already */
1534 	ret = usb_hcd_check_unlink_urb(hcd, urb, status);
1535 	if (ret || !urb->hcpriv)
1536 		goto done;
1537 	temp = readl(&xhci->op_regs->status);
1538 	if (temp == 0xffffffff || (xhci->xhc_state & XHCI_STATE_HALTED)) {
1539 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1540 				"HW died, freeing TD.");
1541 		urb_priv = urb->hcpriv;
1542 		for (i = urb_priv->td_cnt; i < urb_priv->length; i++) {
1543 			td = urb_priv->td[i];
1544 			if (!list_empty(&td->td_list))
1545 				list_del_init(&td->td_list);
1546 			if (!list_empty(&td->cancelled_td_list))
1547 				list_del_init(&td->cancelled_td_list);
1548 		}
1549 
1550 		usb_hcd_unlink_urb_from_ep(hcd, urb);
1551 		spin_unlock_irqrestore(&xhci->lock, flags);
1552 		usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
1553 		xhci_urb_free_priv(urb_priv);
1554 		return ret;
1555 	}
1556 	if ((xhci->xhc_state & XHCI_STATE_DYING) ||
1557 			(xhci->xhc_state & XHCI_STATE_HALTED)) {
1558 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1559 				"Ep 0x%x: URB %p to be canceled on "
1560 				"non-responsive xHCI host.",
1561 				urb->ep->desc.bEndpointAddress, urb);
1562 		/* Let the stop endpoint command watchdog timer (which set this
1563 		 * state) finish cleaning up the endpoint TD lists.  We must
1564 		 * have caught it in the middle of dropping a lock and giving
1565 		 * back an URB.
1566 		 */
1567 		goto done;
1568 	}
1569 
1570 	ep_index = xhci_get_endpoint_index(&urb->ep->desc);
1571 	ep = &xhci->devs[urb->dev->slot_id]->eps[ep_index];
1572 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
1573 	if (!ep_ring) {
1574 		ret = -EINVAL;
1575 		goto done;
1576 	}
1577 
1578 	urb_priv = urb->hcpriv;
1579 	i = urb_priv->td_cnt;
1580 	if (i < urb_priv->length)
1581 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1582 				"Cancel URB %p, dev %s, ep 0x%x, "
1583 				"starting at offset 0x%llx",
1584 				urb, urb->dev->devpath,
1585 				urb->ep->desc.bEndpointAddress,
1586 				(unsigned long long) xhci_trb_virt_to_dma(
1587 					urb_priv->td[i]->start_seg,
1588 					urb_priv->td[i]->first_trb));
1589 
1590 	for (; i < urb_priv->length; i++) {
1591 		td = urb_priv->td[i];
1592 		list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list);
1593 	}
1594 
1595 	/* Queue a stop endpoint command, but only if this is
1596 	 * the first cancellation to be handled.
1597 	 */
1598 	if (!(ep->ep_state & EP_HALT_PENDING)) {
1599 		command = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
1600 		if (!command) {
1601 			ret = -ENOMEM;
1602 			goto done;
1603 		}
1604 		ep->ep_state |= EP_HALT_PENDING;
1605 		ep->stop_cmds_pending++;
1606 		ep->stop_cmd_timer.expires = jiffies +
1607 			XHCI_STOP_EP_CMD_TIMEOUT * HZ;
1608 		add_timer(&ep->stop_cmd_timer);
1609 		xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id,
1610 					 ep_index, 0);
1611 		xhci_ring_cmd_db(xhci);
1612 	}
1613 done:
1614 	spin_unlock_irqrestore(&xhci->lock, flags);
1615 	return ret;
1616 }
1617 
1618 /* Drop an endpoint from a new bandwidth configuration for this device.
1619  * Only one call to this function is allowed per endpoint before
1620  * check_bandwidth() or reset_bandwidth() must be called.
1621  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1622  * add the endpoint to the schedule with possibly new parameters denoted by a
1623  * different endpoint descriptor in usb_host_endpoint.
1624  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1625  * not allowed.
1626  *
1627  * The USB core will not allow URBs to be queued to an endpoint that is being
1628  * disabled, so there's no need for mutual exclusion to protect
1629  * the xhci->devs[slot_id] structure.
1630  */
1631 int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1632 		struct usb_host_endpoint *ep)
1633 {
1634 	struct xhci_hcd *xhci;
1635 	struct xhci_container_ctx *in_ctx, *out_ctx;
1636 	struct xhci_input_control_ctx *ctrl_ctx;
1637 	unsigned int ep_index;
1638 	struct xhci_ep_ctx *ep_ctx;
1639 	u32 drop_flag;
1640 	u32 new_add_flags, new_drop_flags;
1641 	int ret;
1642 
1643 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1644 	if (ret <= 0)
1645 		return ret;
1646 	xhci = hcd_to_xhci(hcd);
1647 	if (xhci->xhc_state & XHCI_STATE_DYING)
1648 		return -ENODEV;
1649 
1650 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
1651 	drop_flag = xhci_get_endpoint_flag(&ep->desc);
1652 	if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
1653 		xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
1654 				__func__, drop_flag);
1655 		return 0;
1656 	}
1657 
1658 	in_ctx = xhci->devs[udev->slot_id]->in_ctx;
1659 	out_ctx = xhci->devs[udev->slot_id]->out_ctx;
1660 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1661 	if (!ctrl_ctx) {
1662 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1663 				__func__);
1664 		return 0;
1665 	}
1666 
1667 	ep_index = xhci_get_endpoint_index(&ep->desc);
1668 	ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1669 	/* If the HC already knows the endpoint is disabled,
1670 	 * or the HCD has noted it is disabled, ignore this request
1671 	 */
1672 	if (((ep_ctx->ep_info & cpu_to_le32(EP_STATE_MASK)) ==
1673 	     cpu_to_le32(EP_STATE_DISABLED)) ||
1674 	    le32_to_cpu(ctrl_ctx->drop_flags) &
1675 	    xhci_get_endpoint_flag(&ep->desc)) {
1676 		/* Do not warn when called after a usb_device_reset */
1677 		if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)
1678 			xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
1679 				  __func__, ep);
1680 		return 0;
1681 	}
1682 
1683 	ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
1684 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1685 
1686 	ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
1687 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1688 
1689 	xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);
1690 
1691 	xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1692 			(unsigned int) ep->desc.bEndpointAddress,
1693 			udev->slot_id,
1694 			(unsigned int) new_drop_flags,
1695 			(unsigned int) new_add_flags);
1696 	return 0;
1697 }
1698 
1699 /* Add an endpoint to a new possible bandwidth configuration for this device.
1700  * Only one call to this function is allowed per endpoint before
1701  * check_bandwidth() or reset_bandwidth() must be called.
1702  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
1703  * add the endpoint to the schedule with possibly new parameters denoted by a
1704  * different endpoint descriptor in usb_host_endpoint.
1705  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
1706  * not allowed.
1707  *
1708  * The USB core will not allow URBs to be queued to an endpoint until the
1709  * configuration or alt setting is installed in the device, so there's no need
1710  * for mutual exclusion to protect the xhci->devs[slot_id] structure.
1711  */
1712 int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
1713 		struct usb_host_endpoint *ep)
1714 {
1715 	struct xhci_hcd *xhci;
1716 	struct xhci_container_ctx *in_ctx;
1717 	unsigned int ep_index;
1718 	struct xhci_input_control_ctx *ctrl_ctx;
1719 	u32 added_ctxs;
1720 	u32 new_add_flags, new_drop_flags;
1721 	struct xhci_virt_device *virt_dev;
1722 	int ret = 0;
1723 
1724 	ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
1725 	if (ret <= 0) {
1726 		/* So we won't queue a reset ep command for a root hub */
1727 		ep->hcpriv = NULL;
1728 		return ret;
1729 	}
1730 	xhci = hcd_to_xhci(hcd);
1731 	if (xhci->xhc_state & XHCI_STATE_DYING)
1732 		return -ENODEV;
1733 
1734 	added_ctxs = xhci_get_endpoint_flag(&ep->desc);
1735 	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
1736 		/* FIXME when we have to issue an evaluate endpoint command to
1737 		 * deal with ep0 max packet size changing once we get the
1738 		 * descriptors
1739 		 */
1740 		xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
1741 				__func__, added_ctxs);
1742 		return 0;
1743 	}
1744 
1745 	virt_dev = xhci->devs[udev->slot_id];
1746 	in_ctx = virt_dev->in_ctx;
1747 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
1748 	if (!ctrl_ctx) {
1749 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1750 				__func__);
1751 		return 0;
1752 	}
1753 
1754 	ep_index = xhci_get_endpoint_index(&ep->desc);
1755 	/* If this endpoint is already in use, and the upper layers are trying
1756 	 * to add it again without dropping it, reject the addition.
1757 	 */
1758 	if (virt_dev->eps[ep_index].ring &&
1759 			!(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {
1760 		xhci_warn(xhci, "Trying to add endpoint 0x%x "
1761 				"without dropping it.\n",
1762 				(unsigned int) ep->desc.bEndpointAddress);
1763 		return -EINVAL;
1764 	}
1765 
1766 	/* If the HCD has already noted the endpoint is enabled,
1767 	 * ignore this request.
1768 	 */
1769 	if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {
1770 		xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
1771 				__func__, ep);
1772 		return 0;
1773 	}
1774 
1775 	/*
1776 	 * Configuration and alternate setting changes must be done in
1777 	 * process context, not interrupt context (or so documenation
1778 	 * for usb_set_interface() and usb_set_configuration() claim).
1779 	 */
1780 	if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
1781 		dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
1782 				__func__, ep->desc.bEndpointAddress);
1783 		return -ENOMEM;
1784 	}
1785 
1786 	ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);
1787 	new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1788 
1789 	/* If xhci_endpoint_disable() was called for this endpoint, but the
1790 	 * xHC hasn't been notified yet through the check_bandwidth() call,
1791 	 * this re-adds a new state for the endpoint from the new endpoint
1792 	 * descriptors.  We must drop and re-add this endpoint, so we leave the
1793 	 * drop flags alone.
1794 	 */
1795 	new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1796 
1797 	/* Store the usb_device pointer for later use */
1798 	ep->hcpriv = udev;
1799 
1800 	xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
1801 			(unsigned int) ep->desc.bEndpointAddress,
1802 			udev->slot_id,
1803 			(unsigned int) new_drop_flags,
1804 			(unsigned int) new_add_flags);
1805 	return 0;
1806 }
1807 
1808 static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
1809 {
1810 	struct xhci_input_control_ctx *ctrl_ctx;
1811 	struct xhci_ep_ctx *ep_ctx;
1812 	struct xhci_slot_ctx *slot_ctx;
1813 	int i;
1814 
1815 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
1816 	if (!ctrl_ctx) {
1817 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
1818 				__func__);
1819 		return;
1820 	}
1821 
1822 	/* When a device's add flag and drop flag are zero, any subsequent
1823 	 * configure endpoint command will leave that endpoint's state
1824 	 * untouched.  Make sure we don't leave any old state in the input
1825 	 * endpoint contexts.
1826 	 */
1827 	ctrl_ctx->drop_flags = 0;
1828 	ctrl_ctx->add_flags = 0;
1829 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
1830 	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
1831 	/* Endpoint 0 is always valid */
1832 	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
1833 	for (i = 1; i < 31; ++i) {
1834 		ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
1835 		ep_ctx->ep_info = 0;
1836 		ep_ctx->ep_info2 = 0;
1837 		ep_ctx->deq = 0;
1838 		ep_ctx->tx_info = 0;
1839 	}
1840 }
1841 
1842 static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
1843 		struct usb_device *udev, u32 *cmd_status)
1844 {
1845 	int ret;
1846 
1847 	switch (*cmd_status) {
1848 	case COMP_CMD_ABORT:
1849 	case COMP_CMD_STOP:
1850 		xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n");
1851 		ret = -ETIME;
1852 		break;
1853 	case COMP_ENOMEM:
1854 		dev_warn(&udev->dev,
1855 			 "Not enough host controller resources for new device state.\n");
1856 		ret = -ENOMEM;
1857 		/* FIXME: can we allocate more resources for the HC? */
1858 		break;
1859 	case COMP_BW_ERR:
1860 	case COMP_2ND_BW_ERR:
1861 		dev_warn(&udev->dev,
1862 			 "Not enough bandwidth for new device state.\n");
1863 		ret = -ENOSPC;
1864 		/* FIXME: can we go back to the old state? */
1865 		break;
1866 	case COMP_TRB_ERR:
1867 		/* the HCD set up something wrong */
1868 		dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
1869 				"add flag = 1, "
1870 				"and endpoint is not disabled.\n");
1871 		ret = -EINVAL;
1872 		break;
1873 	case COMP_DEV_ERR:
1874 		dev_warn(&udev->dev,
1875 			 "ERROR: Incompatible device for endpoint configure command.\n");
1876 		ret = -ENODEV;
1877 		break;
1878 	case COMP_SUCCESS:
1879 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
1880 				"Successful Endpoint Configure command");
1881 		ret = 0;
1882 		break;
1883 	default:
1884 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
1885 				*cmd_status);
1886 		ret = -EINVAL;
1887 		break;
1888 	}
1889 	return ret;
1890 }
1891 
1892 static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
1893 		struct usb_device *udev, u32 *cmd_status)
1894 {
1895 	int ret;
1896 	struct xhci_virt_device *virt_dev = xhci->devs[udev->slot_id];
1897 
1898 	switch (*cmd_status) {
1899 	case COMP_CMD_ABORT:
1900 	case COMP_CMD_STOP:
1901 		xhci_warn(xhci, "Timeout while waiting for evaluate context command\n");
1902 		ret = -ETIME;
1903 		break;
1904 	case COMP_EINVAL:
1905 		dev_warn(&udev->dev,
1906 			 "WARN: xHCI driver setup invalid evaluate context command.\n");
1907 		ret = -EINVAL;
1908 		break;
1909 	case COMP_EBADSLT:
1910 		dev_warn(&udev->dev,
1911 			"WARN: slot not enabled for evaluate context command.\n");
1912 		ret = -EINVAL;
1913 		break;
1914 	case COMP_CTX_STATE:
1915 		dev_warn(&udev->dev,
1916 			"WARN: invalid context state for evaluate context command.\n");
1917 		xhci_dbg_ctx(xhci, virt_dev->out_ctx, 1);
1918 		ret = -EINVAL;
1919 		break;
1920 	case COMP_DEV_ERR:
1921 		dev_warn(&udev->dev,
1922 			"ERROR: Incompatible device for evaluate context command.\n");
1923 		ret = -ENODEV;
1924 		break;
1925 	case COMP_MEL_ERR:
1926 		/* Max Exit Latency too large error */
1927 		dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");
1928 		ret = -EINVAL;
1929 		break;
1930 	case COMP_SUCCESS:
1931 		xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
1932 				"Successful evaluate context command");
1933 		ret = 0;
1934 		break;
1935 	default:
1936 		xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
1937 			*cmd_status);
1938 		ret = -EINVAL;
1939 		break;
1940 	}
1941 	return ret;
1942 }
1943 
1944 static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
1945 		struct xhci_input_control_ctx *ctrl_ctx)
1946 {
1947 	u32 valid_add_flags;
1948 	u32 valid_drop_flags;
1949 
1950 	/* Ignore the slot flag (bit 0), and the default control endpoint flag
1951 	 * (bit 1).  The default control endpoint is added during the Address
1952 	 * Device command and is never removed until the slot is disabled.
1953 	 */
1954 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
1955 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
1956 
1957 	/* Use hweight32 to count the number of ones in the add flags, or
1958 	 * number of endpoints added.  Don't count endpoints that are changed
1959 	 * (both added and dropped).
1960 	 */
1961 	return hweight32(valid_add_flags) -
1962 		hweight32(valid_add_flags & valid_drop_flags);
1963 }
1964 
1965 static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
1966 		struct xhci_input_control_ctx *ctrl_ctx)
1967 {
1968 	u32 valid_add_flags;
1969 	u32 valid_drop_flags;
1970 
1971 	valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
1972 	valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
1973 
1974 	return hweight32(valid_drop_flags) -
1975 		hweight32(valid_add_flags & valid_drop_flags);
1976 }
1977 
1978 /*
1979  * We need to reserve the new number of endpoints before the configure endpoint
1980  * command completes.  We can't subtract the dropped endpoints from the number
1981  * of active endpoints until the command completes because we can oversubscribe
1982  * the host in this case:
1983  *
1984  *  - the first configure endpoint command drops more endpoints than it adds
1985  *  - a second configure endpoint command that adds more endpoints is queued
1986  *  - the first configure endpoint command fails, so the config is unchanged
1987  *  - the second command may succeed, even though there isn't enough resources
1988  *
1989  * Must be called with xhci->lock held.
1990  */
1991 static int xhci_reserve_host_resources(struct xhci_hcd *xhci,
1992 		struct xhci_input_control_ctx *ctrl_ctx)
1993 {
1994 	u32 added_eps;
1995 
1996 	added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
1997 	if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {
1998 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1999 				"Not enough ep ctxs: "
2000 				"%u active, need to add %u, limit is %u.",
2001 				xhci->num_active_eps, added_eps,
2002 				xhci->limit_active_eps);
2003 		return -ENOMEM;
2004 	}
2005 	xhci->num_active_eps += added_eps;
2006 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2007 			"Adding %u ep ctxs, %u now active.", added_eps,
2008 			xhci->num_active_eps);
2009 	return 0;
2010 }
2011 
2012 /*
2013  * The configure endpoint was failed by the xHC for some other reason, so we
2014  * need to revert the resources that failed configuration would have used.
2015  *
2016  * Must be called with xhci->lock held.
2017  */
2018 static void xhci_free_host_resources(struct xhci_hcd *xhci,
2019 		struct xhci_input_control_ctx *ctrl_ctx)
2020 {
2021 	u32 num_failed_eps;
2022 
2023 	num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
2024 	xhci->num_active_eps -= num_failed_eps;
2025 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2026 			"Removing %u failed ep ctxs, %u now active.",
2027 			num_failed_eps,
2028 			xhci->num_active_eps);
2029 }
2030 
2031 /*
2032  * Now that the command has completed, clean up the active endpoint count by
2033  * subtracting out the endpoints that were dropped (but not changed).
2034  *
2035  * Must be called with xhci->lock held.
2036  */
2037 static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,
2038 		struct xhci_input_control_ctx *ctrl_ctx)
2039 {
2040 	u32 num_dropped_eps;
2041 
2042 	num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx);
2043 	xhci->num_active_eps -= num_dropped_eps;
2044 	if (num_dropped_eps)
2045 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2046 				"Removing %u dropped ep ctxs, %u now active.",
2047 				num_dropped_eps,
2048 				xhci->num_active_eps);
2049 }
2050 
2051 static unsigned int xhci_get_block_size(struct usb_device *udev)
2052 {
2053 	switch (udev->speed) {
2054 	case USB_SPEED_LOW:
2055 	case USB_SPEED_FULL:
2056 		return FS_BLOCK;
2057 	case USB_SPEED_HIGH:
2058 		return HS_BLOCK;
2059 	case USB_SPEED_SUPER:
2060 		return SS_BLOCK;
2061 	case USB_SPEED_UNKNOWN:
2062 	case USB_SPEED_WIRELESS:
2063 	default:
2064 		/* Should never happen */
2065 		return 1;
2066 	}
2067 }
2068 
2069 static unsigned int
2070 xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw)
2071 {
2072 	if (interval_bw->overhead[LS_OVERHEAD_TYPE])
2073 		return LS_OVERHEAD;
2074 	if (interval_bw->overhead[FS_OVERHEAD_TYPE])
2075 		return FS_OVERHEAD;
2076 	return HS_OVERHEAD;
2077 }
2078 
2079 /* If we are changing a LS/FS device under a HS hub,
2080  * make sure (if we are activating a new TT) that the HS bus has enough
2081  * bandwidth for this new TT.
2082  */
2083 static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,
2084 		struct xhci_virt_device *virt_dev,
2085 		int old_active_eps)
2086 {
2087 	struct xhci_interval_bw_table *bw_table;
2088 	struct xhci_tt_bw_info *tt_info;
2089 
2090 	/* Find the bandwidth table for the root port this TT is attached to. */
2091 	bw_table = &xhci->rh_bw[virt_dev->real_port - 1].bw_table;
2092 	tt_info = virt_dev->tt_info;
2093 	/* If this TT already had active endpoints, the bandwidth for this TT
2094 	 * has already been added.  Removing all periodic endpoints (and thus
2095 	 * making the TT enactive) will only decrease the bandwidth used.
2096 	 */
2097 	if (old_active_eps)
2098 		return 0;
2099 	if (old_active_eps == 0 && tt_info->active_eps != 0) {
2100 		if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)
2101 			return -ENOMEM;
2102 		return 0;
2103 	}
2104 	/* Not sure why we would have no new active endpoints...
2105 	 *
2106 	 * Maybe because of an Evaluate Context change for a hub update or a
2107 	 * control endpoint 0 max packet size change?
2108 	 * FIXME: skip the bandwidth calculation in that case.
2109 	 */
2110 	return 0;
2111 }
2112 
2113 static int xhci_check_ss_bw(struct xhci_hcd *xhci,
2114 		struct xhci_virt_device *virt_dev)
2115 {
2116 	unsigned int bw_reserved;
2117 
2118 	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100);
2119 	if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))
2120 		return -ENOMEM;
2121 
2122 	bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100);
2123 	if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))
2124 		return -ENOMEM;
2125 
2126 	return 0;
2127 }
2128 
2129 /*
2130  * This algorithm is a very conservative estimate of the worst-case scheduling
2131  * scenario for any one interval.  The hardware dynamically schedules the
2132  * packets, so we can't tell which microframe could be the limiting factor in
2133  * the bandwidth scheduling.  This only takes into account periodic endpoints.
2134  *
2135  * Obviously, we can't solve an NP complete problem to find the minimum worst
2136  * case scenario.  Instead, we come up with an estimate that is no less than
2137  * the worst case bandwidth used for any one microframe, but may be an
2138  * over-estimate.
2139  *
2140  * We walk the requirements for each endpoint by interval, starting with the
2141  * smallest interval, and place packets in the schedule where there is only one
2142  * possible way to schedule packets for that interval.  In order to simplify
2143  * this algorithm, we record the largest max packet size for each interval, and
2144  * assume all packets will be that size.
2145  *
2146  * For interval 0, we obviously must schedule all packets for each interval.
2147  * The bandwidth for interval 0 is just the amount of data to be transmitted
2148  * (the sum of all max ESIT payload sizes, plus any overhead per packet times
2149  * the number of packets).
2150  *
2151  * For interval 1, we have two possible microframes to schedule those packets
2152  * in.  For this algorithm, if we can schedule the same number of packets for
2153  * each possible scheduling opportunity (each microframe), we will do so.  The
2154  * remaining number of packets will be saved to be transmitted in the gaps in
2155  * the next interval's scheduling sequence.
2156  *
2157  * As we move those remaining packets to be scheduled with interval 2 packets,
2158  * we have to double the number of remaining packets to transmit.  This is
2159  * because the intervals are actually powers of 2, and we would be transmitting
2160  * the previous interval's packets twice in this interval.  We also have to be
2161  * sure that when we look at the largest max packet size for this interval, we
2162  * also look at the largest max packet size for the remaining packets and take
2163  * the greater of the two.
2164  *
2165  * The algorithm continues to evenly distribute packets in each scheduling
2166  * opportunity, and push the remaining packets out, until we get to the last
2167  * interval.  Then those packets and their associated overhead are just added
2168  * to the bandwidth used.
2169  */
2170 static int xhci_check_bw_table(struct xhci_hcd *xhci,
2171 		struct xhci_virt_device *virt_dev,
2172 		int old_active_eps)
2173 {
2174 	unsigned int bw_reserved;
2175 	unsigned int max_bandwidth;
2176 	unsigned int bw_used;
2177 	unsigned int block_size;
2178 	struct xhci_interval_bw_table *bw_table;
2179 	unsigned int packet_size = 0;
2180 	unsigned int overhead = 0;
2181 	unsigned int packets_transmitted = 0;
2182 	unsigned int packets_remaining = 0;
2183 	unsigned int i;
2184 
2185 	if (virt_dev->udev->speed == USB_SPEED_SUPER)
2186 		return xhci_check_ss_bw(xhci, virt_dev);
2187 
2188 	if (virt_dev->udev->speed == USB_SPEED_HIGH) {
2189 		max_bandwidth = HS_BW_LIMIT;
2190 		/* Convert percent of bus BW reserved to blocks reserved */
2191 		bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100);
2192 	} else {
2193 		max_bandwidth = FS_BW_LIMIT;
2194 		bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100);
2195 	}
2196 
2197 	bw_table = virt_dev->bw_table;
2198 	/* We need to translate the max packet size and max ESIT payloads into
2199 	 * the units the hardware uses.
2200 	 */
2201 	block_size = xhci_get_block_size(virt_dev->udev);
2202 
2203 	/* If we are manipulating a LS/FS device under a HS hub, double check
2204 	 * that the HS bus has enough bandwidth if we are activing a new TT.
2205 	 */
2206 	if (virt_dev->tt_info) {
2207 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2208 				"Recalculating BW for rootport %u",
2209 				virt_dev->real_port);
2210 		if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {
2211 			xhci_warn(xhci, "Not enough bandwidth on HS bus for "
2212 					"newly activated TT.\n");
2213 			return -ENOMEM;
2214 		}
2215 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2216 				"Recalculating BW for TT slot %u port %u",
2217 				virt_dev->tt_info->slot_id,
2218 				virt_dev->tt_info->ttport);
2219 	} else {
2220 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2221 				"Recalculating BW for rootport %u",
2222 				virt_dev->real_port);
2223 	}
2224 
2225 	/* Add in how much bandwidth will be used for interval zero, or the
2226 	 * rounded max ESIT payload + number of packets * largest overhead.
2227 	 */
2228 	bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +
2229 		bw_table->interval_bw[0].num_packets *
2230 		xhci_get_largest_overhead(&bw_table->interval_bw[0]);
2231 
2232 	for (i = 1; i < XHCI_MAX_INTERVAL; i++) {
2233 		unsigned int bw_added;
2234 		unsigned int largest_mps;
2235 		unsigned int interval_overhead;
2236 
2237 		/*
2238 		 * How many packets could we transmit in this interval?
2239 		 * If packets didn't fit in the previous interval, we will need
2240 		 * to transmit that many packets twice within this interval.
2241 		 */
2242 		packets_remaining = 2 * packets_remaining +
2243 			bw_table->interval_bw[i].num_packets;
2244 
2245 		/* Find the largest max packet size of this or the previous
2246 		 * interval.
2247 		 */
2248 		if (list_empty(&bw_table->interval_bw[i].endpoints))
2249 			largest_mps = 0;
2250 		else {
2251 			struct xhci_virt_ep *virt_ep;
2252 			struct list_head *ep_entry;
2253 
2254 			ep_entry = bw_table->interval_bw[i].endpoints.next;
2255 			virt_ep = list_entry(ep_entry,
2256 					struct xhci_virt_ep, bw_endpoint_list);
2257 			/* Convert to blocks, rounding up */
2258 			largest_mps = DIV_ROUND_UP(
2259 					virt_ep->bw_info.max_packet_size,
2260 					block_size);
2261 		}
2262 		if (largest_mps > packet_size)
2263 			packet_size = largest_mps;
2264 
2265 		/* Use the larger overhead of this or the previous interval. */
2266 		interval_overhead = xhci_get_largest_overhead(
2267 				&bw_table->interval_bw[i]);
2268 		if (interval_overhead > overhead)
2269 			overhead = interval_overhead;
2270 
2271 		/* How many packets can we evenly distribute across
2272 		 * (1 << (i + 1)) possible scheduling opportunities?
2273 		 */
2274 		packets_transmitted = packets_remaining >> (i + 1);
2275 
2276 		/* Add in the bandwidth used for those scheduled packets */
2277 		bw_added = packets_transmitted * (overhead + packet_size);
2278 
2279 		/* How many packets do we have remaining to transmit? */
2280 		packets_remaining = packets_remaining % (1 << (i + 1));
2281 
2282 		/* What largest max packet size should those packets have? */
2283 		/* If we've transmitted all packets, don't carry over the
2284 		 * largest packet size.
2285 		 */
2286 		if (packets_remaining == 0) {
2287 			packet_size = 0;
2288 			overhead = 0;
2289 		} else if (packets_transmitted > 0) {
2290 			/* Otherwise if we do have remaining packets, and we've
2291 			 * scheduled some packets in this interval, take the
2292 			 * largest max packet size from endpoints with this
2293 			 * interval.
2294 			 */
2295 			packet_size = largest_mps;
2296 			overhead = interval_overhead;
2297 		}
2298 		/* Otherwise carry over packet_size and overhead from the last
2299 		 * time we had a remainder.
2300 		 */
2301 		bw_used += bw_added;
2302 		if (bw_used > max_bandwidth) {
2303 			xhci_warn(xhci, "Not enough bandwidth. "
2304 					"Proposed: %u, Max: %u\n",
2305 				bw_used, max_bandwidth);
2306 			return -ENOMEM;
2307 		}
2308 	}
2309 	/*
2310 	 * Ok, we know we have some packets left over after even-handedly
2311 	 * scheduling interval 15.  We don't know which microframes they will
2312 	 * fit into, so we over-schedule and say they will be scheduled every
2313 	 * microframe.
2314 	 */
2315 	if (packets_remaining > 0)
2316 		bw_used += overhead + packet_size;
2317 
2318 	if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {
2319 		unsigned int port_index = virt_dev->real_port - 1;
2320 
2321 		/* OK, we're manipulating a HS device attached to a
2322 		 * root port bandwidth domain.  Include the number of active TTs
2323 		 * in the bandwidth used.
2324 		 */
2325 		bw_used += TT_HS_OVERHEAD *
2326 			xhci->rh_bw[port_index].num_active_tts;
2327 	}
2328 
2329 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2330 		"Final bandwidth: %u, Limit: %u, Reserved: %u, "
2331 		"Available: %u " "percent",
2332 		bw_used, max_bandwidth, bw_reserved,
2333 		(max_bandwidth - bw_used - bw_reserved) * 100 /
2334 		max_bandwidth);
2335 
2336 	bw_used += bw_reserved;
2337 	if (bw_used > max_bandwidth) {
2338 		xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",
2339 				bw_used, max_bandwidth);
2340 		return -ENOMEM;
2341 	}
2342 
2343 	bw_table->bw_used = bw_used;
2344 	return 0;
2345 }
2346 
2347 static bool xhci_is_async_ep(unsigned int ep_type)
2348 {
2349 	return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
2350 					ep_type != ISOC_IN_EP &&
2351 					ep_type != INT_IN_EP);
2352 }
2353 
2354 static bool xhci_is_sync_in_ep(unsigned int ep_type)
2355 {
2356 	return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP);
2357 }
2358 
2359 static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw)
2360 {
2361 	unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);
2362 
2363 	if (ep_bw->ep_interval == 0)
2364 		return SS_OVERHEAD_BURST +
2365 			(ep_bw->mult * ep_bw->num_packets *
2366 					(SS_OVERHEAD + mps));
2367 	return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *
2368 				(SS_OVERHEAD + mps + SS_OVERHEAD_BURST),
2369 				1 << ep_bw->ep_interval);
2370 
2371 }
2372 
2373 void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,
2374 		struct xhci_bw_info *ep_bw,
2375 		struct xhci_interval_bw_table *bw_table,
2376 		struct usb_device *udev,
2377 		struct xhci_virt_ep *virt_ep,
2378 		struct xhci_tt_bw_info *tt_info)
2379 {
2380 	struct xhci_interval_bw	*interval_bw;
2381 	int normalized_interval;
2382 
2383 	if (xhci_is_async_ep(ep_bw->type))
2384 		return;
2385 
2386 	if (udev->speed == USB_SPEED_SUPER) {
2387 		if (xhci_is_sync_in_ep(ep_bw->type))
2388 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=
2389 				xhci_get_ss_bw_consumed(ep_bw);
2390 		else
2391 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=
2392 				xhci_get_ss_bw_consumed(ep_bw);
2393 		return;
2394 	}
2395 
2396 	/* SuperSpeed endpoints never get added to intervals in the table, so
2397 	 * this check is only valid for HS/FS/LS devices.
2398 	 */
2399 	if (list_empty(&virt_ep->bw_endpoint_list))
2400 		return;
2401 	/* For LS/FS devices, we need to translate the interval expressed in
2402 	 * microframes to frames.
2403 	 */
2404 	if (udev->speed == USB_SPEED_HIGH)
2405 		normalized_interval = ep_bw->ep_interval;
2406 	else
2407 		normalized_interval = ep_bw->ep_interval - 3;
2408 
2409 	if (normalized_interval == 0)
2410 		bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;
2411 	interval_bw = &bw_table->interval_bw[normalized_interval];
2412 	interval_bw->num_packets -= ep_bw->num_packets;
2413 	switch (udev->speed) {
2414 	case USB_SPEED_LOW:
2415 		interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;
2416 		break;
2417 	case USB_SPEED_FULL:
2418 		interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;
2419 		break;
2420 	case USB_SPEED_HIGH:
2421 		interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;
2422 		break;
2423 	case USB_SPEED_SUPER:
2424 	case USB_SPEED_UNKNOWN:
2425 	case USB_SPEED_WIRELESS:
2426 		/* Should never happen because only LS/FS/HS endpoints will get
2427 		 * added to the endpoint list.
2428 		 */
2429 		return;
2430 	}
2431 	if (tt_info)
2432 		tt_info->active_eps -= 1;
2433 	list_del_init(&virt_ep->bw_endpoint_list);
2434 }
2435 
2436 static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,
2437 		struct xhci_bw_info *ep_bw,
2438 		struct xhci_interval_bw_table *bw_table,
2439 		struct usb_device *udev,
2440 		struct xhci_virt_ep *virt_ep,
2441 		struct xhci_tt_bw_info *tt_info)
2442 {
2443 	struct xhci_interval_bw	*interval_bw;
2444 	struct xhci_virt_ep *smaller_ep;
2445 	int normalized_interval;
2446 
2447 	if (xhci_is_async_ep(ep_bw->type))
2448 		return;
2449 
2450 	if (udev->speed == USB_SPEED_SUPER) {
2451 		if (xhci_is_sync_in_ep(ep_bw->type))
2452 			xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=
2453 				xhci_get_ss_bw_consumed(ep_bw);
2454 		else
2455 			xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=
2456 				xhci_get_ss_bw_consumed(ep_bw);
2457 		return;
2458 	}
2459 
2460 	/* For LS/FS devices, we need to translate the interval expressed in
2461 	 * microframes to frames.
2462 	 */
2463 	if (udev->speed == USB_SPEED_HIGH)
2464 		normalized_interval = ep_bw->ep_interval;
2465 	else
2466 		normalized_interval = ep_bw->ep_interval - 3;
2467 
2468 	if (normalized_interval == 0)
2469 		bw_table->interval0_esit_payload += ep_bw->max_esit_payload;
2470 	interval_bw = &bw_table->interval_bw[normalized_interval];
2471 	interval_bw->num_packets += ep_bw->num_packets;
2472 	switch (udev->speed) {
2473 	case USB_SPEED_LOW:
2474 		interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;
2475 		break;
2476 	case USB_SPEED_FULL:
2477 		interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;
2478 		break;
2479 	case USB_SPEED_HIGH:
2480 		interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;
2481 		break;
2482 	case USB_SPEED_SUPER:
2483 	case USB_SPEED_UNKNOWN:
2484 	case USB_SPEED_WIRELESS:
2485 		/* Should never happen because only LS/FS/HS endpoints will get
2486 		 * added to the endpoint list.
2487 		 */
2488 		return;
2489 	}
2490 
2491 	if (tt_info)
2492 		tt_info->active_eps += 1;
2493 	/* Insert the endpoint into the list, largest max packet size first. */
2494 	list_for_each_entry(smaller_ep, &interval_bw->endpoints,
2495 			bw_endpoint_list) {
2496 		if (ep_bw->max_packet_size >=
2497 				smaller_ep->bw_info.max_packet_size) {
2498 			/* Add the new ep before the smaller endpoint */
2499 			list_add_tail(&virt_ep->bw_endpoint_list,
2500 					&smaller_ep->bw_endpoint_list);
2501 			return;
2502 		}
2503 	}
2504 	/* Add the new endpoint at the end of the list. */
2505 	list_add_tail(&virt_ep->bw_endpoint_list,
2506 			&interval_bw->endpoints);
2507 }
2508 
2509 void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
2510 		struct xhci_virt_device *virt_dev,
2511 		int old_active_eps)
2512 {
2513 	struct xhci_root_port_bw_info *rh_bw_info;
2514 	if (!virt_dev->tt_info)
2515 		return;
2516 
2517 	rh_bw_info = &xhci->rh_bw[virt_dev->real_port - 1];
2518 	if (old_active_eps == 0 &&
2519 				virt_dev->tt_info->active_eps != 0) {
2520 		rh_bw_info->num_active_tts += 1;
2521 		rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;
2522 	} else if (old_active_eps != 0 &&
2523 				virt_dev->tt_info->active_eps == 0) {
2524 		rh_bw_info->num_active_tts -= 1;
2525 		rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;
2526 	}
2527 }
2528 
2529 static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
2530 		struct xhci_virt_device *virt_dev,
2531 		struct xhci_container_ctx *in_ctx)
2532 {
2533 	struct xhci_bw_info ep_bw_info[31];
2534 	int i;
2535 	struct xhci_input_control_ctx *ctrl_ctx;
2536 	int old_active_eps = 0;
2537 
2538 	if (virt_dev->tt_info)
2539 		old_active_eps = virt_dev->tt_info->active_eps;
2540 
2541 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2542 	if (!ctrl_ctx) {
2543 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2544 				__func__);
2545 		return -ENOMEM;
2546 	}
2547 
2548 	for (i = 0; i < 31; i++) {
2549 		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2550 			continue;
2551 
2552 		/* Make a copy of the BW info in case we need to revert this */
2553 		memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
2554 				sizeof(ep_bw_info[i]));
2555 		/* Drop the endpoint from the interval table if the endpoint is
2556 		 * being dropped or changed.
2557 		 */
2558 		if (EP_IS_DROPPED(ctrl_ctx, i))
2559 			xhci_drop_ep_from_interval_table(xhci,
2560 					&virt_dev->eps[i].bw_info,
2561 					virt_dev->bw_table,
2562 					virt_dev->udev,
2563 					&virt_dev->eps[i],
2564 					virt_dev->tt_info);
2565 	}
2566 	/* Overwrite the information stored in the endpoints' bw_info */
2567 	xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
2568 	for (i = 0; i < 31; i++) {
2569 		/* Add any changed or added endpoints to the interval table */
2570 		if (EP_IS_ADDED(ctrl_ctx, i))
2571 			xhci_add_ep_to_interval_table(xhci,
2572 					&virt_dev->eps[i].bw_info,
2573 					virt_dev->bw_table,
2574 					virt_dev->udev,
2575 					&virt_dev->eps[i],
2576 					virt_dev->tt_info);
2577 	}
2578 
2579 	if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
2580 		/* Ok, this fits in the bandwidth we have.
2581 		 * Update the number of active TTs.
2582 		 */
2583 		xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
2584 		return 0;
2585 	}
2586 
2587 	/* We don't have enough bandwidth for this, revert the stored info. */
2588 	for (i = 0; i < 31; i++) {
2589 		if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
2590 			continue;
2591 
2592 		/* Drop the new copies of any added or changed endpoints from
2593 		 * the interval table.
2594 		 */
2595 		if (EP_IS_ADDED(ctrl_ctx, i)) {
2596 			xhci_drop_ep_from_interval_table(xhci,
2597 					&virt_dev->eps[i].bw_info,
2598 					virt_dev->bw_table,
2599 					virt_dev->udev,
2600 					&virt_dev->eps[i],
2601 					virt_dev->tt_info);
2602 		}
2603 		/* Revert the endpoint back to its old information */
2604 		memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
2605 				sizeof(ep_bw_info[i]));
2606 		/* Add any changed or dropped endpoints back into the table */
2607 		if (EP_IS_DROPPED(ctrl_ctx, i))
2608 			xhci_add_ep_to_interval_table(xhci,
2609 					&virt_dev->eps[i].bw_info,
2610 					virt_dev->bw_table,
2611 					virt_dev->udev,
2612 					&virt_dev->eps[i],
2613 					virt_dev->tt_info);
2614 	}
2615 	return -ENOMEM;
2616 }
2617 
2618 
2619 /* Issue a configure endpoint command or evaluate context command
2620  * and wait for it to finish.
2621  */
2622 static int xhci_configure_endpoint(struct xhci_hcd *xhci,
2623 		struct usb_device *udev,
2624 		struct xhci_command *command,
2625 		bool ctx_change, bool must_succeed)
2626 {
2627 	int ret;
2628 	unsigned long flags;
2629 	struct xhci_input_control_ctx *ctrl_ctx;
2630 	struct xhci_virt_device *virt_dev;
2631 
2632 	if (!command)
2633 		return -EINVAL;
2634 
2635 	spin_lock_irqsave(&xhci->lock, flags);
2636 	virt_dev = xhci->devs[udev->slot_id];
2637 
2638 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2639 	if (!ctrl_ctx) {
2640 		spin_unlock_irqrestore(&xhci->lock, flags);
2641 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2642 				__func__);
2643 		return -ENOMEM;
2644 	}
2645 
2646 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&
2647 			xhci_reserve_host_resources(xhci, ctrl_ctx)) {
2648 		spin_unlock_irqrestore(&xhci->lock, flags);
2649 		xhci_warn(xhci, "Not enough host resources, "
2650 				"active endpoint contexts = %u\n",
2651 				xhci->num_active_eps);
2652 		return -ENOMEM;
2653 	}
2654 	if ((xhci->quirks & XHCI_SW_BW_CHECKING) &&
2655 	    xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) {
2656 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2657 			xhci_free_host_resources(xhci, ctrl_ctx);
2658 		spin_unlock_irqrestore(&xhci->lock, flags);
2659 		xhci_warn(xhci, "Not enough bandwidth\n");
2660 		return -ENOMEM;
2661 	}
2662 
2663 	if (!ctx_change)
2664 		ret = xhci_queue_configure_endpoint(xhci, command,
2665 				command->in_ctx->dma,
2666 				udev->slot_id, must_succeed);
2667 	else
2668 		ret = xhci_queue_evaluate_context(xhci, command,
2669 				command->in_ctx->dma,
2670 				udev->slot_id, must_succeed);
2671 	if (ret < 0) {
2672 		if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
2673 			xhci_free_host_resources(xhci, ctrl_ctx);
2674 		spin_unlock_irqrestore(&xhci->lock, flags);
2675 		xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
2676 				"FIXME allocate a new ring segment");
2677 		return -ENOMEM;
2678 	}
2679 	xhci_ring_cmd_db(xhci);
2680 	spin_unlock_irqrestore(&xhci->lock, flags);
2681 
2682 	/* Wait for the configure endpoint command to complete */
2683 	wait_for_completion(command->completion);
2684 
2685 	if (!ctx_change)
2686 		ret = xhci_configure_endpoint_result(xhci, udev,
2687 						     &command->status);
2688 	else
2689 		ret = xhci_evaluate_context_result(xhci, udev,
2690 						   &command->status);
2691 
2692 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
2693 		spin_lock_irqsave(&xhci->lock, flags);
2694 		/* If the command failed, remove the reserved resources.
2695 		 * Otherwise, clean up the estimate to include dropped eps.
2696 		 */
2697 		if (ret)
2698 			xhci_free_host_resources(xhci, ctrl_ctx);
2699 		else
2700 			xhci_finish_resource_reservation(xhci, ctrl_ctx);
2701 		spin_unlock_irqrestore(&xhci->lock, flags);
2702 	}
2703 	return ret;
2704 }
2705 
2706 static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci,
2707 	struct xhci_virt_device *vdev, int i)
2708 {
2709 	struct xhci_virt_ep *ep = &vdev->eps[i];
2710 
2711 	if (ep->ep_state & EP_HAS_STREAMS) {
2712 		xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n",
2713 				xhci_get_endpoint_address(i));
2714 		xhci_free_stream_info(xhci, ep->stream_info);
2715 		ep->stream_info = NULL;
2716 		ep->ep_state &= ~EP_HAS_STREAMS;
2717 	}
2718 }
2719 
2720 /* Called after one or more calls to xhci_add_endpoint() or
2721  * xhci_drop_endpoint().  If this call fails, the USB core is expected
2722  * to call xhci_reset_bandwidth().
2723  *
2724  * Since we are in the middle of changing either configuration or
2725  * installing a new alt setting, the USB core won't allow URBs to be
2726  * enqueued for any endpoint on the old config or interface.  Nothing
2727  * else should be touching the xhci->devs[slot_id] structure, so we
2728  * don't need to take the xhci->lock for manipulating that.
2729  */
2730 int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2731 {
2732 	int i;
2733 	int ret = 0;
2734 	struct xhci_hcd *xhci;
2735 	struct xhci_virt_device	*virt_dev;
2736 	struct xhci_input_control_ctx *ctrl_ctx;
2737 	struct xhci_slot_ctx *slot_ctx;
2738 	struct xhci_command *command;
2739 
2740 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2741 	if (ret <= 0)
2742 		return ret;
2743 	xhci = hcd_to_xhci(hcd);
2744 	if (xhci->xhc_state & XHCI_STATE_DYING)
2745 		return -ENODEV;
2746 
2747 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2748 	virt_dev = xhci->devs[udev->slot_id];
2749 
2750 	command = xhci_alloc_command(xhci, false, true, GFP_KERNEL);
2751 	if (!command)
2752 		return -ENOMEM;
2753 
2754 	command->in_ctx = virt_dev->in_ctx;
2755 
2756 	/* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */
2757 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
2758 	if (!ctrl_ctx) {
2759 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2760 				__func__);
2761 		ret = -ENOMEM;
2762 		goto command_cleanup;
2763 	}
2764 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
2765 	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
2766 	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
2767 
2768 	/* Don't issue the command if there's no endpoints to update. */
2769 	if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
2770 	    ctrl_ctx->drop_flags == 0) {
2771 		ret = 0;
2772 		goto command_cleanup;
2773 	}
2774 	/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
2775 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
2776 	for (i = 31; i >= 1; i--) {
2777 		__le32 le32 = cpu_to_le32(BIT(i));
2778 
2779 		if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))
2780 		    || (ctrl_ctx->add_flags & le32) || i == 1) {
2781 			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
2782 			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
2783 			break;
2784 		}
2785 	}
2786 	xhci_dbg(xhci, "New Input Control Context:\n");
2787 	xhci_dbg_ctx(xhci, virt_dev->in_ctx,
2788 		     LAST_CTX_TO_EP_NUM(le32_to_cpu(slot_ctx->dev_info)));
2789 
2790 	ret = xhci_configure_endpoint(xhci, udev, command,
2791 			false, false);
2792 	if (ret)
2793 		/* Callee should call reset_bandwidth() */
2794 		goto command_cleanup;
2795 
2796 	xhci_dbg(xhci, "Output context after successful config ep cmd:\n");
2797 	xhci_dbg_ctx(xhci, virt_dev->out_ctx,
2798 		     LAST_CTX_TO_EP_NUM(le32_to_cpu(slot_ctx->dev_info)));
2799 
2800 	/* Free any rings that were dropped, but not changed. */
2801 	for (i = 1; i < 31; ++i) {
2802 		if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
2803 		    !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {
2804 			xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
2805 			xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
2806 		}
2807 	}
2808 	xhci_zero_in_ctx(xhci, virt_dev);
2809 	/*
2810 	 * Install any rings for completely new endpoints or changed endpoints,
2811 	 * and free or cache any old rings from changed endpoints.
2812 	 */
2813 	for (i = 1; i < 31; ++i) {
2814 		if (!virt_dev->eps[i].new_ring)
2815 			continue;
2816 		/* Only cache or free the old ring if it exists.
2817 		 * It may not if this is the first add of an endpoint.
2818 		 */
2819 		if (virt_dev->eps[i].ring) {
2820 			xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
2821 		}
2822 		xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
2823 		virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
2824 		virt_dev->eps[i].new_ring = NULL;
2825 	}
2826 command_cleanup:
2827 	kfree(command->completion);
2828 	kfree(command);
2829 
2830 	return ret;
2831 }
2832 
2833 void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
2834 {
2835 	struct xhci_hcd *xhci;
2836 	struct xhci_virt_device	*virt_dev;
2837 	int i, ret;
2838 
2839 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
2840 	if (ret <= 0)
2841 		return;
2842 	xhci = hcd_to_xhci(hcd);
2843 
2844 	xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
2845 	virt_dev = xhci->devs[udev->slot_id];
2846 	/* Free any rings allocated for added endpoints */
2847 	for (i = 0; i < 31; ++i) {
2848 		if (virt_dev->eps[i].new_ring) {
2849 			xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
2850 			virt_dev->eps[i].new_ring = NULL;
2851 		}
2852 	}
2853 	xhci_zero_in_ctx(xhci, virt_dev);
2854 }
2855 
2856 static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
2857 		struct xhci_container_ctx *in_ctx,
2858 		struct xhci_container_ctx *out_ctx,
2859 		struct xhci_input_control_ctx *ctrl_ctx,
2860 		u32 add_flags, u32 drop_flags)
2861 {
2862 	ctrl_ctx->add_flags = cpu_to_le32(add_flags);
2863 	ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
2864 	xhci_slot_copy(xhci, in_ctx, out_ctx);
2865 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
2866 
2867 	xhci_dbg(xhci, "Input Context:\n");
2868 	xhci_dbg_ctx(xhci, in_ctx, xhci_last_valid_endpoint(add_flags));
2869 }
2870 
2871 static void xhci_setup_input_ctx_for_quirk(struct xhci_hcd *xhci,
2872 		unsigned int slot_id, unsigned int ep_index,
2873 		struct xhci_dequeue_state *deq_state)
2874 {
2875 	struct xhci_input_control_ctx *ctrl_ctx;
2876 	struct xhci_container_ctx *in_ctx;
2877 	struct xhci_ep_ctx *ep_ctx;
2878 	u32 added_ctxs;
2879 	dma_addr_t addr;
2880 
2881 	in_ctx = xhci->devs[slot_id]->in_ctx;
2882 	ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
2883 	if (!ctrl_ctx) {
2884 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
2885 				__func__);
2886 		return;
2887 	}
2888 
2889 	xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
2890 			xhci->devs[slot_id]->out_ctx, ep_index);
2891 	ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
2892 	addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
2893 			deq_state->new_deq_ptr);
2894 	if (addr == 0) {
2895 		xhci_warn(xhci, "WARN Cannot submit config ep after "
2896 				"reset ep command\n");
2897 		xhci_warn(xhci, "WARN deq seg = %p, deq ptr = %p\n",
2898 				deq_state->new_deq_seg,
2899 				deq_state->new_deq_ptr);
2900 		return;
2901 	}
2902 	ep_ctx->deq = cpu_to_le64(addr | deq_state->new_cycle_state);
2903 
2904 	added_ctxs = xhci_get_endpoint_flag_from_index(ep_index);
2905 	xhci_setup_input_ctx_for_config_ep(xhci, xhci->devs[slot_id]->in_ctx,
2906 			xhci->devs[slot_id]->out_ctx, ctrl_ctx,
2907 			added_ctxs, added_ctxs);
2908 }
2909 
2910 void xhci_cleanup_stalled_ring(struct xhci_hcd *xhci,
2911 			unsigned int ep_index, struct xhci_td *td)
2912 {
2913 	struct xhci_dequeue_state deq_state;
2914 	struct xhci_virt_ep *ep;
2915 	struct usb_device *udev = td->urb->dev;
2916 
2917 	xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
2918 			"Cleaning up stalled endpoint ring");
2919 	ep = &xhci->devs[udev->slot_id]->eps[ep_index];
2920 	/* We need to move the HW's dequeue pointer past this TD,
2921 	 * or it will attempt to resend it on the next doorbell ring.
2922 	 */
2923 	xhci_find_new_dequeue_state(xhci, udev->slot_id,
2924 			ep_index, ep->stopped_stream, td, &deq_state);
2925 
2926 	if (!deq_state.new_deq_ptr || !deq_state.new_deq_seg)
2927 		return;
2928 
2929 	/* HW with the reset endpoint quirk will use the saved dequeue state to
2930 	 * issue a configure endpoint command later.
2931 	 */
2932 	if (!(xhci->quirks & XHCI_RESET_EP_QUIRK)) {
2933 		xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
2934 				"Queueing new dequeue state");
2935 		xhci_queue_new_dequeue_state(xhci, udev->slot_id,
2936 				ep_index, ep->stopped_stream, &deq_state);
2937 	} else {
2938 		/* Better hope no one uses the input context between now and the
2939 		 * reset endpoint completion!
2940 		 * XXX: No idea how this hardware will react when stream rings
2941 		 * are enabled.
2942 		 */
2943 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
2944 				"Setting up input context for "
2945 				"configure endpoint command");
2946 		xhci_setup_input_ctx_for_quirk(xhci, udev->slot_id,
2947 				ep_index, &deq_state);
2948 	}
2949 }
2950 
2951 /* Called when clearing halted device. The core should have sent the control
2952  * message to clear the device halt condition. The host side of the halt should
2953  * already be cleared with a reset endpoint command issued when the STALL tx
2954  * event was received.
2955  *
2956  * Context: in_interrupt
2957  */
2958 
2959 void xhci_endpoint_reset(struct usb_hcd *hcd,
2960 		struct usb_host_endpoint *ep)
2961 {
2962 	struct xhci_hcd *xhci;
2963 
2964 	xhci = hcd_to_xhci(hcd);
2965 
2966 	/*
2967 	 * We might need to implement the config ep cmd in xhci 4.8.1 note:
2968 	 * The Reset Endpoint Command may only be issued to endpoints in the
2969 	 * Halted state. If software wishes reset the Data Toggle or Sequence
2970 	 * Number of an endpoint that isn't in the Halted state, then software
2971 	 * may issue a Configure Endpoint Command with the Drop and Add bits set
2972 	 * for the target endpoint. that is in the Stopped state.
2973 	 */
2974 
2975 	/* For now just print debug to follow the situation */
2976 	xhci_dbg(xhci, "Endpoint 0x%x ep reset callback called\n",
2977 		 ep->desc.bEndpointAddress);
2978 }
2979 
2980 static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
2981 		struct usb_device *udev, struct usb_host_endpoint *ep,
2982 		unsigned int slot_id)
2983 {
2984 	int ret;
2985 	unsigned int ep_index;
2986 	unsigned int ep_state;
2987 
2988 	if (!ep)
2989 		return -EINVAL;
2990 	ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
2991 	if (ret <= 0)
2992 		return -EINVAL;
2993 	if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {
2994 		xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
2995 				" descriptor for ep 0x%x does not support streams\n",
2996 				ep->desc.bEndpointAddress);
2997 		return -EINVAL;
2998 	}
2999 
3000 	ep_index = xhci_get_endpoint_index(&ep->desc);
3001 	ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3002 	if (ep_state & EP_HAS_STREAMS ||
3003 			ep_state & EP_GETTING_STREAMS) {
3004 		xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
3005 				"already has streams set up.\n",
3006 				ep->desc.bEndpointAddress);
3007 		xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
3008 				"dynamic stream context array reallocation.\n");
3009 		return -EINVAL;
3010 	}
3011 	if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
3012 		xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
3013 				"endpoint 0x%x; URBs are pending.\n",
3014 				ep->desc.bEndpointAddress);
3015 		return -EINVAL;
3016 	}
3017 	return 0;
3018 }
3019 
3020 static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
3021 		unsigned int *num_streams, unsigned int *num_stream_ctxs)
3022 {
3023 	unsigned int max_streams;
3024 
3025 	/* The stream context array size must be a power of two */
3026 	*num_stream_ctxs = roundup_pow_of_two(*num_streams);
3027 	/*
3028 	 * Find out how many primary stream array entries the host controller
3029 	 * supports.  Later we may use secondary stream arrays (similar to 2nd
3030 	 * level page entries), but that's an optional feature for xHCI host
3031 	 * controllers. xHCs must support at least 4 stream IDs.
3032 	 */
3033 	max_streams = HCC_MAX_PSA(xhci->hcc_params);
3034 	if (*num_stream_ctxs > max_streams) {
3035 		xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
3036 				max_streams);
3037 		*num_stream_ctxs = max_streams;
3038 		*num_streams = max_streams;
3039 	}
3040 }
3041 
3042 /* Returns an error code if one of the endpoint already has streams.
3043  * This does not change any data structures, it only checks and gathers
3044  * information.
3045  */
3046 static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
3047 		struct usb_device *udev,
3048 		struct usb_host_endpoint **eps, unsigned int num_eps,
3049 		unsigned int *num_streams, u32 *changed_ep_bitmask)
3050 {
3051 	unsigned int max_streams;
3052 	unsigned int endpoint_flag;
3053 	int i;
3054 	int ret;
3055 
3056 	for (i = 0; i < num_eps; i++) {
3057 		ret = xhci_check_streams_endpoint(xhci, udev,
3058 				eps[i], udev->slot_id);
3059 		if (ret < 0)
3060 			return ret;
3061 
3062 		max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);
3063 		if (max_streams < (*num_streams - 1)) {
3064 			xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
3065 					eps[i]->desc.bEndpointAddress,
3066 					max_streams);
3067 			*num_streams = max_streams+1;
3068 		}
3069 
3070 		endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
3071 		if (*changed_ep_bitmask & endpoint_flag)
3072 			return -EINVAL;
3073 		*changed_ep_bitmask |= endpoint_flag;
3074 	}
3075 	return 0;
3076 }
3077 
3078 static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
3079 		struct usb_device *udev,
3080 		struct usb_host_endpoint **eps, unsigned int num_eps)
3081 {
3082 	u32 changed_ep_bitmask = 0;
3083 	unsigned int slot_id;
3084 	unsigned int ep_index;
3085 	unsigned int ep_state;
3086 	int i;
3087 
3088 	slot_id = udev->slot_id;
3089 	if (!xhci->devs[slot_id])
3090 		return 0;
3091 
3092 	for (i = 0; i < num_eps; i++) {
3093 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3094 		ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
3095 		/* Are streams already being freed for the endpoint? */
3096 		if (ep_state & EP_GETTING_NO_STREAMS) {
3097 			xhci_warn(xhci, "WARN Can't disable streams for "
3098 					"endpoint 0x%x, "
3099 					"streams are being disabled already\n",
3100 					eps[i]->desc.bEndpointAddress);
3101 			return 0;
3102 		}
3103 		/* Are there actually any streams to free? */
3104 		if (!(ep_state & EP_HAS_STREAMS) &&
3105 				!(ep_state & EP_GETTING_STREAMS)) {
3106 			xhci_warn(xhci, "WARN Can't disable streams for "
3107 					"endpoint 0x%x, "
3108 					"streams are already disabled!\n",
3109 					eps[i]->desc.bEndpointAddress);
3110 			xhci_warn(xhci, "WARN xhci_free_streams() called "
3111 					"with non-streams endpoint\n");
3112 			return 0;
3113 		}
3114 		changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
3115 	}
3116 	return changed_ep_bitmask;
3117 }
3118 
3119 /*
3120  * The USB device drivers use this function (though the HCD interface in USB
3121  * core) to prepare a set of bulk endpoints to use streams.  Streams are used to
3122  * coordinate mass storage command queueing across multiple endpoints (basically
3123  * a stream ID == a task ID).
3124  *
3125  * Setting up streams involves allocating the same size stream context array
3126  * for each endpoint and issuing a configure endpoint command for all endpoints.
3127  *
3128  * Don't allow the call to succeed if one endpoint only supports one stream
3129  * (which means it doesn't support streams at all).
3130  *
3131  * Drivers may get less stream IDs than they asked for, if the host controller
3132  * hardware or endpoints claim they can't support the number of requested
3133  * stream IDs.
3134  */
3135 int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
3136 		struct usb_host_endpoint **eps, unsigned int num_eps,
3137 		unsigned int num_streams, gfp_t mem_flags)
3138 {
3139 	int i, ret;
3140 	struct xhci_hcd *xhci;
3141 	struct xhci_virt_device *vdev;
3142 	struct xhci_command *config_cmd;
3143 	struct xhci_input_control_ctx *ctrl_ctx;
3144 	unsigned int ep_index;
3145 	unsigned int num_stream_ctxs;
3146 	unsigned long flags;
3147 	u32 changed_ep_bitmask = 0;
3148 
3149 	if (!eps)
3150 		return -EINVAL;
3151 
3152 	/* Add one to the number of streams requested to account for
3153 	 * stream 0 that is reserved for xHCI usage.
3154 	 */
3155 	num_streams += 1;
3156 	xhci = hcd_to_xhci(hcd);
3157 	xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
3158 			num_streams);
3159 
3160 	/* MaxPSASize value 0 (2 streams) means streams are not supported */
3161 	if ((xhci->quirks & XHCI_BROKEN_STREAMS) ||
3162 			HCC_MAX_PSA(xhci->hcc_params) < 4) {
3163 		xhci_dbg(xhci, "xHCI controller does not support streams.\n");
3164 		return -ENOSYS;
3165 	}
3166 
3167 	config_cmd = xhci_alloc_command(xhci, true, true, mem_flags);
3168 	if (!config_cmd) {
3169 		xhci_dbg(xhci, "Could not allocate xHCI command structure.\n");
3170 		return -ENOMEM;
3171 	}
3172 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
3173 	if (!ctrl_ctx) {
3174 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3175 				__func__);
3176 		xhci_free_command(xhci, config_cmd);
3177 		return -ENOMEM;
3178 	}
3179 
3180 	/* Check to make sure all endpoints are not already configured for
3181 	 * streams.  While we're at it, find the maximum number of streams that
3182 	 * all the endpoints will support and check for duplicate endpoints.
3183 	 */
3184 	spin_lock_irqsave(&xhci->lock, flags);
3185 	ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
3186 			num_eps, &num_streams, &changed_ep_bitmask);
3187 	if (ret < 0) {
3188 		xhci_free_command(xhci, config_cmd);
3189 		spin_unlock_irqrestore(&xhci->lock, flags);
3190 		return ret;
3191 	}
3192 	if (num_streams <= 1) {
3193 		xhci_warn(xhci, "WARN: endpoints can't handle "
3194 				"more than one stream.\n");
3195 		xhci_free_command(xhci, config_cmd);
3196 		spin_unlock_irqrestore(&xhci->lock, flags);
3197 		return -EINVAL;
3198 	}
3199 	vdev = xhci->devs[udev->slot_id];
3200 	/* Mark each endpoint as being in transition, so
3201 	 * xhci_urb_enqueue() will reject all URBs.
3202 	 */
3203 	for (i = 0; i < num_eps; i++) {
3204 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3205 		vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
3206 	}
3207 	spin_unlock_irqrestore(&xhci->lock, flags);
3208 
3209 	/* Setup internal data structures and allocate HW data structures for
3210 	 * streams (but don't install the HW structures in the input context
3211 	 * until we're sure all memory allocation succeeded).
3212 	 */
3213 	xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
3214 	xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
3215 			num_stream_ctxs, num_streams);
3216 
3217 	for (i = 0; i < num_eps; i++) {
3218 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3219 		vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
3220 				num_stream_ctxs,
3221 				num_streams, mem_flags);
3222 		if (!vdev->eps[ep_index].stream_info)
3223 			goto cleanup;
3224 		/* Set maxPstreams in endpoint context and update deq ptr to
3225 		 * point to stream context array. FIXME
3226 		 */
3227 	}
3228 
3229 	/* Set up the input context for a configure endpoint command. */
3230 	for (i = 0; i < num_eps; i++) {
3231 		struct xhci_ep_ctx *ep_ctx;
3232 
3233 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3234 		ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);
3235 
3236 		xhci_endpoint_copy(xhci, config_cmd->in_ctx,
3237 				vdev->out_ctx, ep_index);
3238 		xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
3239 				vdev->eps[ep_index].stream_info);
3240 	}
3241 	/* Tell the HW to drop its old copy of the endpoint context info
3242 	 * and add the updated copy from the input context.
3243 	 */
3244 	xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
3245 			vdev->out_ctx, ctrl_ctx,
3246 			changed_ep_bitmask, changed_ep_bitmask);
3247 
3248 	/* Issue and wait for the configure endpoint command */
3249 	ret = xhci_configure_endpoint(xhci, udev, config_cmd,
3250 			false, false);
3251 
3252 	/* xHC rejected the configure endpoint command for some reason, so we
3253 	 * leave the old ring intact and free our internal streams data
3254 	 * structure.
3255 	 */
3256 	if (ret < 0)
3257 		goto cleanup;
3258 
3259 	spin_lock_irqsave(&xhci->lock, flags);
3260 	for (i = 0; i < num_eps; i++) {
3261 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3262 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3263 		xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
3264 			 udev->slot_id, ep_index);
3265 		vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
3266 	}
3267 	xhci_free_command(xhci, config_cmd);
3268 	spin_unlock_irqrestore(&xhci->lock, flags);
3269 
3270 	/* Subtract 1 for stream 0, which drivers can't use */
3271 	return num_streams - 1;
3272 
3273 cleanup:
3274 	/* If it didn't work, free the streams! */
3275 	for (i = 0; i < num_eps; i++) {
3276 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3277 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3278 		vdev->eps[ep_index].stream_info = NULL;
3279 		/* FIXME Unset maxPstreams in endpoint context and
3280 		 * update deq ptr to point to normal string ring.
3281 		 */
3282 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
3283 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3284 		xhci_endpoint_zero(xhci, vdev, eps[i]);
3285 	}
3286 	xhci_free_command(xhci, config_cmd);
3287 	return -ENOMEM;
3288 }
3289 
3290 /* Transition the endpoint from using streams to being a "normal" endpoint
3291  * without streams.
3292  *
3293  * Modify the endpoint context state, submit a configure endpoint command,
3294  * and free all endpoint rings for streams if that completes successfully.
3295  */
3296 int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
3297 		struct usb_host_endpoint **eps, unsigned int num_eps,
3298 		gfp_t mem_flags)
3299 {
3300 	int i, ret;
3301 	struct xhci_hcd *xhci;
3302 	struct xhci_virt_device *vdev;
3303 	struct xhci_command *command;
3304 	struct xhci_input_control_ctx *ctrl_ctx;
3305 	unsigned int ep_index;
3306 	unsigned long flags;
3307 	u32 changed_ep_bitmask;
3308 
3309 	xhci = hcd_to_xhci(hcd);
3310 	vdev = xhci->devs[udev->slot_id];
3311 
3312 	/* Set up a configure endpoint command to remove the streams rings */
3313 	spin_lock_irqsave(&xhci->lock, flags);
3314 	changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
3315 			udev, eps, num_eps);
3316 	if (changed_ep_bitmask == 0) {
3317 		spin_unlock_irqrestore(&xhci->lock, flags);
3318 		return -EINVAL;
3319 	}
3320 
3321 	/* Use the xhci_command structure from the first endpoint.  We may have
3322 	 * allocated too many, but the driver may call xhci_free_streams() for
3323 	 * each endpoint it grouped into one call to xhci_alloc_streams().
3324 	 */
3325 	ep_index = xhci_get_endpoint_index(&eps[0]->desc);
3326 	command = vdev->eps[ep_index].stream_info->free_streams_command;
3327 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
3328 	if (!ctrl_ctx) {
3329 		spin_unlock_irqrestore(&xhci->lock, flags);
3330 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3331 				__func__);
3332 		return -EINVAL;
3333 	}
3334 
3335 	for (i = 0; i < num_eps; i++) {
3336 		struct xhci_ep_ctx *ep_ctx;
3337 
3338 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3339 		ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
3340 		xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
3341 			EP_GETTING_NO_STREAMS;
3342 
3343 		xhci_endpoint_copy(xhci, command->in_ctx,
3344 				vdev->out_ctx, ep_index);
3345 		xhci_setup_no_streams_ep_input_ctx(ep_ctx,
3346 				&vdev->eps[ep_index]);
3347 	}
3348 	xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
3349 			vdev->out_ctx, ctrl_ctx,
3350 			changed_ep_bitmask, changed_ep_bitmask);
3351 	spin_unlock_irqrestore(&xhci->lock, flags);
3352 
3353 	/* Issue and wait for the configure endpoint command,
3354 	 * which must succeed.
3355 	 */
3356 	ret = xhci_configure_endpoint(xhci, udev, command,
3357 			false, true);
3358 
3359 	/* xHC rejected the configure endpoint command for some reason, so we
3360 	 * leave the streams rings intact.
3361 	 */
3362 	if (ret < 0)
3363 		return ret;
3364 
3365 	spin_lock_irqsave(&xhci->lock, flags);
3366 	for (i = 0; i < num_eps; i++) {
3367 		ep_index = xhci_get_endpoint_index(&eps[i]->desc);
3368 		xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
3369 		vdev->eps[ep_index].stream_info = NULL;
3370 		/* FIXME Unset maxPstreams in endpoint context and
3371 		 * update deq ptr to point to normal string ring.
3372 		 */
3373 		vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
3374 		vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
3375 	}
3376 	spin_unlock_irqrestore(&xhci->lock, flags);
3377 
3378 	return 0;
3379 }
3380 
3381 /*
3382  * Deletes endpoint resources for endpoints that were active before a Reset
3383  * Device command, or a Disable Slot command.  The Reset Device command leaves
3384  * the control endpoint intact, whereas the Disable Slot command deletes it.
3385  *
3386  * Must be called with xhci->lock held.
3387  */
3388 void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
3389 	struct xhci_virt_device *virt_dev, bool drop_control_ep)
3390 {
3391 	int i;
3392 	unsigned int num_dropped_eps = 0;
3393 	unsigned int drop_flags = 0;
3394 
3395 	for (i = (drop_control_ep ? 0 : 1); i < 31; i++) {
3396 		if (virt_dev->eps[i].ring) {
3397 			drop_flags |= 1 << i;
3398 			num_dropped_eps++;
3399 		}
3400 	}
3401 	xhci->num_active_eps -= num_dropped_eps;
3402 	if (num_dropped_eps)
3403 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3404 				"Dropped %u ep ctxs, flags = 0x%x, "
3405 				"%u now active.",
3406 				num_dropped_eps, drop_flags,
3407 				xhci->num_active_eps);
3408 }
3409 
3410 /*
3411  * This submits a Reset Device Command, which will set the device state to 0,
3412  * set the device address to 0, and disable all the endpoints except the default
3413  * control endpoint.  The USB core should come back and call
3414  * xhci_address_device(), and then re-set up the configuration.  If this is
3415  * called because of a usb_reset_and_verify_device(), then the old alternate
3416  * settings will be re-installed through the normal bandwidth allocation
3417  * functions.
3418  *
3419  * Wait for the Reset Device command to finish.  Remove all structures
3420  * associated with the endpoints that were disabled.  Clear the input device
3421  * structure?  Cache the rings?  Reset the control endpoint 0 max packet size?
3422  *
3423  * If the virt_dev to be reset does not exist or does not match the udev,
3424  * it means the device is lost, possibly due to the xHC restore error and
3425  * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
3426  * re-allocate the device.
3427  */
3428 int xhci_discover_or_reset_device(struct usb_hcd *hcd, struct usb_device *udev)
3429 {
3430 	int ret, i;
3431 	unsigned long flags;
3432 	struct xhci_hcd *xhci;
3433 	unsigned int slot_id;
3434 	struct xhci_virt_device *virt_dev;
3435 	struct xhci_command *reset_device_cmd;
3436 	int last_freed_endpoint;
3437 	struct xhci_slot_ctx *slot_ctx;
3438 	int old_active_eps = 0;
3439 
3440 	ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
3441 	if (ret <= 0)
3442 		return ret;
3443 	xhci = hcd_to_xhci(hcd);
3444 	slot_id = udev->slot_id;
3445 	virt_dev = xhci->devs[slot_id];
3446 	if (!virt_dev) {
3447 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3448 				"not exist. Re-allocate the device\n", slot_id);
3449 		ret = xhci_alloc_dev(hcd, udev);
3450 		if (ret == 1)
3451 			return 0;
3452 		else
3453 			return -EINVAL;
3454 	}
3455 
3456 	if (virt_dev->udev != udev) {
3457 		/* If the virt_dev and the udev does not match, this virt_dev
3458 		 * may belong to another udev.
3459 		 * Re-allocate the device.
3460 		 */
3461 		xhci_dbg(xhci, "The device to be reset with slot ID %u does "
3462 				"not match the udev. Re-allocate the device\n",
3463 				slot_id);
3464 		ret = xhci_alloc_dev(hcd, udev);
3465 		if (ret == 1)
3466 			return 0;
3467 		else
3468 			return -EINVAL;
3469 	}
3470 
3471 	/* If device is not setup, there is no point in resetting it */
3472 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3473 	if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
3474 						SLOT_STATE_DISABLED)
3475 		return 0;
3476 
3477 	xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
3478 	/* Allocate the command structure that holds the struct completion.
3479 	 * Assume we're in process context, since the normal device reset
3480 	 * process has to wait for the device anyway.  Storage devices are
3481 	 * reset as part of error handling, so use GFP_NOIO instead of
3482 	 * GFP_KERNEL.
3483 	 */
3484 	reset_device_cmd = xhci_alloc_command(xhci, false, true, GFP_NOIO);
3485 	if (!reset_device_cmd) {
3486 		xhci_dbg(xhci, "Couldn't allocate command structure.\n");
3487 		return -ENOMEM;
3488 	}
3489 
3490 	/* Attempt to submit the Reset Device command to the command ring */
3491 	spin_lock_irqsave(&xhci->lock, flags);
3492 
3493 	ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id);
3494 	if (ret) {
3495 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3496 		spin_unlock_irqrestore(&xhci->lock, flags);
3497 		goto command_cleanup;
3498 	}
3499 	xhci_ring_cmd_db(xhci);
3500 	spin_unlock_irqrestore(&xhci->lock, flags);
3501 
3502 	/* Wait for the Reset Device command to finish */
3503 	wait_for_completion(reset_device_cmd->completion);
3504 
3505 	/* The Reset Device command can't fail, according to the 0.95/0.96 spec,
3506 	 * unless we tried to reset a slot ID that wasn't enabled,
3507 	 * or the device wasn't in the addressed or configured state.
3508 	 */
3509 	ret = reset_device_cmd->status;
3510 	switch (ret) {
3511 	case COMP_CMD_ABORT:
3512 	case COMP_CMD_STOP:
3513 		xhci_warn(xhci, "Timeout waiting for reset device command\n");
3514 		ret = -ETIME;
3515 		goto command_cleanup;
3516 	case COMP_EBADSLT: /* 0.95 completion code for bad slot ID */
3517 	case COMP_CTX_STATE: /* 0.96 completion code for same thing */
3518 		xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n",
3519 				slot_id,
3520 				xhci_get_slot_state(xhci, virt_dev->out_ctx));
3521 		xhci_dbg(xhci, "Not freeing device rings.\n");
3522 		/* Don't treat this as an error.  May change my mind later. */
3523 		ret = 0;
3524 		goto command_cleanup;
3525 	case COMP_SUCCESS:
3526 		xhci_dbg(xhci, "Successful reset device command.\n");
3527 		break;
3528 	default:
3529 		if (xhci_is_vendor_info_code(xhci, ret))
3530 			break;
3531 		xhci_warn(xhci, "Unknown completion code %u for "
3532 				"reset device command.\n", ret);
3533 		ret = -EINVAL;
3534 		goto command_cleanup;
3535 	}
3536 
3537 	/* Free up host controller endpoint resources */
3538 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
3539 		spin_lock_irqsave(&xhci->lock, flags);
3540 		/* Don't delete the default control endpoint resources */
3541 		xhci_free_device_endpoint_resources(xhci, virt_dev, false);
3542 		spin_unlock_irqrestore(&xhci->lock, flags);
3543 	}
3544 
3545 	/* Everything but endpoint 0 is disabled, so free or cache the rings. */
3546 	last_freed_endpoint = 1;
3547 	for (i = 1; i < 31; ++i) {
3548 		struct xhci_virt_ep *ep = &virt_dev->eps[i];
3549 
3550 		if (ep->ep_state & EP_HAS_STREAMS) {
3551 			xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n",
3552 					xhci_get_endpoint_address(i));
3553 			xhci_free_stream_info(xhci, ep->stream_info);
3554 			ep->stream_info = NULL;
3555 			ep->ep_state &= ~EP_HAS_STREAMS;
3556 		}
3557 
3558 		if (ep->ring) {
3559 			xhci_free_or_cache_endpoint_ring(xhci, virt_dev, i);
3560 			last_freed_endpoint = i;
3561 		}
3562 		if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))
3563 			xhci_drop_ep_from_interval_table(xhci,
3564 					&virt_dev->eps[i].bw_info,
3565 					virt_dev->bw_table,
3566 					udev,
3567 					&virt_dev->eps[i],
3568 					virt_dev->tt_info);
3569 		xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);
3570 	}
3571 	/* If necessary, update the number of active TTs on this root port */
3572 	xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
3573 
3574 	xhci_dbg(xhci, "Output context after successful reset device cmd:\n");
3575 	xhci_dbg_ctx(xhci, virt_dev->out_ctx, last_freed_endpoint);
3576 	ret = 0;
3577 
3578 command_cleanup:
3579 	xhci_free_command(xhci, reset_device_cmd);
3580 	return ret;
3581 }
3582 
3583 /*
3584  * At this point, the struct usb_device is about to go away, the device has
3585  * disconnected, and all traffic has been stopped and the endpoints have been
3586  * disabled.  Free any HC data structures associated with that device.
3587  */
3588 void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
3589 {
3590 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3591 	struct xhci_virt_device *virt_dev;
3592 	unsigned long flags;
3593 	u32 state;
3594 	int i, ret;
3595 	struct xhci_command *command;
3596 
3597 	command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
3598 	if (!command)
3599 		return;
3600 
3601 #ifndef CONFIG_USB_DEFAULT_PERSIST
3602 	/*
3603 	 * We called pm_runtime_get_noresume when the device was attached.
3604 	 * Decrement the counter here to allow controller to runtime suspend
3605 	 * if no devices remain.
3606 	 */
3607 	if (xhci->quirks & XHCI_RESET_ON_RESUME)
3608 		pm_runtime_put_noidle(hcd->self.controller);
3609 #endif
3610 
3611 	ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
3612 	/* If the host is halted due to driver unload, we still need to free the
3613 	 * device.
3614 	 */
3615 	if (ret <= 0 && ret != -ENODEV) {
3616 		kfree(command);
3617 		return;
3618 	}
3619 
3620 	virt_dev = xhci->devs[udev->slot_id];
3621 
3622 	/* Stop any wayward timer functions (which may grab the lock) */
3623 	for (i = 0; i < 31; ++i) {
3624 		virt_dev->eps[i].ep_state &= ~EP_HALT_PENDING;
3625 		del_timer_sync(&virt_dev->eps[i].stop_cmd_timer);
3626 	}
3627 
3628 	spin_lock_irqsave(&xhci->lock, flags);
3629 	/* Don't disable the slot if the host controller is dead. */
3630 	state = readl(&xhci->op_regs->status);
3631 	if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
3632 			(xhci->xhc_state & XHCI_STATE_HALTED)) {
3633 		xhci_free_virt_device(xhci, udev->slot_id);
3634 		spin_unlock_irqrestore(&xhci->lock, flags);
3635 		kfree(command);
3636 		return;
3637 	}
3638 
3639 	if (xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
3640 				    udev->slot_id)) {
3641 		spin_unlock_irqrestore(&xhci->lock, flags);
3642 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3643 		return;
3644 	}
3645 	xhci_ring_cmd_db(xhci);
3646 	spin_unlock_irqrestore(&xhci->lock, flags);
3647 
3648 	/*
3649 	 * Event command completion handler will free any data structures
3650 	 * associated with the slot.  XXX Can free sleep?
3651 	 */
3652 }
3653 
3654 /*
3655  * Checks if we have enough host controller resources for the default control
3656  * endpoint.
3657  *
3658  * Must be called with xhci->lock held.
3659  */
3660 static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci)
3661 {
3662 	if (xhci->num_active_eps + 1 > xhci->limit_active_eps) {
3663 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3664 				"Not enough ep ctxs: "
3665 				"%u active, need to add 1, limit is %u.",
3666 				xhci->num_active_eps, xhci->limit_active_eps);
3667 		return -ENOMEM;
3668 	}
3669 	xhci->num_active_eps += 1;
3670 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
3671 			"Adding 1 ep ctx, %u now active.",
3672 			xhci->num_active_eps);
3673 	return 0;
3674 }
3675 
3676 
3677 /*
3678  * Returns 0 if the xHC ran out of device slots, the Enable Slot command
3679  * timed out, or allocating memory failed.  Returns 1 on success.
3680  */
3681 int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
3682 {
3683 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3684 	unsigned long flags;
3685 	int ret;
3686 	struct xhci_command *command;
3687 
3688 	command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
3689 	if (!command)
3690 		return 0;
3691 
3692 	spin_lock_irqsave(&xhci->lock, flags);
3693 	command->completion = &xhci->addr_dev;
3694 	ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);
3695 	if (ret) {
3696 		spin_unlock_irqrestore(&xhci->lock, flags);
3697 		xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
3698 		kfree(command);
3699 		return 0;
3700 	}
3701 	xhci_ring_cmd_db(xhci);
3702 	spin_unlock_irqrestore(&xhci->lock, flags);
3703 
3704 	wait_for_completion(command->completion);
3705 
3706 	if (!xhci->slot_id || command->status != COMP_SUCCESS) {
3707 		xhci_err(xhci, "Error while assigning device slot ID\n");
3708 		xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",
3709 				HCS_MAX_SLOTS(
3710 					readl(&xhci->cap_regs->hcs_params1)));
3711 		kfree(command);
3712 		return 0;
3713 	}
3714 
3715 	if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
3716 		spin_lock_irqsave(&xhci->lock, flags);
3717 		ret = xhci_reserve_host_control_ep_resources(xhci);
3718 		if (ret) {
3719 			spin_unlock_irqrestore(&xhci->lock, flags);
3720 			xhci_warn(xhci, "Not enough host resources, "
3721 					"active endpoint contexts = %u\n",
3722 					xhci->num_active_eps);
3723 			goto disable_slot;
3724 		}
3725 		spin_unlock_irqrestore(&xhci->lock, flags);
3726 	}
3727 	/* Use GFP_NOIO, since this function can be called from
3728 	 * xhci_discover_or_reset_device(), which may be called as part of
3729 	 * mass storage driver error handling.
3730 	 */
3731 	if (!xhci_alloc_virt_device(xhci, xhci->slot_id, udev, GFP_NOIO)) {
3732 		xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");
3733 		goto disable_slot;
3734 	}
3735 	udev->slot_id = xhci->slot_id;
3736 
3737 #ifndef CONFIG_USB_DEFAULT_PERSIST
3738 	/*
3739 	 * If resetting upon resume, we can't put the controller into runtime
3740 	 * suspend if there is a device attached.
3741 	 */
3742 	if (xhci->quirks & XHCI_RESET_ON_RESUME)
3743 		pm_runtime_get_noresume(hcd->self.controller);
3744 #endif
3745 
3746 
3747 	kfree(command);
3748 	/* Is this a LS or FS device under a HS hub? */
3749 	/* Hub or peripherial? */
3750 	return 1;
3751 
3752 disable_slot:
3753 	/* Disable slot, if we can do it without mem alloc */
3754 	spin_lock_irqsave(&xhci->lock, flags);
3755 	command->completion = NULL;
3756 	command->status = 0;
3757 	if (!xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
3758 				     udev->slot_id))
3759 		xhci_ring_cmd_db(xhci);
3760 	spin_unlock_irqrestore(&xhci->lock, flags);
3761 	return 0;
3762 }
3763 
3764 /*
3765  * Issue an Address Device command and optionally send a corresponding
3766  * SetAddress request to the device.
3767  * We should be protected by the usb_address0_mutex in hub_wq's hub_port_init,
3768  * so we should only issue and wait on one address command at the same time.
3769  */
3770 static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
3771 			     enum xhci_setup_dev setup)
3772 {
3773 	const char *act = setup == SETUP_CONTEXT_ONLY ? "context" : "address";
3774 	unsigned long flags;
3775 	struct xhci_virt_device *virt_dev;
3776 	int ret = 0;
3777 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3778 	struct xhci_slot_ctx *slot_ctx;
3779 	struct xhci_input_control_ctx *ctrl_ctx;
3780 	u64 temp_64;
3781 	struct xhci_command *command;
3782 
3783 	if (!udev->slot_id) {
3784 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3785 				"Bad Slot ID %d", udev->slot_id);
3786 		return -EINVAL;
3787 	}
3788 
3789 	virt_dev = xhci->devs[udev->slot_id];
3790 
3791 	if (WARN_ON(!virt_dev)) {
3792 		/*
3793 		 * In plug/unplug torture test with an NEC controller,
3794 		 * a zero-dereference was observed once due to virt_dev = 0.
3795 		 * Print useful debug rather than crash if it is observed again!
3796 		 */
3797 		xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n",
3798 			udev->slot_id);
3799 		return -EINVAL;
3800 	}
3801 
3802 	if (setup == SETUP_CONTEXT_ONLY) {
3803 		slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3804 		if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
3805 		    SLOT_STATE_DEFAULT) {
3806 			xhci_dbg(xhci, "Slot already in default state\n");
3807 			return 0;
3808 		}
3809 	}
3810 
3811 	command = xhci_alloc_command(xhci, false, false, GFP_KERNEL);
3812 	if (!command)
3813 		return -ENOMEM;
3814 
3815 	command->in_ctx = virt_dev->in_ctx;
3816 	command->completion = &xhci->addr_dev;
3817 
3818 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
3819 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
3820 	if (!ctrl_ctx) {
3821 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
3822 				__func__);
3823 		kfree(command);
3824 		return -EINVAL;
3825 	}
3826 	/*
3827 	 * If this is the first Set Address since device plug-in or
3828 	 * virt_device realloaction after a resume with an xHCI power loss,
3829 	 * then set up the slot context.
3830 	 */
3831 	if (!slot_ctx->dev_info)
3832 		xhci_setup_addressable_virt_dev(xhci, udev);
3833 	/* Otherwise, update the control endpoint ring enqueue pointer. */
3834 	else
3835 		xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
3836 	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
3837 	ctrl_ctx->drop_flags = 0;
3838 
3839 	xhci_dbg(xhci, "Slot ID %d Input Context:\n", udev->slot_id);
3840 	xhci_dbg_ctx(xhci, virt_dev->in_ctx, 2);
3841 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
3842 				le32_to_cpu(slot_ctx->dev_info) >> 27);
3843 
3844 	spin_lock_irqsave(&xhci->lock, flags);
3845 	ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
3846 					udev->slot_id, setup);
3847 	if (ret) {
3848 		spin_unlock_irqrestore(&xhci->lock, flags);
3849 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3850 				"FIXME: allocate a command ring segment");
3851 		kfree(command);
3852 		return ret;
3853 	}
3854 	xhci_ring_cmd_db(xhci);
3855 	spin_unlock_irqrestore(&xhci->lock, flags);
3856 
3857 	/* ctrl tx can take up to 5 sec; XXX: need more time for xHC? */
3858 	wait_for_completion(command->completion);
3859 
3860 	/* FIXME: From section 4.3.4: "Software shall be responsible for timing
3861 	 * the SetAddress() "recovery interval" required by USB and aborting the
3862 	 * command on a timeout.
3863 	 */
3864 	switch (command->status) {
3865 	case COMP_CMD_ABORT:
3866 	case COMP_CMD_STOP:
3867 		xhci_warn(xhci, "Timeout while waiting for setup device command\n");
3868 		ret = -ETIME;
3869 		break;
3870 	case COMP_CTX_STATE:
3871 	case COMP_EBADSLT:
3872 		xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n",
3873 			 act, udev->slot_id);
3874 		ret = -EINVAL;
3875 		break;
3876 	case COMP_TX_ERR:
3877 		dev_warn(&udev->dev, "Device not responding to setup %s.\n", act);
3878 		ret = -EPROTO;
3879 		break;
3880 	case COMP_DEV_ERR:
3881 		dev_warn(&udev->dev,
3882 			 "ERROR: Incompatible device for setup %s command\n", act);
3883 		ret = -ENODEV;
3884 		break;
3885 	case COMP_SUCCESS:
3886 		xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3887 			       "Successful setup %s command", act);
3888 		break;
3889 	default:
3890 		xhci_err(xhci,
3891 			 "ERROR: unexpected setup %s command completion code 0x%x.\n",
3892 			 act, command->status);
3893 		xhci_dbg(xhci, "Slot ID %d Output Context:\n", udev->slot_id);
3894 		xhci_dbg_ctx(xhci, virt_dev->out_ctx, 2);
3895 		trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1);
3896 		ret = -EINVAL;
3897 		break;
3898 	}
3899 	if (ret) {
3900 		kfree(command);
3901 		return ret;
3902 	}
3903 	temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
3904 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3905 			"Op regs DCBAA ptr = %#016llx", temp_64);
3906 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3907 		"Slot ID %d dcbaa entry @%p = %#016llx",
3908 		udev->slot_id,
3909 		&xhci->dcbaa->dev_context_ptrs[udev->slot_id],
3910 		(unsigned long long)
3911 		le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));
3912 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3913 			"Output Context DMA address = %#08llx",
3914 			(unsigned long long)virt_dev->out_ctx->dma);
3915 	xhci_dbg(xhci, "Slot ID %d Input Context:\n", udev->slot_id);
3916 	xhci_dbg_ctx(xhci, virt_dev->in_ctx, 2);
3917 	trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
3918 				le32_to_cpu(slot_ctx->dev_info) >> 27);
3919 	xhci_dbg(xhci, "Slot ID %d Output Context:\n", udev->slot_id);
3920 	xhci_dbg_ctx(xhci, virt_dev->out_ctx, 2);
3921 	/*
3922 	 * USB core uses address 1 for the roothubs, so we add one to the
3923 	 * address given back to us by the HC.
3924 	 */
3925 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
3926 	trace_xhci_address_ctx(xhci, virt_dev->out_ctx,
3927 				le32_to_cpu(slot_ctx->dev_info) >> 27);
3928 	/* Zero the input context control for later use */
3929 	ctrl_ctx->add_flags = 0;
3930 	ctrl_ctx->drop_flags = 0;
3931 
3932 	xhci_dbg_trace(xhci, trace_xhci_dbg_address,
3933 		       "Internal device address = %d",
3934 		       le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
3935 	kfree(command);
3936 	return 0;
3937 }
3938 
3939 int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev)
3940 {
3941 	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ADDRESS);
3942 }
3943 
3944 int xhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev)
3945 {
3946 	return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ONLY);
3947 }
3948 
3949 /*
3950  * Transfer the port index into real index in the HW port status
3951  * registers. Caculate offset between the port's PORTSC register
3952  * and port status base. Divide the number of per port register
3953  * to get the real index. The raw port number bases 1.
3954  */
3955 int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1)
3956 {
3957 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
3958 	__le32 __iomem *base_addr = &xhci->op_regs->port_status_base;
3959 	__le32 __iomem *addr;
3960 	int raw_port;
3961 
3962 	if (hcd->speed != HCD_USB3)
3963 		addr = xhci->usb2_ports[port1 - 1];
3964 	else
3965 		addr = xhci->usb3_ports[port1 - 1];
3966 
3967 	raw_port = (addr - base_addr)/NUM_PORT_REGS + 1;
3968 	return raw_port;
3969 }
3970 
3971 /*
3972  * Issue an Evaluate Context command to change the Maximum Exit Latency in the
3973  * slot context.  If that succeeds, store the new MEL in the xhci_virt_device.
3974  */
3975 static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,
3976 			struct usb_device *udev, u16 max_exit_latency)
3977 {
3978 	struct xhci_virt_device *virt_dev;
3979 	struct xhci_command *command;
3980 	struct xhci_input_control_ctx *ctrl_ctx;
3981 	struct xhci_slot_ctx *slot_ctx;
3982 	unsigned long flags;
3983 	int ret;
3984 
3985 	spin_lock_irqsave(&xhci->lock, flags);
3986 
3987 	virt_dev = xhci->devs[udev->slot_id];
3988 
3989 	/*
3990 	 * virt_dev might not exists yet if xHC resumed from hibernate (S4) and
3991 	 * xHC was re-initialized. Exit latency will be set later after
3992 	 * hub_port_finish_reset() is done and xhci->devs[] are re-allocated
3993 	 */
3994 
3995 	if (!virt_dev || max_exit_latency == virt_dev->current_mel) {
3996 		spin_unlock_irqrestore(&xhci->lock, flags);
3997 		return 0;
3998 	}
3999 
4000 	/* Attempt to issue an Evaluate Context command to change the MEL. */
4001 	command = xhci->lpm_command;
4002 	ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
4003 	if (!ctrl_ctx) {
4004 		spin_unlock_irqrestore(&xhci->lock, flags);
4005 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4006 				__func__);
4007 		return -ENOMEM;
4008 	}
4009 
4010 	xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx);
4011 	spin_unlock_irqrestore(&xhci->lock, flags);
4012 
4013 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
4014 	slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
4015 	slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));
4016 	slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);
4017 	slot_ctx->dev_state = 0;
4018 
4019 	xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
4020 			"Set up evaluate context for LPM MEL change.");
4021 	xhci_dbg(xhci, "Slot %u Input Context:\n", udev->slot_id);
4022 	xhci_dbg_ctx(xhci, command->in_ctx, 0);
4023 
4024 	/* Issue and wait for the evaluate context command. */
4025 	ret = xhci_configure_endpoint(xhci, udev, command,
4026 			true, true);
4027 	xhci_dbg(xhci, "Slot %u Output Context:\n", udev->slot_id);
4028 	xhci_dbg_ctx(xhci, virt_dev->out_ctx, 0);
4029 
4030 	if (!ret) {
4031 		spin_lock_irqsave(&xhci->lock, flags);
4032 		virt_dev->current_mel = max_exit_latency;
4033 		spin_unlock_irqrestore(&xhci->lock, flags);
4034 	}
4035 	return ret;
4036 }
4037 
4038 #ifdef CONFIG_PM
4039 
4040 /* BESL to HIRD Encoding array for USB2 LPM */
4041 static int xhci_besl_encoding[16] = {125, 150, 200, 300, 400, 500, 1000, 2000,
4042 	3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};
4043 
4044 /* Calculate HIRD/BESL for USB2 PORTPMSC*/
4045 static int xhci_calculate_hird_besl(struct xhci_hcd *xhci,
4046 					struct usb_device *udev)
4047 {
4048 	int u2del, besl, besl_host;
4049 	int besl_device = 0;
4050 	u32 field;
4051 
4052 	u2del = HCS_U2_LATENCY(xhci->hcs_params3);
4053 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4054 
4055 	if (field & USB_BESL_SUPPORT) {
4056 		for (besl_host = 0; besl_host < 16; besl_host++) {
4057 			if (xhci_besl_encoding[besl_host] >= u2del)
4058 				break;
4059 		}
4060 		/* Use baseline BESL value as default */
4061 		if (field & USB_BESL_BASELINE_VALID)
4062 			besl_device = USB_GET_BESL_BASELINE(field);
4063 		else if (field & USB_BESL_DEEP_VALID)
4064 			besl_device = USB_GET_BESL_DEEP(field);
4065 	} else {
4066 		if (u2del <= 50)
4067 			besl_host = 0;
4068 		else
4069 			besl_host = (u2del - 51) / 75 + 1;
4070 	}
4071 
4072 	besl = besl_host + besl_device;
4073 	if (besl > 15)
4074 		besl = 15;
4075 
4076 	return besl;
4077 }
4078 
4079 /* Calculate BESLD, L1 timeout and HIRDM for USB2 PORTHLPMC */
4080 static int xhci_calculate_usb2_hw_lpm_params(struct usb_device *udev)
4081 {
4082 	u32 field;
4083 	int l1;
4084 	int besld = 0;
4085 	int hirdm = 0;
4086 
4087 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4088 
4089 	/* xHCI l1 is set in steps of 256us, xHCI 1.0 section 5.4.11.2 */
4090 	l1 = udev->l1_params.timeout / 256;
4091 
4092 	/* device has preferred BESLD */
4093 	if (field & USB_BESL_DEEP_VALID) {
4094 		besld = USB_GET_BESL_DEEP(field);
4095 		hirdm = 1;
4096 	}
4097 
4098 	return PORT_BESLD(besld) | PORT_L1_TIMEOUT(l1) | PORT_HIRDM(hirdm);
4099 }
4100 
4101 int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
4102 			struct usb_device *udev, int enable)
4103 {
4104 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
4105 	__le32 __iomem	**port_array;
4106 	__le32 __iomem	*pm_addr, *hlpm_addr;
4107 	u32		pm_val, hlpm_val, field;
4108 	unsigned int	port_num;
4109 	unsigned long	flags;
4110 	int		hird, exit_latency;
4111 	int		ret;
4112 
4113 	if (hcd->speed == HCD_USB3 || !xhci->hw_lpm_support ||
4114 			!udev->lpm_capable)
4115 		return -EPERM;
4116 
4117 	if (!udev->parent || udev->parent->parent ||
4118 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4119 		return -EPERM;
4120 
4121 	if (udev->usb2_hw_lpm_capable != 1)
4122 		return -EPERM;
4123 
4124 	spin_lock_irqsave(&xhci->lock, flags);
4125 
4126 	port_array = xhci->usb2_ports;
4127 	port_num = udev->portnum - 1;
4128 	pm_addr = port_array[port_num] + PORTPMSC;
4129 	pm_val = readl(pm_addr);
4130 	hlpm_addr = port_array[port_num] + PORTHLPMC;
4131 	field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
4132 
4133 	xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n",
4134 			enable ? "enable" : "disable", port_num + 1);
4135 
4136 	if (enable) {
4137 		/* Host supports BESL timeout instead of HIRD */
4138 		if (udev->usb2_hw_lpm_besl_capable) {
4139 			/* if device doesn't have a preferred BESL value use a
4140 			 * default one which works with mixed HIRD and BESL
4141 			 * systems. See XHCI_DEFAULT_BESL definition in xhci.h
4142 			 */
4143 			if ((field & USB_BESL_SUPPORT) &&
4144 			    (field & USB_BESL_BASELINE_VALID))
4145 				hird = USB_GET_BESL_BASELINE(field);
4146 			else
4147 				hird = udev->l1_params.besl;
4148 
4149 			exit_latency = xhci_besl_encoding[hird];
4150 			spin_unlock_irqrestore(&xhci->lock, flags);
4151 
4152 			/* USB 3.0 code dedicate one xhci->lpm_command->in_ctx
4153 			 * input context for link powermanagement evaluate
4154 			 * context commands. It is protected by hcd->bandwidth
4155 			 * mutex and is shared by all devices. We need to set
4156 			 * the max ext latency in USB 2 BESL LPM as well, so
4157 			 * use the same mutex and xhci_change_max_exit_latency()
4158 			 */
4159 			mutex_lock(hcd->bandwidth_mutex);
4160 			ret = xhci_change_max_exit_latency(xhci, udev,
4161 							   exit_latency);
4162 			mutex_unlock(hcd->bandwidth_mutex);
4163 
4164 			if (ret < 0)
4165 				return ret;
4166 			spin_lock_irqsave(&xhci->lock, flags);
4167 
4168 			hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev);
4169 			writel(hlpm_val, hlpm_addr);
4170 			/* flush write */
4171 			readl(hlpm_addr);
4172 		} else {
4173 			hird = xhci_calculate_hird_besl(xhci, udev);
4174 		}
4175 
4176 		pm_val &= ~PORT_HIRD_MASK;
4177 		pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id);
4178 		writel(pm_val, pm_addr);
4179 		pm_val = readl(pm_addr);
4180 		pm_val |= PORT_HLE;
4181 		writel(pm_val, pm_addr);
4182 		/* flush write */
4183 		readl(pm_addr);
4184 	} else {
4185 		pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK);
4186 		writel(pm_val, pm_addr);
4187 		/* flush write */
4188 		readl(pm_addr);
4189 		if (udev->usb2_hw_lpm_besl_capable) {
4190 			spin_unlock_irqrestore(&xhci->lock, flags);
4191 			mutex_lock(hcd->bandwidth_mutex);
4192 			xhci_change_max_exit_latency(xhci, udev, 0);
4193 			mutex_unlock(hcd->bandwidth_mutex);
4194 			return 0;
4195 		}
4196 	}
4197 
4198 	spin_unlock_irqrestore(&xhci->lock, flags);
4199 	return 0;
4200 }
4201 
4202 /* check if a usb2 port supports a given extened capability protocol
4203  * only USB2 ports extended protocol capability values are cached.
4204  * Return 1 if capability is supported
4205  */
4206 static int xhci_check_usb2_port_capability(struct xhci_hcd *xhci, int port,
4207 					   unsigned capability)
4208 {
4209 	u32 port_offset, port_count;
4210 	int i;
4211 
4212 	for (i = 0; i < xhci->num_ext_caps; i++) {
4213 		if (xhci->ext_caps[i] & capability) {
4214 			/* port offsets starts at 1 */
4215 			port_offset = XHCI_EXT_PORT_OFF(xhci->ext_caps[i]) - 1;
4216 			port_count = XHCI_EXT_PORT_COUNT(xhci->ext_caps[i]);
4217 			if (port >= port_offset &&
4218 			    port < port_offset + port_count)
4219 				return 1;
4220 		}
4221 	}
4222 	return 0;
4223 }
4224 
4225 int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
4226 {
4227 	struct xhci_hcd	*xhci = hcd_to_xhci(hcd);
4228 	int		portnum = udev->portnum - 1;
4229 
4230 	if (hcd->speed == HCD_USB3 || !xhci->sw_lpm_support ||
4231 			!udev->lpm_capable)
4232 		return 0;
4233 
4234 	/* we only support lpm for non-hub device connected to root hub yet */
4235 	if (!udev->parent || udev->parent->parent ||
4236 			udev->descriptor.bDeviceClass == USB_CLASS_HUB)
4237 		return 0;
4238 
4239 	if (xhci->hw_lpm_support == 1 &&
4240 			xhci_check_usb2_port_capability(
4241 				xhci, portnum, XHCI_HLC)) {
4242 		udev->usb2_hw_lpm_capable = 1;
4243 		udev->l1_params.timeout = XHCI_L1_TIMEOUT;
4244 		udev->l1_params.besl = XHCI_DEFAULT_BESL;
4245 		if (xhci_check_usb2_port_capability(xhci, portnum,
4246 					XHCI_BLC))
4247 			udev->usb2_hw_lpm_besl_capable = 1;
4248 	}
4249 
4250 	return 0;
4251 }
4252 
4253 /*---------------------- USB 3.0 Link PM functions ------------------------*/
4254 
4255 /* Service interval in nanoseconds = 2^(bInterval - 1) * 125us * 1000ns / 1us */
4256 static unsigned long long xhci_service_interval_to_ns(
4257 		struct usb_endpoint_descriptor *desc)
4258 {
4259 	return (1ULL << (desc->bInterval - 1)) * 125 * 1000;
4260 }
4261 
4262 static u16 xhci_get_timeout_no_hub_lpm(struct usb_device *udev,
4263 		enum usb3_link_state state)
4264 {
4265 	unsigned long long sel;
4266 	unsigned long long pel;
4267 	unsigned int max_sel_pel;
4268 	char *state_name;
4269 
4270 	switch (state) {
4271 	case USB3_LPM_U1:
4272 		/* Convert SEL and PEL stored in nanoseconds to microseconds */
4273 		sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
4274 		pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);
4275 		max_sel_pel = USB3_LPM_MAX_U1_SEL_PEL;
4276 		state_name = "U1";
4277 		break;
4278 	case USB3_LPM_U2:
4279 		sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
4280 		pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);
4281 		max_sel_pel = USB3_LPM_MAX_U2_SEL_PEL;
4282 		state_name = "U2";
4283 		break;
4284 	default:
4285 		dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n",
4286 				__func__);
4287 		return USB3_LPM_DISABLED;
4288 	}
4289 
4290 	if (sel <= max_sel_pel && pel <= max_sel_pel)
4291 		return USB3_LPM_DEVICE_INITIATED;
4292 
4293 	if (sel > max_sel_pel)
4294 		dev_dbg(&udev->dev, "Device-initiated %s disabled "
4295 				"due to long SEL %llu ms\n",
4296 				state_name, sel);
4297 	else
4298 		dev_dbg(&udev->dev, "Device-initiated %s disabled "
4299 				"due to long PEL %llu ms\n",
4300 				state_name, pel);
4301 	return USB3_LPM_DISABLED;
4302 }
4303 
4304 /* The U1 timeout should be the maximum of the following values:
4305  *  - For control endpoints, U1 system exit latency (SEL) * 3
4306  *  - For bulk endpoints, U1 SEL * 5
4307  *  - For interrupt endpoints:
4308  *    - Notification EPs, U1 SEL * 3
4309  *    - Periodic EPs, max(105% of bInterval, U1 SEL * 2)
4310  *  - For isochronous endpoints, max(105% of bInterval, U1 SEL * 2)
4311  */
4312 static unsigned long long xhci_calculate_intel_u1_timeout(
4313 		struct usb_device *udev,
4314 		struct usb_endpoint_descriptor *desc)
4315 {
4316 	unsigned long long timeout_ns;
4317 	int ep_type;
4318 	int intr_type;
4319 
4320 	ep_type = usb_endpoint_type(desc);
4321 	switch (ep_type) {
4322 	case USB_ENDPOINT_XFER_CONTROL:
4323 		timeout_ns = udev->u1_params.sel * 3;
4324 		break;
4325 	case USB_ENDPOINT_XFER_BULK:
4326 		timeout_ns = udev->u1_params.sel * 5;
4327 		break;
4328 	case USB_ENDPOINT_XFER_INT:
4329 		intr_type = usb_endpoint_interrupt_type(desc);
4330 		if (intr_type == USB_ENDPOINT_INTR_NOTIFICATION) {
4331 			timeout_ns = udev->u1_params.sel * 3;
4332 			break;
4333 		}
4334 		/* Otherwise the calculation is the same as isoc eps */
4335 	case USB_ENDPOINT_XFER_ISOC:
4336 		timeout_ns = xhci_service_interval_to_ns(desc);
4337 		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns * 105, 100);
4338 		if (timeout_ns < udev->u1_params.sel * 2)
4339 			timeout_ns = udev->u1_params.sel * 2;
4340 		break;
4341 	default:
4342 		return 0;
4343 	}
4344 
4345 	return timeout_ns;
4346 }
4347 
4348 /* Returns the hub-encoded U1 timeout value. */
4349 static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,
4350 		struct usb_device *udev,
4351 		struct usb_endpoint_descriptor *desc)
4352 {
4353 	unsigned long long timeout_ns;
4354 
4355 	if (xhci->quirks & XHCI_INTEL_HOST)
4356 		timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc);
4357 	else
4358 		timeout_ns = udev->u1_params.sel;
4359 
4360 	/* The U1 timeout is encoded in 1us intervals.
4361 	 * Don't return a timeout of zero, because that's USB3_LPM_DISABLED.
4362 	 */
4363 	if (timeout_ns == USB3_LPM_DISABLED)
4364 		timeout_ns = 1;
4365 	else
4366 		timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 1000);
4367 
4368 	/* If the necessary timeout value is bigger than what we can set in the
4369 	 * USB 3.0 hub, we have to disable hub-initiated U1.
4370 	 */
4371 	if (timeout_ns <= USB3_LPM_U1_MAX_TIMEOUT)
4372 		return timeout_ns;
4373 	dev_dbg(&udev->dev, "Hub-initiated U1 disabled "
4374 			"due to long timeout %llu ms\n", timeout_ns);
4375 	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U1);
4376 }
4377 
4378 /* The U2 timeout should be the maximum of:
4379  *  - 10 ms (to avoid the bandwidth impact on the scheduler)
4380  *  - largest bInterval of any active periodic endpoint (to avoid going
4381  *    into lower power link states between intervals).
4382  *  - the U2 Exit Latency of the device
4383  */
4384 static unsigned long long xhci_calculate_intel_u2_timeout(
4385 		struct usb_device *udev,
4386 		struct usb_endpoint_descriptor *desc)
4387 {
4388 	unsigned long long timeout_ns;
4389 	unsigned long long u2_del_ns;
4390 
4391 	timeout_ns = 10 * 1000 * 1000;
4392 
4393 	if ((usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) &&
4394 			(xhci_service_interval_to_ns(desc) > timeout_ns))
4395 		timeout_ns = xhci_service_interval_to_ns(desc);
4396 
4397 	u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL;
4398 	if (u2_del_ns > timeout_ns)
4399 		timeout_ns = u2_del_ns;
4400 
4401 	return timeout_ns;
4402 }
4403 
4404 /* Returns the hub-encoded U2 timeout value. */
4405 static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,
4406 		struct usb_device *udev,
4407 		struct usb_endpoint_descriptor *desc)
4408 {
4409 	unsigned long long timeout_ns;
4410 
4411 	if (xhci->quirks & XHCI_INTEL_HOST)
4412 		timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc);
4413 	else
4414 		timeout_ns = udev->u2_params.sel;
4415 
4416 	/* The U2 timeout is encoded in 256us intervals */
4417 	timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000);
4418 	/* If the necessary timeout value is bigger than what we can set in the
4419 	 * USB 3.0 hub, we have to disable hub-initiated U2.
4420 	 */
4421 	if (timeout_ns <= USB3_LPM_U2_MAX_TIMEOUT)
4422 		return timeout_ns;
4423 	dev_dbg(&udev->dev, "Hub-initiated U2 disabled "
4424 			"due to long timeout %llu ms\n", timeout_ns);
4425 	return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U2);
4426 }
4427 
4428 static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4429 		struct usb_device *udev,
4430 		struct usb_endpoint_descriptor *desc,
4431 		enum usb3_link_state state,
4432 		u16 *timeout)
4433 {
4434 	if (state == USB3_LPM_U1)
4435 		return xhci_calculate_u1_timeout(xhci, udev, desc);
4436 	else if (state == USB3_LPM_U2)
4437 		return xhci_calculate_u2_timeout(xhci, udev, desc);
4438 
4439 	return USB3_LPM_DISABLED;
4440 }
4441 
4442 static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci,
4443 		struct usb_device *udev,
4444 		struct usb_endpoint_descriptor *desc,
4445 		enum usb3_link_state state,
4446 		u16 *timeout)
4447 {
4448 	u16 alt_timeout;
4449 
4450 	alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev,
4451 		desc, state, timeout);
4452 
4453 	/* If we found we can't enable hub-initiated LPM, or
4454 	 * the U1 or U2 exit latency was too high to allow
4455 	 * device-initiated LPM as well, just stop searching.
4456 	 */
4457 	if (alt_timeout == USB3_LPM_DISABLED ||
4458 			alt_timeout == USB3_LPM_DEVICE_INITIATED) {
4459 		*timeout = alt_timeout;
4460 		return -E2BIG;
4461 	}
4462 	if (alt_timeout > *timeout)
4463 		*timeout = alt_timeout;
4464 	return 0;
4465 }
4466 
4467 static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci,
4468 		struct usb_device *udev,
4469 		struct usb_host_interface *alt,
4470 		enum usb3_link_state state,
4471 		u16 *timeout)
4472 {
4473 	int j;
4474 
4475 	for (j = 0; j < alt->desc.bNumEndpoints; j++) {
4476 		if (xhci_update_timeout_for_endpoint(xhci, udev,
4477 					&alt->endpoint[j].desc, state, timeout))
4478 			return -E2BIG;
4479 		continue;
4480 	}
4481 	return 0;
4482 }
4483 
4484 static int xhci_check_intel_tier_policy(struct usb_device *udev,
4485 		enum usb3_link_state state)
4486 {
4487 	struct usb_device *parent;
4488 	unsigned int num_hubs;
4489 
4490 	if (state == USB3_LPM_U2)
4491 		return 0;
4492 
4493 	/* Don't enable U1 if the device is on a 2nd tier hub or lower. */
4494 	for (parent = udev->parent, num_hubs = 0; parent->parent;
4495 			parent = parent->parent)
4496 		num_hubs++;
4497 
4498 	if (num_hubs < 2)
4499 		return 0;
4500 
4501 	dev_dbg(&udev->dev, "Disabling U1 link state for device"
4502 			" below second-tier hub.\n");
4503 	dev_dbg(&udev->dev, "Plug device into first-tier hub "
4504 			"to decrease power consumption.\n");
4505 	return -E2BIG;
4506 }
4507 
4508 static int xhci_check_tier_policy(struct xhci_hcd *xhci,
4509 		struct usb_device *udev,
4510 		enum usb3_link_state state)
4511 {
4512 	if (xhci->quirks & XHCI_INTEL_HOST)
4513 		return xhci_check_intel_tier_policy(udev, state);
4514 	else
4515 		return 0;
4516 }
4517 
4518 /* Returns the U1 or U2 timeout that should be enabled.
4519  * If the tier check or timeout setting functions return with a non-zero exit
4520  * code, that means the timeout value has been finalized and we shouldn't look
4521  * at any more endpoints.
4522  */
4523 static u16 xhci_calculate_lpm_timeout(struct usb_hcd *hcd,
4524 			struct usb_device *udev, enum usb3_link_state state)
4525 {
4526 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4527 	struct usb_host_config *config;
4528 	char *state_name;
4529 	int i;
4530 	u16 timeout = USB3_LPM_DISABLED;
4531 
4532 	if (state == USB3_LPM_U1)
4533 		state_name = "U1";
4534 	else if (state == USB3_LPM_U2)
4535 		state_name = "U2";
4536 	else {
4537 		dev_warn(&udev->dev, "Can't enable unknown link state %i\n",
4538 				state);
4539 		return timeout;
4540 	}
4541 
4542 	if (xhci_check_tier_policy(xhci, udev, state) < 0)
4543 		return timeout;
4544 
4545 	/* Gather some information about the currently installed configuration
4546 	 * and alternate interface settings.
4547 	 */
4548 	if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc,
4549 			state, &timeout))
4550 		return timeout;
4551 
4552 	config = udev->actconfig;
4553 	if (!config)
4554 		return timeout;
4555 
4556 	for (i = 0; i < config->desc.bNumInterfaces; i++) {
4557 		struct usb_driver *driver;
4558 		struct usb_interface *intf = config->interface[i];
4559 
4560 		if (!intf)
4561 			continue;
4562 
4563 		/* Check if any currently bound drivers want hub-initiated LPM
4564 		 * disabled.
4565 		 */
4566 		if (intf->dev.driver) {
4567 			driver = to_usb_driver(intf->dev.driver);
4568 			if (driver && driver->disable_hub_initiated_lpm) {
4569 				dev_dbg(&udev->dev, "Hub-initiated %s disabled "
4570 						"at request of driver %s\n",
4571 						state_name, driver->name);
4572 				return xhci_get_timeout_no_hub_lpm(udev, state);
4573 			}
4574 		}
4575 
4576 		/* Not sure how this could happen... */
4577 		if (!intf->cur_altsetting)
4578 			continue;
4579 
4580 		if (xhci_update_timeout_for_interface(xhci, udev,
4581 					intf->cur_altsetting,
4582 					state, &timeout))
4583 			return timeout;
4584 	}
4585 	return timeout;
4586 }
4587 
4588 static int calculate_max_exit_latency(struct usb_device *udev,
4589 		enum usb3_link_state state_changed,
4590 		u16 hub_encoded_timeout)
4591 {
4592 	unsigned long long u1_mel_us = 0;
4593 	unsigned long long u2_mel_us = 0;
4594 	unsigned long long mel_us = 0;
4595 	bool disabling_u1;
4596 	bool disabling_u2;
4597 	bool enabling_u1;
4598 	bool enabling_u2;
4599 
4600 	disabling_u1 = (state_changed == USB3_LPM_U1 &&
4601 			hub_encoded_timeout == USB3_LPM_DISABLED);
4602 	disabling_u2 = (state_changed == USB3_LPM_U2 &&
4603 			hub_encoded_timeout == USB3_LPM_DISABLED);
4604 
4605 	enabling_u1 = (state_changed == USB3_LPM_U1 &&
4606 			hub_encoded_timeout != USB3_LPM_DISABLED);
4607 	enabling_u2 = (state_changed == USB3_LPM_U2 &&
4608 			hub_encoded_timeout != USB3_LPM_DISABLED);
4609 
4610 	/* If U1 was already enabled and we're not disabling it,
4611 	 * or we're going to enable U1, account for the U1 max exit latency.
4612 	 */
4613 	if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) ||
4614 			enabling_u1)
4615 		u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000);
4616 	if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) ||
4617 			enabling_u2)
4618 		u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000);
4619 
4620 	if (u1_mel_us > u2_mel_us)
4621 		mel_us = u1_mel_us;
4622 	else
4623 		mel_us = u2_mel_us;
4624 	/* xHCI host controller max exit latency field is only 16 bits wide. */
4625 	if (mel_us > MAX_EXIT) {
4626 		dev_warn(&udev->dev, "Link PM max exit latency of %lluus "
4627 				"is too big.\n", mel_us);
4628 		return -E2BIG;
4629 	}
4630 	return mel_us;
4631 }
4632 
4633 /* Returns the USB3 hub-encoded value for the U1/U2 timeout. */
4634 int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
4635 			struct usb_device *udev, enum usb3_link_state state)
4636 {
4637 	struct xhci_hcd	*xhci;
4638 	u16 hub_encoded_timeout;
4639 	int mel;
4640 	int ret;
4641 
4642 	xhci = hcd_to_xhci(hcd);
4643 	/* The LPM timeout values are pretty host-controller specific, so don't
4644 	 * enable hub-initiated timeouts unless the vendor has provided
4645 	 * information about their timeout algorithm.
4646 	 */
4647 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
4648 			!xhci->devs[udev->slot_id])
4649 		return USB3_LPM_DISABLED;
4650 
4651 	hub_encoded_timeout = xhci_calculate_lpm_timeout(hcd, udev, state);
4652 	mel = calculate_max_exit_latency(udev, state, hub_encoded_timeout);
4653 	if (mel < 0) {
4654 		/* Max Exit Latency is too big, disable LPM. */
4655 		hub_encoded_timeout = USB3_LPM_DISABLED;
4656 		mel = 0;
4657 	}
4658 
4659 	ret = xhci_change_max_exit_latency(xhci, udev, mel);
4660 	if (ret)
4661 		return ret;
4662 	return hub_encoded_timeout;
4663 }
4664 
4665 int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
4666 			struct usb_device *udev, enum usb3_link_state state)
4667 {
4668 	struct xhci_hcd	*xhci;
4669 	u16 mel;
4670 	int ret;
4671 
4672 	xhci = hcd_to_xhci(hcd);
4673 	if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
4674 			!xhci->devs[udev->slot_id])
4675 		return 0;
4676 
4677 	mel = calculate_max_exit_latency(udev, state, USB3_LPM_DISABLED);
4678 	ret = xhci_change_max_exit_latency(xhci, udev, mel);
4679 	if (ret)
4680 		return ret;
4681 	return 0;
4682 }
4683 #else /* CONFIG_PM */
4684 
4685 int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
4686 				struct usb_device *udev, int enable)
4687 {
4688 	return 0;
4689 }
4690 
4691 int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
4692 {
4693 	return 0;
4694 }
4695 
4696 int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
4697 			struct usb_device *udev, enum usb3_link_state state)
4698 {
4699 	return USB3_LPM_DISABLED;
4700 }
4701 
4702 int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
4703 			struct usb_device *udev, enum usb3_link_state state)
4704 {
4705 	return 0;
4706 }
4707 #endif	/* CONFIG_PM */
4708 
4709 /*-------------------------------------------------------------------------*/
4710 
4711 /* Once a hub descriptor is fetched for a device, we need to update the xHC's
4712  * internal data structures for the device.
4713  */
4714 int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
4715 			struct usb_tt *tt, gfp_t mem_flags)
4716 {
4717 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4718 	struct xhci_virt_device *vdev;
4719 	struct xhci_command *config_cmd;
4720 	struct xhci_input_control_ctx *ctrl_ctx;
4721 	struct xhci_slot_ctx *slot_ctx;
4722 	unsigned long flags;
4723 	unsigned think_time;
4724 	int ret;
4725 
4726 	/* Ignore root hubs */
4727 	if (!hdev->parent)
4728 		return 0;
4729 
4730 	vdev = xhci->devs[hdev->slot_id];
4731 	if (!vdev) {
4732 		xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");
4733 		return -EINVAL;
4734 	}
4735 	config_cmd = xhci_alloc_command(xhci, true, true, mem_flags);
4736 	if (!config_cmd) {
4737 		xhci_dbg(xhci, "Could not allocate xHCI command structure.\n");
4738 		return -ENOMEM;
4739 	}
4740 	ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
4741 	if (!ctrl_ctx) {
4742 		xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
4743 				__func__);
4744 		xhci_free_command(xhci, config_cmd);
4745 		return -ENOMEM;
4746 	}
4747 
4748 	spin_lock_irqsave(&xhci->lock, flags);
4749 	if (hdev->speed == USB_SPEED_HIGH &&
4750 			xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) {
4751 		xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n");
4752 		xhci_free_command(xhci, config_cmd);
4753 		spin_unlock_irqrestore(&xhci->lock, flags);
4754 		return -ENOMEM;
4755 	}
4756 
4757 	xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
4758 	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
4759 	slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
4760 	slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);
4761 	if (tt->multi)
4762 		slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
4763 	if (xhci->hci_version > 0x95) {
4764 		xhci_dbg(xhci, "xHCI version %x needs hub "
4765 				"TT think time and number of ports\n",
4766 				(unsigned int) xhci->hci_version);
4767 		slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild));
4768 		/* Set TT think time - convert from ns to FS bit times.
4769 		 * 0 = 8 FS bit times, 1 = 16 FS bit times,
4770 		 * 2 = 24 FS bit times, 3 = 32 FS bit times.
4771 		 *
4772 		 * xHCI 1.0: this field shall be 0 if the device is not a
4773 		 * High-spped hub.
4774 		 */
4775 		think_time = tt->think_time;
4776 		if (think_time != 0)
4777 			think_time = (think_time / 666) - 1;
4778 		if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH)
4779 			slot_ctx->tt_info |=
4780 				cpu_to_le32(TT_THINK_TIME(think_time));
4781 	} else {
4782 		xhci_dbg(xhci, "xHCI version %x doesn't need hub "
4783 				"TT think time or number of ports\n",
4784 				(unsigned int) xhci->hci_version);
4785 	}
4786 	slot_ctx->dev_state = 0;
4787 	spin_unlock_irqrestore(&xhci->lock, flags);
4788 
4789 	xhci_dbg(xhci, "Set up %s for hub device.\n",
4790 			(xhci->hci_version > 0x95) ?
4791 			"configure endpoint" : "evaluate context");
4792 	xhci_dbg(xhci, "Slot %u Input Context:\n", hdev->slot_id);
4793 	xhci_dbg_ctx(xhci, config_cmd->in_ctx, 0);
4794 
4795 	/* Issue and wait for the configure endpoint or
4796 	 * evaluate context command.
4797 	 */
4798 	if (xhci->hci_version > 0x95)
4799 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
4800 				false, false);
4801 	else
4802 		ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
4803 				true, false);
4804 
4805 	xhci_dbg(xhci, "Slot %u Output Context:\n", hdev->slot_id);
4806 	xhci_dbg_ctx(xhci, vdev->out_ctx, 0);
4807 
4808 	xhci_free_command(xhci, config_cmd);
4809 	return ret;
4810 }
4811 
4812 int xhci_get_frame(struct usb_hcd *hcd)
4813 {
4814 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
4815 	/* EHCI mods by the periodic size.  Why? */
4816 	return readl(&xhci->run_regs->microframe_index) >> 3;
4817 }
4818 
4819 int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
4820 {
4821 	struct xhci_hcd		*xhci;
4822 	struct device		*dev = hcd->self.controller;
4823 	int			retval;
4824 
4825 	/* Accept arbitrarily long scatter-gather lists */
4826 	hcd->self.sg_tablesize = ~0;
4827 
4828 	/* support to build packet from discontinuous buffers */
4829 	hcd->self.no_sg_constraint = 1;
4830 
4831 	/* XHCI controllers don't stop the ep queue on short packets :| */
4832 	hcd->self.no_stop_on_short = 1;
4833 
4834 	if (usb_hcd_is_primary_hcd(hcd)) {
4835 		xhci = kzalloc(sizeof(struct xhci_hcd), GFP_KERNEL);
4836 		if (!xhci)
4837 			return -ENOMEM;
4838 		*((struct xhci_hcd **) hcd->hcd_priv) = xhci;
4839 		xhci->main_hcd = hcd;
4840 		/* Mark the first roothub as being USB 2.0.
4841 		 * The xHCI driver will register the USB 3.0 roothub.
4842 		 */
4843 		hcd->speed = HCD_USB2;
4844 		hcd->self.root_hub->speed = USB_SPEED_HIGH;
4845 		/*
4846 		 * USB 2.0 roothub under xHCI has an integrated TT,
4847 		 * (rate matching hub) as opposed to having an OHCI/UHCI
4848 		 * companion controller.
4849 		 */
4850 		hcd->has_tt = 1;
4851 	} else {
4852 		/* xHCI private pointer was set in xhci_pci_probe for the second
4853 		 * registered roothub.
4854 		 */
4855 		return 0;
4856 	}
4857 
4858 	xhci->cap_regs = hcd->regs;
4859 	xhci->op_regs = hcd->regs +
4860 		HC_LENGTH(readl(&xhci->cap_regs->hc_capbase));
4861 	xhci->run_regs = hcd->regs +
4862 		(readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK);
4863 	/* Cache read-only capability registers */
4864 	xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1);
4865 	xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2);
4866 	xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3);
4867 	xhci->hcc_params = readl(&xhci->cap_regs->hc_capbase);
4868 	xhci->hci_version = HC_VERSION(xhci->hcc_params);
4869 	xhci->hcc_params = readl(&xhci->cap_regs->hcc_params);
4870 	xhci_print_registers(xhci);
4871 
4872 	xhci->quirks = quirks;
4873 
4874 	get_quirks(dev, xhci);
4875 
4876 	/* In xhci controllers which follow xhci 1.0 spec gives a spurious
4877 	 * success event after a short transfer. This quirk will ignore such
4878 	 * spurious event.
4879 	 */
4880 	if (xhci->hci_version > 0x96)
4881 		xhci->quirks |= XHCI_SPURIOUS_SUCCESS;
4882 
4883 	/* Make sure the HC is halted. */
4884 	retval = xhci_halt(xhci);
4885 	if (retval)
4886 		goto error;
4887 
4888 	xhci_dbg(xhci, "Resetting HCD\n");
4889 	/* Reset the internal HC memory state and registers. */
4890 	retval = xhci_reset(xhci);
4891 	if (retval)
4892 		goto error;
4893 	xhci_dbg(xhci, "Reset complete\n");
4894 
4895 	/* Set dma_mask and coherent_dma_mask to 64-bits,
4896 	 * if xHC supports 64-bit addressing */
4897 	if (HCC_64BIT_ADDR(xhci->hcc_params) &&
4898 			!dma_set_mask(dev, DMA_BIT_MASK(64))) {
4899 		xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
4900 		dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
4901 	}
4902 
4903 	xhci_dbg(xhci, "Calling HCD init\n");
4904 	/* Initialize HCD and host controller data structures. */
4905 	retval = xhci_init(hcd);
4906 	if (retval)
4907 		goto error;
4908 	xhci_dbg(xhci, "Called HCD init\n");
4909 
4910 	xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%08x\n",
4911 		  xhci->hcc_params, xhci->hci_version, xhci->quirks);
4912 
4913 	return 0;
4914 error:
4915 	kfree(xhci);
4916 	return retval;
4917 }
4918 EXPORT_SYMBOL_GPL(xhci_gen_setup);
4919 
4920 static const struct hc_driver xhci_hc_driver = {
4921 	.description =		"xhci-hcd",
4922 	.product_desc =		"xHCI Host Controller",
4923 	.hcd_priv_size =	sizeof(struct xhci_hcd *),
4924 
4925 	/*
4926 	 * generic hardware linkage
4927 	 */
4928 	.irq =			xhci_irq,
4929 	.flags =		HCD_MEMORY | HCD_USB3 | HCD_SHARED,
4930 
4931 	/*
4932 	 * basic lifecycle operations
4933 	 */
4934 	.reset =		NULL, /* set in xhci_init_driver() */
4935 	.start =		xhci_run,
4936 	.stop =			xhci_stop,
4937 	.shutdown =		xhci_shutdown,
4938 
4939 	/*
4940 	 * managing i/o requests and associated device resources
4941 	 */
4942 	.urb_enqueue =		xhci_urb_enqueue,
4943 	.urb_dequeue =		xhci_urb_dequeue,
4944 	.alloc_dev =		xhci_alloc_dev,
4945 	.free_dev =		xhci_free_dev,
4946 	.alloc_streams =	xhci_alloc_streams,
4947 	.free_streams =		xhci_free_streams,
4948 	.add_endpoint =		xhci_add_endpoint,
4949 	.drop_endpoint =	xhci_drop_endpoint,
4950 	.endpoint_reset =	xhci_endpoint_reset,
4951 	.check_bandwidth =	xhci_check_bandwidth,
4952 	.reset_bandwidth =	xhci_reset_bandwidth,
4953 	.address_device =	xhci_address_device,
4954 	.enable_device =	xhci_enable_device,
4955 	.update_hub_device =	xhci_update_hub_device,
4956 	.reset_device =		xhci_discover_or_reset_device,
4957 
4958 	/*
4959 	 * scheduling support
4960 	 */
4961 	.get_frame_number =	xhci_get_frame,
4962 
4963 	/*
4964 	 * root hub support
4965 	 */
4966 	.hub_control =		xhci_hub_control,
4967 	.hub_status_data =	xhci_hub_status_data,
4968 	.bus_suspend =		xhci_bus_suspend,
4969 	.bus_resume =		xhci_bus_resume,
4970 
4971 	/*
4972 	 * call back when device connected and addressed
4973 	 */
4974 	.update_device =        xhci_update_device,
4975 	.set_usb2_hw_lpm =	xhci_set_usb2_hardware_lpm,
4976 	.enable_usb3_lpm_timeout =	xhci_enable_usb3_lpm_timeout,
4977 	.disable_usb3_lpm_timeout =	xhci_disable_usb3_lpm_timeout,
4978 	.find_raw_port_number =	xhci_find_raw_port_number,
4979 };
4980 
4981 void xhci_init_driver(struct hc_driver *drv, int (*setup_fn)(struct usb_hcd *))
4982 {
4983 	BUG_ON(!setup_fn);
4984 	*drv = xhci_hc_driver;
4985 	drv->reset = setup_fn;
4986 }
4987 EXPORT_SYMBOL_GPL(xhci_init_driver);
4988 
4989 MODULE_DESCRIPTION(DRIVER_DESC);
4990 MODULE_AUTHOR(DRIVER_AUTHOR);
4991 MODULE_LICENSE("GPL");
4992 
4993 static int __init xhci_hcd_init(void)
4994 {
4995 	/*
4996 	 * Check the compiler generated sizes of structures that must be laid
4997 	 * out in specific ways for hardware access.
4998 	 */
4999 	BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
5000 	BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8);
5001 	BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8);
5002 	/* xhci_device_control has eight fields, and also
5003 	 * embeds one xhci_slot_ctx and 31 xhci_ep_ctx
5004 	 */
5005 	BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8);
5006 	BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8);
5007 	BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8);
5008 	BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 7*32/8);
5009 	BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8);
5010 	/* xhci_run_regs has eight fields and embeds 128 xhci_intr_regs */
5011 	BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8);
5012 	return 0;
5013 }
5014 module_init(xhci_hcd_init);
5015