xref: /openbmc/linux/drivers/usb/host/xhci-mem.c (revision e639c869)
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * xHCI host controller driver
4  *
5  * Copyright (C) 2008 Intel Corp.
6  *
7  * Author: Sarah Sharp
8  * Some code borrowed from the Linux EHCI driver.
9  */
10 
11 #include <linux/usb.h>
12 #include <linux/pci.h>
13 #include <linux/slab.h>
14 #include <linux/dmapool.h>
15 #include <linux/dma-mapping.h>
16 
17 #include "xhci.h"
18 #include "xhci-trace.h"
19 #include "xhci-debugfs.h"
20 
21 /*
22  * Allocates a generic ring segment from the ring pool, sets the dma address,
23  * initializes the segment to zero, and sets the private next pointer to NULL.
24  *
25  * Section 4.11.1.1:
26  * "All components of all Command and Transfer TRBs shall be initialized to '0'"
27  */
28 static struct xhci_segment *xhci_segment_alloc(struct xhci_hcd *xhci,
29 					       unsigned int cycle_state,
30 					       unsigned int max_packet,
31 					       gfp_t flags)
32 {
33 	struct xhci_segment *seg;
34 	dma_addr_t	dma;
35 	int		i;
36 
37 	seg = kzalloc(sizeof *seg, flags);
38 	if (!seg)
39 		return NULL;
40 
41 	seg->trbs = dma_pool_zalloc(xhci->segment_pool, flags, &dma);
42 	if (!seg->trbs) {
43 		kfree(seg);
44 		return NULL;
45 	}
46 
47 	if (max_packet) {
48 		seg->bounce_buf = kzalloc(max_packet, flags);
49 		if (!seg->bounce_buf) {
50 			dma_pool_free(xhci->segment_pool, seg->trbs, dma);
51 			kfree(seg);
52 			return NULL;
53 		}
54 	}
55 	/* If the cycle state is 0, set the cycle bit to 1 for all the TRBs */
56 	if (cycle_state == 0) {
57 		for (i = 0; i < TRBS_PER_SEGMENT; i++)
58 			seg->trbs[i].link.control |= cpu_to_le32(TRB_CYCLE);
59 	}
60 	seg->dma = dma;
61 	seg->next = NULL;
62 
63 	return seg;
64 }
65 
66 static void xhci_segment_free(struct xhci_hcd *xhci, struct xhci_segment *seg)
67 {
68 	if (seg->trbs) {
69 		dma_pool_free(xhci->segment_pool, seg->trbs, seg->dma);
70 		seg->trbs = NULL;
71 	}
72 	kfree(seg->bounce_buf);
73 	kfree(seg);
74 }
75 
76 static void xhci_free_segments_for_ring(struct xhci_hcd *xhci,
77 				struct xhci_segment *first)
78 {
79 	struct xhci_segment *seg;
80 
81 	seg = first->next;
82 	while (seg != first) {
83 		struct xhci_segment *next = seg->next;
84 		xhci_segment_free(xhci, seg);
85 		seg = next;
86 	}
87 	xhci_segment_free(xhci, first);
88 }
89 
90 /*
91  * Make the prev segment point to the next segment.
92  *
93  * Change the last TRB in the prev segment to be a Link TRB which points to the
94  * DMA address of the next segment.  The caller needs to set any Link TRB
95  * related flags, such as End TRB, Toggle Cycle, and no snoop.
96  */
97 static void xhci_link_segments(struct xhci_hcd *xhci, struct xhci_segment *prev,
98 		struct xhci_segment *next, enum xhci_ring_type type)
99 {
100 	u32 val;
101 
102 	if (!prev || !next)
103 		return;
104 	prev->next = next;
105 	if (type != TYPE_EVENT) {
106 		prev->trbs[TRBS_PER_SEGMENT-1].link.segment_ptr =
107 			cpu_to_le64(next->dma);
108 
109 		/* Set the last TRB in the segment to have a TRB type ID of Link TRB */
110 		val = le32_to_cpu(prev->trbs[TRBS_PER_SEGMENT-1].link.control);
111 		val &= ~TRB_TYPE_BITMASK;
112 		val |= TRB_TYPE(TRB_LINK);
113 		/* Always set the chain bit with 0.95 hardware */
114 		/* Set chain bit for isoc rings on AMD 0.96 host */
115 		if (xhci_link_trb_quirk(xhci) ||
116 				(type == TYPE_ISOC &&
117 				 (xhci->quirks & XHCI_AMD_0x96_HOST)))
118 			val |= TRB_CHAIN;
119 		prev->trbs[TRBS_PER_SEGMENT-1].link.control = cpu_to_le32(val);
120 	}
121 }
122 
123 /*
124  * Link the ring to the new segments.
125  * Set Toggle Cycle for the new ring if needed.
126  */
127 static void xhci_link_rings(struct xhci_hcd *xhci, struct xhci_ring *ring,
128 		struct xhci_segment *first, struct xhci_segment *last,
129 		unsigned int num_segs)
130 {
131 	struct xhci_segment *next;
132 
133 	if (!ring || !first || !last)
134 		return;
135 
136 	next = ring->enq_seg->next;
137 	xhci_link_segments(xhci, ring->enq_seg, first, ring->type);
138 	xhci_link_segments(xhci, last, next, ring->type);
139 	ring->num_segs += num_segs;
140 	ring->num_trbs_free += (TRBS_PER_SEGMENT - 1) * num_segs;
141 
142 	if (ring->type != TYPE_EVENT && ring->enq_seg == ring->last_seg) {
143 		ring->last_seg->trbs[TRBS_PER_SEGMENT-1].link.control
144 			&= ~cpu_to_le32(LINK_TOGGLE);
145 		last->trbs[TRBS_PER_SEGMENT-1].link.control
146 			|= cpu_to_le32(LINK_TOGGLE);
147 		ring->last_seg = last;
148 	}
149 }
150 
151 /*
152  * We need a radix tree for mapping physical addresses of TRBs to which stream
153  * ID they belong to.  We need to do this because the host controller won't tell
154  * us which stream ring the TRB came from.  We could store the stream ID in an
155  * event data TRB, but that doesn't help us for the cancellation case, since the
156  * endpoint may stop before it reaches that event data TRB.
157  *
158  * The radix tree maps the upper portion of the TRB DMA address to a ring
159  * segment that has the same upper portion of DMA addresses.  For example, say I
160  * have segments of size 1KB, that are always 1KB aligned.  A segment may
161  * start at 0x10c91000 and end at 0x10c913f0.  If I use the upper 10 bits, the
162  * key to the stream ID is 0x43244.  I can use the DMA address of the TRB to
163  * pass the radix tree a key to get the right stream ID:
164  *
165  *	0x10c90fff >> 10 = 0x43243
166  *	0x10c912c0 >> 10 = 0x43244
167  *	0x10c91400 >> 10 = 0x43245
168  *
169  * Obviously, only those TRBs with DMA addresses that are within the segment
170  * will make the radix tree return the stream ID for that ring.
171  *
172  * Caveats for the radix tree:
173  *
174  * The radix tree uses an unsigned long as a key pair.  On 32-bit systems, an
175  * unsigned long will be 32-bits; on a 64-bit system an unsigned long will be
176  * 64-bits.  Since we only request 32-bit DMA addresses, we can use that as the
177  * key on 32-bit or 64-bit systems (it would also be fine if we asked for 64-bit
178  * PCI DMA addresses on a 64-bit system).  There might be a problem on 32-bit
179  * extended systems (where the DMA address can be bigger than 32-bits),
180  * if we allow the PCI dma mask to be bigger than 32-bits.  So don't do that.
181  */
182 static int xhci_insert_segment_mapping(struct radix_tree_root *trb_address_map,
183 		struct xhci_ring *ring,
184 		struct xhci_segment *seg,
185 		gfp_t mem_flags)
186 {
187 	unsigned long key;
188 	int ret;
189 
190 	key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
191 	/* Skip any segments that were already added. */
192 	if (radix_tree_lookup(trb_address_map, key))
193 		return 0;
194 
195 	ret = radix_tree_maybe_preload(mem_flags);
196 	if (ret)
197 		return ret;
198 	ret = radix_tree_insert(trb_address_map,
199 			key, ring);
200 	radix_tree_preload_end();
201 	return ret;
202 }
203 
204 static void xhci_remove_segment_mapping(struct radix_tree_root *trb_address_map,
205 		struct xhci_segment *seg)
206 {
207 	unsigned long key;
208 
209 	key = (unsigned long)(seg->dma >> TRB_SEGMENT_SHIFT);
210 	if (radix_tree_lookup(trb_address_map, key))
211 		radix_tree_delete(trb_address_map, key);
212 }
213 
214 static int xhci_update_stream_segment_mapping(
215 		struct radix_tree_root *trb_address_map,
216 		struct xhci_ring *ring,
217 		struct xhci_segment *first_seg,
218 		struct xhci_segment *last_seg,
219 		gfp_t mem_flags)
220 {
221 	struct xhci_segment *seg;
222 	struct xhci_segment *failed_seg;
223 	int ret;
224 
225 	if (WARN_ON_ONCE(trb_address_map == NULL))
226 		return 0;
227 
228 	seg = first_seg;
229 	do {
230 		ret = xhci_insert_segment_mapping(trb_address_map,
231 				ring, seg, mem_flags);
232 		if (ret)
233 			goto remove_streams;
234 		if (seg == last_seg)
235 			return 0;
236 		seg = seg->next;
237 	} while (seg != first_seg);
238 
239 	return 0;
240 
241 remove_streams:
242 	failed_seg = seg;
243 	seg = first_seg;
244 	do {
245 		xhci_remove_segment_mapping(trb_address_map, seg);
246 		if (seg == failed_seg)
247 			return ret;
248 		seg = seg->next;
249 	} while (seg != first_seg);
250 
251 	return ret;
252 }
253 
254 static void xhci_remove_stream_mapping(struct xhci_ring *ring)
255 {
256 	struct xhci_segment *seg;
257 
258 	if (WARN_ON_ONCE(ring->trb_address_map == NULL))
259 		return;
260 
261 	seg = ring->first_seg;
262 	do {
263 		xhci_remove_segment_mapping(ring->trb_address_map, seg);
264 		seg = seg->next;
265 	} while (seg != ring->first_seg);
266 }
267 
268 static int xhci_update_stream_mapping(struct xhci_ring *ring, gfp_t mem_flags)
269 {
270 	return xhci_update_stream_segment_mapping(ring->trb_address_map, ring,
271 			ring->first_seg, ring->last_seg, mem_flags);
272 }
273 
274 /* XXX: Do we need the hcd structure in all these functions? */
275 void xhci_ring_free(struct xhci_hcd *xhci, struct xhci_ring *ring)
276 {
277 	if (!ring)
278 		return;
279 
280 	trace_xhci_ring_free(ring);
281 
282 	if (ring->first_seg) {
283 		if (ring->type == TYPE_STREAM)
284 			xhci_remove_stream_mapping(ring);
285 		xhci_free_segments_for_ring(xhci, ring->first_seg);
286 	}
287 
288 	kfree(ring);
289 }
290 
291 static void xhci_initialize_ring_info(struct xhci_ring *ring,
292 					unsigned int cycle_state)
293 {
294 	/* The ring is empty, so the enqueue pointer == dequeue pointer */
295 	ring->enqueue = ring->first_seg->trbs;
296 	ring->enq_seg = ring->first_seg;
297 	ring->dequeue = ring->enqueue;
298 	ring->deq_seg = ring->first_seg;
299 	/* The ring is initialized to 0. The producer must write 1 to the cycle
300 	 * bit to handover ownership of the TRB, so PCS = 1.  The consumer must
301 	 * compare CCS to the cycle bit to check ownership, so CCS = 1.
302 	 *
303 	 * New rings are initialized with cycle state equal to 1; if we are
304 	 * handling ring expansion, set the cycle state equal to the old ring.
305 	 */
306 	ring->cycle_state = cycle_state;
307 
308 	/*
309 	 * Each segment has a link TRB, and leave an extra TRB for SW
310 	 * accounting purpose
311 	 */
312 	ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
313 }
314 
315 /* Allocate segments and link them for a ring */
316 static int xhci_alloc_segments_for_ring(struct xhci_hcd *xhci,
317 		struct xhci_segment **first, struct xhci_segment **last,
318 		unsigned int num_segs, unsigned int cycle_state,
319 		enum xhci_ring_type type, unsigned int max_packet, gfp_t flags)
320 {
321 	struct xhci_segment *prev;
322 
323 	prev = xhci_segment_alloc(xhci, cycle_state, max_packet, flags);
324 	if (!prev)
325 		return -ENOMEM;
326 	num_segs--;
327 
328 	*first = prev;
329 	while (num_segs > 0) {
330 		struct xhci_segment	*next;
331 
332 		next = xhci_segment_alloc(xhci, cycle_state, max_packet, flags);
333 		if (!next) {
334 			prev = *first;
335 			while (prev) {
336 				next = prev->next;
337 				xhci_segment_free(xhci, prev);
338 				prev = next;
339 			}
340 			return -ENOMEM;
341 		}
342 		xhci_link_segments(xhci, prev, next, type);
343 
344 		prev = next;
345 		num_segs--;
346 	}
347 	xhci_link_segments(xhci, prev, *first, type);
348 	*last = prev;
349 
350 	return 0;
351 }
352 
353 /**
354  * Create a new ring with zero or more segments.
355  *
356  * Link each segment together into a ring.
357  * Set the end flag and the cycle toggle bit on the last segment.
358  * See section 4.9.1 and figures 15 and 16.
359  */
360 static struct xhci_ring *xhci_ring_alloc(struct xhci_hcd *xhci,
361 		unsigned int num_segs, unsigned int cycle_state,
362 		enum xhci_ring_type type, unsigned int max_packet, gfp_t flags)
363 {
364 	struct xhci_ring	*ring;
365 	int ret;
366 
367 	ring = kzalloc(sizeof *(ring), flags);
368 	if (!ring)
369 		return NULL;
370 
371 	ring->num_segs = num_segs;
372 	ring->bounce_buf_len = max_packet;
373 	INIT_LIST_HEAD(&ring->td_list);
374 	ring->type = type;
375 	if (num_segs == 0)
376 		return ring;
377 
378 	ret = xhci_alloc_segments_for_ring(xhci, &ring->first_seg,
379 			&ring->last_seg, num_segs, cycle_state, type,
380 			max_packet, flags);
381 	if (ret)
382 		goto fail;
383 
384 	/* Only event ring does not use link TRB */
385 	if (type != TYPE_EVENT) {
386 		/* See section 4.9.2.1 and 6.4.4.1 */
387 		ring->last_seg->trbs[TRBS_PER_SEGMENT - 1].link.control |=
388 			cpu_to_le32(LINK_TOGGLE);
389 	}
390 	xhci_initialize_ring_info(ring, cycle_state);
391 	trace_xhci_ring_alloc(ring);
392 	return ring;
393 
394 fail:
395 	kfree(ring);
396 	return NULL;
397 }
398 
399 void xhci_free_endpoint_ring(struct xhci_hcd *xhci,
400 		struct xhci_virt_device *virt_dev,
401 		unsigned int ep_index)
402 {
403 	xhci_ring_free(xhci, virt_dev->eps[ep_index].ring);
404 	virt_dev->eps[ep_index].ring = NULL;
405 }
406 
407 /*
408  * Expand an existing ring.
409  * Allocate a new ring which has same segment numbers and link the two rings.
410  */
411 int xhci_ring_expansion(struct xhci_hcd *xhci, struct xhci_ring *ring,
412 				unsigned int num_trbs, gfp_t flags)
413 {
414 	struct xhci_segment	*first;
415 	struct xhci_segment	*last;
416 	unsigned int		num_segs;
417 	unsigned int		num_segs_needed;
418 	int			ret;
419 
420 	num_segs_needed = (num_trbs + (TRBS_PER_SEGMENT - 1) - 1) /
421 				(TRBS_PER_SEGMENT - 1);
422 
423 	/* Allocate number of segments we needed, or double the ring size */
424 	num_segs = ring->num_segs > num_segs_needed ?
425 			ring->num_segs : num_segs_needed;
426 
427 	ret = xhci_alloc_segments_for_ring(xhci, &first, &last,
428 			num_segs, ring->cycle_state, ring->type,
429 			ring->bounce_buf_len, flags);
430 	if (ret)
431 		return -ENOMEM;
432 
433 	if (ring->type == TYPE_STREAM)
434 		ret = xhci_update_stream_segment_mapping(ring->trb_address_map,
435 						ring, first, last, flags);
436 	if (ret) {
437 		struct xhci_segment *next;
438 		do {
439 			next = first->next;
440 			xhci_segment_free(xhci, first);
441 			if (first == last)
442 				break;
443 			first = next;
444 		} while (true);
445 		return ret;
446 	}
447 
448 	xhci_link_rings(xhci, ring, first, last, num_segs);
449 	trace_xhci_ring_expansion(ring);
450 	xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion,
451 			"ring expansion succeed, now has %d segments",
452 			ring->num_segs);
453 
454 	return 0;
455 }
456 
457 static struct xhci_container_ctx *xhci_alloc_container_ctx(struct xhci_hcd *xhci,
458 						    int type, gfp_t flags)
459 {
460 	struct xhci_container_ctx *ctx;
461 
462 	if ((type != XHCI_CTX_TYPE_DEVICE) && (type != XHCI_CTX_TYPE_INPUT))
463 		return NULL;
464 
465 	ctx = kzalloc(sizeof(*ctx), flags);
466 	if (!ctx)
467 		return NULL;
468 
469 	ctx->type = type;
470 	ctx->size = HCC_64BYTE_CONTEXT(xhci->hcc_params) ? 2048 : 1024;
471 	if (type == XHCI_CTX_TYPE_INPUT)
472 		ctx->size += CTX_SIZE(xhci->hcc_params);
473 
474 	ctx->bytes = dma_pool_zalloc(xhci->device_pool, flags, &ctx->dma);
475 	if (!ctx->bytes) {
476 		kfree(ctx);
477 		return NULL;
478 	}
479 	return ctx;
480 }
481 
482 static void xhci_free_container_ctx(struct xhci_hcd *xhci,
483 			     struct xhci_container_ctx *ctx)
484 {
485 	if (!ctx)
486 		return;
487 	dma_pool_free(xhci->device_pool, ctx->bytes, ctx->dma);
488 	kfree(ctx);
489 }
490 
491 struct xhci_input_control_ctx *xhci_get_input_control_ctx(
492 					      struct xhci_container_ctx *ctx)
493 {
494 	if (ctx->type != XHCI_CTX_TYPE_INPUT)
495 		return NULL;
496 
497 	return (struct xhci_input_control_ctx *)ctx->bytes;
498 }
499 
500 struct xhci_slot_ctx *xhci_get_slot_ctx(struct xhci_hcd *xhci,
501 					struct xhci_container_ctx *ctx)
502 {
503 	if (ctx->type == XHCI_CTX_TYPE_DEVICE)
504 		return (struct xhci_slot_ctx *)ctx->bytes;
505 
506 	return (struct xhci_slot_ctx *)
507 		(ctx->bytes + CTX_SIZE(xhci->hcc_params));
508 }
509 
510 struct xhci_ep_ctx *xhci_get_ep_ctx(struct xhci_hcd *xhci,
511 				    struct xhci_container_ctx *ctx,
512 				    unsigned int ep_index)
513 {
514 	/* increment ep index by offset of start of ep ctx array */
515 	ep_index++;
516 	if (ctx->type == XHCI_CTX_TYPE_INPUT)
517 		ep_index++;
518 
519 	return (struct xhci_ep_ctx *)
520 		(ctx->bytes + (ep_index * CTX_SIZE(xhci->hcc_params)));
521 }
522 
523 
524 /***************** Streams structures manipulation *************************/
525 
526 static void xhci_free_stream_ctx(struct xhci_hcd *xhci,
527 		unsigned int num_stream_ctxs,
528 		struct xhci_stream_ctx *stream_ctx, dma_addr_t dma)
529 {
530 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
531 	size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs;
532 
533 	if (size > MEDIUM_STREAM_ARRAY_SIZE)
534 		dma_free_coherent(dev, size,
535 				stream_ctx, dma);
536 	else if (size <= SMALL_STREAM_ARRAY_SIZE)
537 		return dma_pool_free(xhci->small_streams_pool,
538 				stream_ctx, dma);
539 	else
540 		return dma_pool_free(xhci->medium_streams_pool,
541 				stream_ctx, dma);
542 }
543 
544 /*
545  * The stream context array for each endpoint with bulk streams enabled can
546  * vary in size, based on:
547  *  - how many streams the endpoint supports,
548  *  - the maximum primary stream array size the host controller supports,
549  *  - and how many streams the device driver asks for.
550  *
551  * The stream context array must be a power of 2, and can be as small as
552  * 64 bytes or as large as 1MB.
553  */
554 static struct xhci_stream_ctx *xhci_alloc_stream_ctx(struct xhci_hcd *xhci,
555 		unsigned int num_stream_ctxs, dma_addr_t *dma,
556 		gfp_t mem_flags)
557 {
558 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
559 	size_t size = sizeof(struct xhci_stream_ctx) * num_stream_ctxs;
560 
561 	if (size > MEDIUM_STREAM_ARRAY_SIZE)
562 		return dma_alloc_coherent(dev, size,
563 				dma, mem_flags);
564 	else if (size <= SMALL_STREAM_ARRAY_SIZE)
565 		return dma_pool_alloc(xhci->small_streams_pool,
566 				mem_flags, dma);
567 	else
568 		return dma_pool_alloc(xhci->medium_streams_pool,
569 				mem_flags, dma);
570 }
571 
572 struct xhci_ring *xhci_dma_to_transfer_ring(
573 		struct xhci_virt_ep *ep,
574 		u64 address)
575 {
576 	if (ep->ep_state & EP_HAS_STREAMS)
577 		return radix_tree_lookup(&ep->stream_info->trb_address_map,
578 				address >> TRB_SEGMENT_SHIFT);
579 	return ep->ring;
580 }
581 
582 struct xhci_ring *xhci_stream_id_to_ring(
583 		struct xhci_virt_device *dev,
584 		unsigned int ep_index,
585 		unsigned int stream_id)
586 {
587 	struct xhci_virt_ep *ep = &dev->eps[ep_index];
588 
589 	if (stream_id == 0)
590 		return ep->ring;
591 	if (!ep->stream_info)
592 		return NULL;
593 
594 	if (stream_id > ep->stream_info->num_streams)
595 		return NULL;
596 	return ep->stream_info->stream_rings[stream_id];
597 }
598 
599 /*
600  * Change an endpoint's internal structure so it supports stream IDs.  The
601  * number of requested streams includes stream 0, which cannot be used by device
602  * drivers.
603  *
604  * The number of stream contexts in the stream context array may be bigger than
605  * the number of streams the driver wants to use.  This is because the number of
606  * stream context array entries must be a power of two.
607  */
608 struct xhci_stream_info *xhci_alloc_stream_info(struct xhci_hcd *xhci,
609 		unsigned int num_stream_ctxs,
610 		unsigned int num_streams,
611 		unsigned int max_packet, gfp_t mem_flags)
612 {
613 	struct xhci_stream_info *stream_info;
614 	u32 cur_stream;
615 	struct xhci_ring *cur_ring;
616 	u64 addr;
617 	int ret;
618 
619 	xhci_dbg(xhci, "Allocating %u streams and %u "
620 			"stream context array entries.\n",
621 			num_streams, num_stream_ctxs);
622 	if (xhci->cmd_ring_reserved_trbs == MAX_RSVD_CMD_TRBS) {
623 		xhci_dbg(xhci, "Command ring has no reserved TRBs available\n");
624 		return NULL;
625 	}
626 	xhci->cmd_ring_reserved_trbs++;
627 
628 	stream_info = kzalloc(sizeof(struct xhci_stream_info), mem_flags);
629 	if (!stream_info)
630 		goto cleanup_trbs;
631 
632 	stream_info->num_streams = num_streams;
633 	stream_info->num_stream_ctxs = num_stream_ctxs;
634 
635 	/* Initialize the array of virtual pointers to stream rings. */
636 	stream_info->stream_rings = kzalloc(
637 			sizeof(struct xhci_ring *)*num_streams,
638 			mem_flags);
639 	if (!stream_info->stream_rings)
640 		goto cleanup_info;
641 
642 	/* Initialize the array of DMA addresses for stream rings for the HW. */
643 	stream_info->stream_ctx_array = xhci_alloc_stream_ctx(xhci,
644 			num_stream_ctxs, &stream_info->ctx_array_dma,
645 			mem_flags);
646 	if (!stream_info->stream_ctx_array)
647 		goto cleanup_ctx;
648 	memset(stream_info->stream_ctx_array, 0,
649 			sizeof(struct xhci_stream_ctx)*num_stream_ctxs);
650 
651 	/* Allocate everything needed to free the stream rings later */
652 	stream_info->free_streams_command =
653 		xhci_alloc_command(xhci, true, true, mem_flags);
654 	if (!stream_info->free_streams_command)
655 		goto cleanup_ctx;
656 
657 	INIT_RADIX_TREE(&stream_info->trb_address_map, GFP_ATOMIC);
658 
659 	/* Allocate rings for all the streams that the driver will use,
660 	 * and add their segment DMA addresses to the radix tree.
661 	 * Stream 0 is reserved.
662 	 */
663 
664 	for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
665 		stream_info->stream_rings[cur_stream] =
666 			xhci_ring_alloc(xhci, 2, 1, TYPE_STREAM, max_packet,
667 					mem_flags);
668 		cur_ring = stream_info->stream_rings[cur_stream];
669 		if (!cur_ring)
670 			goto cleanup_rings;
671 		cur_ring->stream_id = cur_stream;
672 		cur_ring->trb_address_map = &stream_info->trb_address_map;
673 		/* Set deq ptr, cycle bit, and stream context type */
674 		addr = cur_ring->first_seg->dma |
675 			SCT_FOR_CTX(SCT_PRI_TR) |
676 			cur_ring->cycle_state;
677 		stream_info->stream_ctx_array[cur_stream].stream_ring =
678 			cpu_to_le64(addr);
679 		xhci_dbg(xhci, "Setting stream %d ring ptr to 0x%08llx\n",
680 				cur_stream, (unsigned long long) addr);
681 
682 		ret = xhci_update_stream_mapping(cur_ring, mem_flags);
683 		if (ret) {
684 			xhci_ring_free(xhci, cur_ring);
685 			stream_info->stream_rings[cur_stream] = NULL;
686 			goto cleanup_rings;
687 		}
688 	}
689 	/* Leave the other unused stream ring pointers in the stream context
690 	 * array initialized to zero.  This will cause the xHC to give us an
691 	 * error if the device asks for a stream ID we don't have setup (if it
692 	 * was any other way, the host controller would assume the ring is
693 	 * "empty" and wait forever for data to be queued to that stream ID).
694 	 */
695 
696 	return stream_info;
697 
698 cleanup_rings:
699 	for (cur_stream = 1; cur_stream < num_streams; cur_stream++) {
700 		cur_ring = stream_info->stream_rings[cur_stream];
701 		if (cur_ring) {
702 			xhci_ring_free(xhci, cur_ring);
703 			stream_info->stream_rings[cur_stream] = NULL;
704 		}
705 	}
706 	xhci_free_command(xhci, stream_info->free_streams_command);
707 cleanup_ctx:
708 	kfree(stream_info->stream_rings);
709 cleanup_info:
710 	kfree(stream_info);
711 cleanup_trbs:
712 	xhci->cmd_ring_reserved_trbs--;
713 	return NULL;
714 }
715 /*
716  * Sets the MaxPStreams field and the Linear Stream Array field.
717  * Sets the dequeue pointer to the stream context array.
718  */
719 void xhci_setup_streams_ep_input_ctx(struct xhci_hcd *xhci,
720 		struct xhci_ep_ctx *ep_ctx,
721 		struct xhci_stream_info *stream_info)
722 {
723 	u32 max_primary_streams;
724 	/* MaxPStreams is the number of stream context array entries, not the
725 	 * number we're actually using.  Must be in 2^(MaxPstreams + 1) format.
726 	 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
727 	 */
728 	max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
729 	xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
730 			"Setting number of stream ctx array entries to %u",
731 			1 << (max_primary_streams + 1));
732 	ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
733 	ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
734 				       | EP_HAS_LSA);
735 	ep_ctx->deq  = cpu_to_le64(stream_info->ctx_array_dma);
736 }
737 
738 /*
739  * Sets the MaxPStreams field and the Linear Stream Array field to 0.
740  * Reinstalls the "normal" endpoint ring (at its previous dequeue mark,
741  * not at the beginning of the ring).
742  */
743 void xhci_setup_no_streams_ep_input_ctx(struct xhci_ep_ctx *ep_ctx,
744 		struct xhci_virt_ep *ep)
745 {
746 	dma_addr_t addr;
747 	ep_ctx->ep_info &= cpu_to_le32(~(EP_MAXPSTREAMS_MASK | EP_HAS_LSA));
748 	addr = xhci_trb_virt_to_dma(ep->ring->deq_seg, ep->ring->dequeue);
749 	ep_ctx->deq  = cpu_to_le64(addr | ep->ring->cycle_state);
750 }
751 
752 /* Frees all stream contexts associated with the endpoint,
753  *
754  * Caller should fix the endpoint context streams fields.
755  */
756 void xhci_free_stream_info(struct xhci_hcd *xhci,
757 		struct xhci_stream_info *stream_info)
758 {
759 	int cur_stream;
760 	struct xhci_ring *cur_ring;
761 
762 	if (!stream_info)
763 		return;
764 
765 	for (cur_stream = 1; cur_stream < stream_info->num_streams;
766 			cur_stream++) {
767 		cur_ring = stream_info->stream_rings[cur_stream];
768 		if (cur_ring) {
769 			xhci_ring_free(xhci, cur_ring);
770 			stream_info->stream_rings[cur_stream] = NULL;
771 		}
772 	}
773 	xhci_free_command(xhci, stream_info->free_streams_command);
774 	xhci->cmd_ring_reserved_trbs--;
775 	if (stream_info->stream_ctx_array)
776 		xhci_free_stream_ctx(xhci,
777 				stream_info->num_stream_ctxs,
778 				stream_info->stream_ctx_array,
779 				stream_info->ctx_array_dma);
780 
781 	kfree(stream_info->stream_rings);
782 	kfree(stream_info);
783 }
784 
785 
786 /***************** Device context manipulation *************************/
787 
788 static void xhci_init_endpoint_timer(struct xhci_hcd *xhci,
789 		struct xhci_virt_ep *ep)
790 {
791 	timer_setup(&ep->stop_cmd_timer, xhci_stop_endpoint_command_watchdog,
792 		    0);
793 	ep->xhci = xhci;
794 }
795 
796 static void xhci_free_tt_info(struct xhci_hcd *xhci,
797 		struct xhci_virt_device *virt_dev,
798 		int slot_id)
799 {
800 	struct list_head *tt_list_head;
801 	struct xhci_tt_bw_info *tt_info, *next;
802 	bool slot_found = false;
803 
804 	/* If the device never made it past the Set Address stage,
805 	 * it may not have the real_port set correctly.
806 	 */
807 	if (virt_dev->real_port == 0 ||
808 			virt_dev->real_port > HCS_MAX_PORTS(xhci->hcs_params1)) {
809 		xhci_dbg(xhci, "Bad real port.\n");
810 		return;
811 	}
812 
813 	tt_list_head = &(xhci->rh_bw[virt_dev->real_port - 1].tts);
814 	list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) {
815 		/* Multi-TT hubs will have more than one entry */
816 		if (tt_info->slot_id == slot_id) {
817 			slot_found = true;
818 			list_del(&tt_info->tt_list);
819 			kfree(tt_info);
820 		} else if (slot_found) {
821 			break;
822 		}
823 	}
824 }
825 
826 int xhci_alloc_tt_info(struct xhci_hcd *xhci,
827 		struct xhci_virt_device *virt_dev,
828 		struct usb_device *hdev,
829 		struct usb_tt *tt, gfp_t mem_flags)
830 {
831 	struct xhci_tt_bw_info		*tt_info;
832 	unsigned int			num_ports;
833 	int				i, j;
834 
835 	if (!tt->multi)
836 		num_ports = 1;
837 	else
838 		num_ports = hdev->maxchild;
839 
840 	for (i = 0; i < num_ports; i++, tt_info++) {
841 		struct xhci_interval_bw_table *bw_table;
842 
843 		tt_info = kzalloc(sizeof(*tt_info), mem_flags);
844 		if (!tt_info)
845 			goto free_tts;
846 		INIT_LIST_HEAD(&tt_info->tt_list);
847 		list_add(&tt_info->tt_list,
848 				&xhci->rh_bw[virt_dev->real_port - 1].tts);
849 		tt_info->slot_id = virt_dev->udev->slot_id;
850 		if (tt->multi)
851 			tt_info->ttport = i+1;
852 		bw_table = &tt_info->bw_table;
853 		for (j = 0; j < XHCI_MAX_INTERVAL; j++)
854 			INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints);
855 	}
856 	return 0;
857 
858 free_tts:
859 	xhci_free_tt_info(xhci, virt_dev, virt_dev->udev->slot_id);
860 	return -ENOMEM;
861 }
862 
863 
864 /* All the xhci_tds in the ring's TD list should be freed at this point.
865  * Should be called with xhci->lock held if there is any chance the TT lists
866  * will be manipulated by the configure endpoint, allocate device, or update
867  * hub functions while this function is removing the TT entries from the list.
868  */
869 void xhci_free_virt_device(struct xhci_hcd *xhci, int slot_id)
870 {
871 	struct xhci_virt_device *dev;
872 	int i;
873 	int old_active_eps = 0;
874 
875 	/* Slot ID 0 is reserved */
876 	if (slot_id == 0 || !xhci->devs[slot_id])
877 		return;
878 
879 	dev = xhci->devs[slot_id];
880 
881 	trace_xhci_free_virt_device(dev);
882 
883 	xhci->dcbaa->dev_context_ptrs[slot_id] = 0;
884 	if (!dev)
885 		return;
886 
887 	if (dev->tt_info)
888 		old_active_eps = dev->tt_info->active_eps;
889 
890 	for (i = 0; i < 31; i++) {
891 		if (dev->eps[i].ring)
892 			xhci_ring_free(xhci, dev->eps[i].ring);
893 		if (dev->eps[i].stream_info)
894 			xhci_free_stream_info(xhci,
895 					dev->eps[i].stream_info);
896 		/* Endpoints on the TT/root port lists should have been removed
897 		 * when usb_disable_device() was called for the device.
898 		 * We can't drop them anyway, because the udev might have gone
899 		 * away by this point, and we can't tell what speed it was.
900 		 */
901 		if (!list_empty(&dev->eps[i].bw_endpoint_list))
902 			xhci_warn(xhci, "Slot %u endpoint %u "
903 					"not removed from BW list!\n",
904 					slot_id, i);
905 	}
906 	/* If this is a hub, free the TT(s) from the TT list */
907 	xhci_free_tt_info(xhci, dev, slot_id);
908 	/* If necessary, update the number of active TTs on this root port */
909 	xhci_update_tt_active_eps(xhci, dev, old_active_eps);
910 
911 	if (dev->in_ctx)
912 		xhci_free_container_ctx(xhci, dev->in_ctx);
913 	if (dev->out_ctx)
914 		xhci_free_container_ctx(xhci, dev->out_ctx);
915 
916 	kfree(xhci->devs[slot_id]);
917 	xhci->devs[slot_id] = NULL;
918 }
919 
920 /*
921  * Free a virt_device structure.
922  * If the virt_device added a tt_info (a hub) and has children pointing to
923  * that tt_info, then free the child first. Recursive.
924  * We can't rely on udev at this point to find child-parent relationships.
925  */
926 void xhci_free_virt_devices_depth_first(struct xhci_hcd *xhci, int slot_id)
927 {
928 	struct xhci_virt_device *vdev;
929 	struct list_head *tt_list_head;
930 	struct xhci_tt_bw_info *tt_info, *next;
931 	int i;
932 
933 	vdev = xhci->devs[slot_id];
934 	if (!vdev)
935 		return;
936 
937 	tt_list_head = &(xhci->rh_bw[vdev->real_port - 1].tts);
938 	list_for_each_entry_safe(tt_info, next, tt_list_head, tt_list) {
939 		/* is this a hub device that added a tt_info to the tts list */
940 		if (tt_info->slot_id == slot_id) {
941 			/* are any devices using this tt_info? */
942 			for (i = 1; i < HCS_MAX_SLOTS(xhci->hcs_params1); i++) {
943 				vdev = xhci->devs[i];
944 				if (vdev && (vdev->tt_info == tt_info))
945 					xhci_free_virt_devices_depth_first(
946 						xhci, i);
947 			}
948 		}
949 	}
950 	/* we are now at a leaf device */
951 	xhci_debugfs_remove_slot(xhci, slot_id);
952 	xhci_free_virt_device(xhci, slot_id);
953 }
954 
955 int xhci_alloc_virt_device(struct xhci_hcd *xhci, int slot_id,
956 		struct usb_device *udev, gfp_t flags)
957 {
958 	struct xhci_virt_device *dev;
959 	int i;
960 
961 	/* Slot ID 0 is reserved */
962 	if (slot_id == 0 || xhci->devs[slot_id]) {
963 		xhci_warn(xhci, "Bad Slot ID %d\n", slot_id);
964 		return 0;
965 	}
966 
967 	xhci->devs[slot_id] = kzalloc(sizeof(*xhci->devs[slot_id]), flags);
968 	if (!xhci->devs[slot_id])
969 		return 0;
970 	dev = xhci->devs[slot_id];
971 
972 	/* Allocate the (output) device context that will be used in the HC. */
973 	dev->out_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_DEVICE, flags);
974 	if (!dev->out_ctx)
975 		goto fail;
976 
977 	xhci_dbg(xhci, "Slot %d output ctx = 0x%llx (dma)\n", slot_id,
978 			(unsigned long long)dev->out_ctx->dma);
979 
980 	/* Allocate the (input) device context for address device command */
981 	dev->in_ctx = xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT, flags);
982 	if (!dev->in_ctx)
983 		goto fail;
984 
985 	xhci_dbg(xhci, "Slot %d input ctx = 0x%llx (dma)\n", slot_id,
986 			(unsigned long long)dev->in_ctx->dma);
987 
988 	/* Initialize the cancellation list and watchdog timers for each ep */
989 	for (i = 0; i < 31; i++) {
990 		xhci_init_endpoint_timer(xhci, &dev->eps[i]);
991 		INIT_LIST_HEAD(&dev->eps[i].cancelled_td_list);
992 		INIT_LIST_HEAD(&dev->eps[i].bw_endpoint_list);
993 	}
994 
995 	/* Allocate endpoint 0 ring */
996 	dev->eps[0].ring = xhci_ring_alloc(xhci, 2, 1, TYPE_CTRL, 0, flags);
997 	if (!dev->eps[0].ring)
998 		goto fail;
999 
1000 	dev->udev = udev;
1001 
1002 	/* Point to output device context in dcbaa. */
1003 	xhci->dcbaa->dev_context_ptrs[slot_id] = cpu_to_le64(dev->out_ctx->dma);
1004 	xhci_dbg(xhci, "Set slot id %d dcbaa entry %p to 0x%llx\n",
1005 		 slot_id,
1006 		 &xhci->dcbaa->dev_context_ptrs[slot_id],
1007 		 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[slot_id]));
1008 
1009 	trace_xhci_alloc_virt_device(dev);
1010 
1011 	return 1;
1012 fail:
1013 	xhci_free_virt_device(xhci, slot_id);
1014 	return 0;
1015 }
1016 
1017 void xhci_copy_ep0_dequeue_into_input_ctx(struct xhci_hcd *xhci,
1018 		struct usb_device *udev)
1019 {
1020 	struct xhci_virt_device *virt_dev;
1021 	struct xhci_ep_ctx	*ep0_ctx;
1022 	struct xhci_ring	*ep_ring;
1023 
1024 	virt_dev = xhci->devs[udev->slot_id];
1025 	ep0_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, 0);
1026 	ep_ring = virt_dev->eps[0].ring;
1027 	/*
1028 	 * FIXME we don't keep track of the dequeue pointer very well after a
1029 	 * Set TR dequeue pointer, so we're setting the dequeue pointer of the
1030 	 * host to our enqueue pointer.  This should only be called after a
1031 	 * configured device has reset, so all control transfers should have
1032 	 * been completed or cancelled before the reset.
1033 	 */
1034 	ep0_ctx->deq = cpu_to_le64(xhci_trb_virt_to_dma(ep_ring->enq_seg,
1035 							ep_ring->enqueue)
1036 				   | ep_ring->cycle_state);
1037 }
1038 
1039 /*
1040  * The xHCI roothub may have ports of differing speeds in any order in the port
1041  * status registers.  xhci->port_array provides an array of the port speed for
1042  * each offset into the port status registers.
1043  *
1044  * The xHCI hardware wants to know the roothub port number that the USB device
1045  * is attached to (or the roothub port its ancestor hub is attached to).  All we
1046  * know is the index of that port under either the USB 2.0 or the USB 3.0
1047  * roothub, but that doesn't give us the real index into the HW port status
1048  * registers. Call xhci_find_raw_port_number() to get real index.
1049  */
1050 static u32 xhci_find_real_port_number(struct xhci_hcd *xhci,
1051 		struct usb_device *udev)
1052 {
1053 	struct usb_device *top_dev;
1054 	struct usb_hcd *hcd;
1055 
1056 	if (udev->speed >= USB_SPEED_SUPER)
1057 		hcd = xhci->shared_hcd;
1058 	else
1059 		hcd = xhci->main_hcd;
1060 
1061 	for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
1062 			top_dev = top_dev->parent)
1063 		/* Found device below root hub */;
1064 
1065 	return	xhci_find_raw_port_number(hcd, top_dev->portnum);
1066 }
1067 
1068 /* Setup an xHCI virtual device for a Set Address command */
1069 int xhci_setup_addressable_virt_dev(struct xhci_hcd *xhci, struct usb_device *udev)
1070 {
1071 	struct xhci_virt_device *dev;
1072 	struct xhci_ep_ctx	*ep0_ctx;
1073 	struct xhci_slot_ctx    *slot_ctx;
1074 	u32			port_num;
1075 	u32			max_packets;
1076 	struct usb_device *top_dev;
1077 
1078 	dev = xhci->devs[udev->slot_id];
1079 	/* Slot ID 0 is reserved */
1080 	if (udev->slot_id == 0 || !dev) {
1081 		xhci_warn(xhci, "Slot ID %d is not assigned to this device\n",
1082 				udev->slot_id);
1083 		return -EINVAL;
1084 	}
1085 	ep0_ctx = xhci_get_ep_ctx(xhci, dev->in_ctx, 0);
1086 	slot_ctx = xhci_get_slot_ctx(xhci, dev->in_ctx);
1087 
1088 	/* 3) Only the control endpoint is valid - one endpoint context */
1089 	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1) | udev->route);
1090 	switch (udev->speed) {
1091 	case USB_SPEED_SUPER_PLUS:
1092 		slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SSP);
1093 		max_packets = MAX_PACKET(512);
1094 		break;
1095 	case USB_SPEED_SUPER:
1096 		slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_SS);
1097 		max_packets = MAX_PACKET(512);
1098 		break;
1099 	case USB_SPEED_HIGH:
1100 		slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_HS);
1101 		max_packets = MAX_PACKET(64);
1102 		break;
1103 	/* USB core guesses at a 64-byte max packet first for FS devices */
1104 	case USB_SPEED_FULL:
1105 		slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_FS);
1106 		max_packets = MAX_PACKET(64);
1107 		break;
1108 	case USB_SPEED_LOW:
1109 		slot_ctx->dev_info |= cpu_to_le32(SLOT_SPEED_LS);
1110 		max_packets = MAX_PACKET(8);
1111 		break;
1112 	case USB_SPEED_WIRELESS:
1113 		xhci_dbg(xhci, "FIXME xHCI doesn't support wireless speeds\n");
1114 		return -EINVAL;
1115 		break;
1116 	default:
1117 		/* Speed was set earlier, this shouldn't happen. */
1118 		return -EINVAL;
1119 	}
1120 	/* Find the root hub port this device is under */
1121 	port_num = xhci_find_real_port_number(xhci, udev);
1122 	if (!port_num)
1123 		return -EINVAL;
1124 	slot_ctx->dev_info2 |= cpu_to_le32(ROOT_HUB_PORT(port_num));
1125 	/* Set the port number in the virtual_device to the faked port number */
1126 	for (top_dev = udev; top_dev->parent && top_dev->parent->parent;
1127 			top_dev = top_dev->parent)
1128 		/* Found device below root hub */;
1129 	dev->fake_port = top_dev->portnum;
1130 	dev->real_port = port_num;
1131 	xhci_dbg(xhci, "Set root hub portnum to %d\n", port_num);
1132 	xhci_dbg(xhci, "Set fake root hub portnum to %d\n", dev->fake_port);
1133 
1134 	/* Find the right bandwidth table that this device will be a part of.
1135 	 * If this is a full speed device attached directly to a root port (or a
1136 	 * decendent of one), it counts as a primary bandwidth domain, not a
1137 	 * secondary bandwidth domain under a TT.  An xhci_tt_info structure
1138 	 * will never be created for the HS root hub.
1139 	 */
1140 	if (!udev->tt || !udev->tt->hub->parent) {
1141 		dev->bw_table = &xhci->rh_bw[port_num - 1].bw_table;
1142 	} else {
1143 		struct xhci_root_port_bw_info *rh_bw;
1144 		struct xhci_tt_bw_info *tt_bw;
1145 
1146 		rh_bw = &xhci->rh_bw[port_num - 1];
1147 		/* Find the right TT. */
1148 		list_for_each_entry(tt_bw, &rh_bw->tts, tt_list) {
1149 			if (tt_bw->slot_id != udev->tt->hub->slot_id)
1150 				continue;
1151 
1152 			if (!dev->udev->tt->multi ||
1153 					(udev->tt->multi &&
1154 					 tt_bw->ttport == dev->udev->ttport)) {
1155 				dev->bw_table = &tt_bw->bw_table;
1156 				dev->tt_info = tt_bw;
1157 				break;
1158 			}
1159 		}
1160 		if (!dev->tt_info)
1161 			xhci_warn(xhci, "WARN: Didn't find a matching TT\n");
1162 	}
1163 
1164 	/* Is this a LS/FS device under an external HS hub? */
1165 	if (udev->tt && udev->tt->hub->parent) {
1166 		slot_ctx->tt_info = cpu_to_le32(udev->tt->hub->slot_id |
1167 						(udev->ttport << 8));
1168 		if (udev->tt->multi)
1169 			slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
1170 	}
1171 	xhci_dbg(xhci, "udev->tt = %p\n", udev->tt);
1172 	xhci_dbg(xhci, "udev->ttport = 0x%x\n", udev->ttport);
1173 
1174 	/* Step 4 - ring already allocated */
1175 	/* Step 5 */
1176 	ep0_ctx->ep_info2 = cpu_to_le32(EP_TYPE(CTRL_EP));
1177 
1178 	/* EP 0 can handle "burst" sizes of 1, so Max Burst Size field is 0 */
1179 	ep0_ctx->ep_info2 |= cpu_to_le32(MAX_BURST(0) | ERROR_COUNT(3) |
1180 					 max_packets);
1181 
1182 	ep0_ctx->deq = cpu_to_le64(dev->eps[0].ring->first_seg->dma |
1183 				   dev->eps[0].ring->cycle_state);
1184 
1185 	trace_xhci_setup_addressable_virt_device(dev);
1186 
1187 	/* Steps 7 and 8 were done in xhci_alloc_virt_device() */
1188 
1189 	return 0;
1190 }
1191 
1192 /*
1193  * Convert interval expressed as 2^(bInterval - 1) == interval into
1194  * straight exponent value 2^n == interval.
1195  *
1196  */
1197 static unsigned int xhci_parse_exponent_interval(struct usb_device *udev,
1198 		struct usb_host_endpoint *ep)
1199 {
1200 	unsigned int interval;
1201 
1202 	interval = clamp_val(ep->desc.bInterval, 1, 16) - 1;
1203 	if (interval != ep->desc.bInterval - 1)
1204 		dev_warn(&udev->dev,
1205 			 "ep %#x - rounding interval to %d %sframes\n",
1206 			 ep->desc.bEndpointAddress,
1207 			 1 << interval,
1208 			 udev->speed == USB_SPEED_FULL ? "" : "micro");
1209 
1210 	if (udev->speed == USB_SPEED_FULL) {
1211 		/*
1212 		 * Full speed isoc endpoints specify interval in frames,
1213 		 * not microframes. We are using microframes everywhere,
1214 		 * so adjust accordingly.
1215 		 */
1216 		interval += 3;	/* 1 frame = 2^3 uframes */
1217 	}
1218 
1219 	return interval;
1220 }
1221 
1222 /*
1223  * Convert bInterval expressed in microframes (in 1-255 range) to exponent of
1224  * microframes, rounded down to nearest power of 2.
1225  */
1226 static unsigned int xhci_microframes_to_exponent(struct usb_device *udev,
1227 		struct usb_host_endpoint *ep, unsigned int desc_interval,
1228 		unsigned int min_exponent, unsigned int max_exponent)
1229 {
1230 	unsigned int interval;
1231 
1232 	interval = fls(desc_interval) - 1;
1233 	interval = clamp_val(interval, min_exponent, max_exponent);
1234 	if ((1 << interval) != desc_interval)
1235 		dev_dbg(&udev->dev,
1236 			 "ep %#x - rounding interval to %d microframes, ep desc says %d microframes\n",
1237 			 ep->desc.bEndpointAddress,
1238 			 1 << interval,
1239 			 desc_interval);
1240 
1241 	return interval;
1242 }
1243 
1244 static unsigned int xhci_parse_microframe_interval(struct usb_device *udev,
1245 		struct usb_host_endpoint *ep)
1246 {
1247 	if (ep->desc.bInterval == 0)
1248 		return 0;
1249 	return xhci_microframes_to_exponent(udev, ep,
1250 			ep->desc.bInterval, 0, 15);
1251 }
1252 
1253 
1254 static unsigned int xhci_parse_frame_interval(struct usb_device *udev,
1255 		struct usb_host_endpoint *ep)
1256 {
1257 	return xhci_microframes_to_exponent(udev, ep,
1258 			ep->desc.bInterval * 8, 3, 10);
1259 }
1260 
1261 /* Return the polling or NAK interval.
1262  *
1263  * The polling interval is expressed in "microframes".  If xHCI's Interval field
1264  * is set to N, it will service the endpoint every 2^(Interval)*125us.
1265  *
1266  * The NAK interval is one NAK per 1 to 255 microframes, or no NAKs if interval
1267  * is set to 0.
1268  */
1269 static unsigned int xhci_get_endpoint_interval(struct usb_device *udev,
1270 		struct usb_host_endpoint *ep)
1271 {
1272 	unsigned int interval = 0;
1273 
1274 	switch (udev->speed) {
1275 	case USB_SPEED_HIGH:
1276 		/* Max NAK rate */
1277 		if (usb_endpoint_xfer_control(&ep->desc) ||
1278 		    usb_endpoint_xfer_bulk(&ep->desc)) {
1279 			interval = xhci_parse_microframe_interval(udev, ep);
1280 			break;
1281 		}
1282 		/* Fall through - SS and HS isoc/int have same decoding */
1283 
1284 	case USB_SPEED_SUPER_PLUS:
1285 	case USB_SPEED_SUPER:
1286 		if (usb_endpoint_xfer_int(&ep->desc) ||
1287 		    usb_endpoint_xfer_isoc(&ep->desc)) {
1288 			interval = xhci_parse_exponent_interval(udev, ep);
1289 		}
1290 		break;
1291 
1292 	case USB_SPEED_FULL:
1293 		if (usb_endpoint_xfer_isoc(&ep->desc)) {
1294 			interval = xhci_parse_exponent_interval(udev, ep);
1295 			break;
1296 		}
1297 		/*
1298 		 * Fall through for interrupt endpoint interval decoding
1299 		 * since it uses the same rules as low speed interrupt
1300 		 * endpoints.
1301 		 */
1302 		/* fall through */
1303 
1304 	case USB_SPEED_LOW:
1305 		if (usb_endpoint_xfer_int(&ep->desc) ||
1306 		    usb_endpoint_xfer_isoc(&ep->desc)) {
1307 
1308 			interval = xhci_parse_frame_interval(udev, ep);
1309 		}
1310 		break;
1311 
1312 	default:
1313 		BUG();
1314 	}
1315 	return interval;
1316 }
1317 
1318 /* The "Mult" field in the endpoint context is only set for SuperSpeed isoc eps.
1319  * High speed endpoint descriptors can define "the number of additional
1320  * transaction opportunities per microframe", but that goes in the Max Burst
1321  * endpoint context field.
1322  */
1323 static u32 xhci_get_endpoint_mult(struct usb_device *udev,
1324 		struct usb_host_endpoint *ep)
1325 {
1326 	if (udev->speed < USB_SPEED_SUPER ||
1327 			!usb_endpoint_xfer_isoc(&ep->desc))
1328 		return 0;
1329 	return ep->ss_ep_comp.bmAttributes;
1330 }
1331 
1332 static u32 xhci_get_endpoint_max_burst(struct usb_device *udev,
1333 				       struct usb_host_endpoint *ep)
1334 {
1335 	/* Super speed and Plus have max burst in ep companion desc */
1336 	if (udev->speed >= USB_SPEED_SUPER)
1337 		return ep->ss_ep_comp.bMaxBurst;
1338 
1339 	if (udev->speed == USB_SPEED_HIGH &&
1340 	    (usb_endpoint_xfer_isoc(&ep->desc) ||
1341 	     usb_endpoint_xfer_int(&ep->desc)))
1342 		return usb_endpoint_maxp_mult(&ep->desc) - 1;
1343 
1344 	return 0;
1345 }
1346 
1347 static u32 xhci_get_endpoint_type(struct usb_host_endpoint *ep)
1348 {
1349 	int in;
1350 
1351 	in = usb_endpoint_dir_in(&ep->desc);
1352 
1353 	switch (usb_endpoint_type(&ep->desc)) {
1354 	case USB_ENDPOINT_XFER_CONTROL:
1355 		return CTRL_EP;
1356 	case USB_ENDPOINT_XFER_BULK:
1357 		return in ? BULK_IN_EP : BULK_OUT_EP;
1358 	case USB_ENDPOINT_XFER_ISOC:
1359 		return in ? ISOC_IN_EP : ISOC_OUT_EP;
1360 	case USB_ENDPOINT_XFER_INT:
1361 		return in ? INT_IN_EP : INT_OUT_EP;
1362 	}
1363 	return 0;
1364 }
1365 
1366 /* Return the maximum endpoint service interval time (ESIT) payload.
1367  * Basically, this is the maxpacket size, multiplied by the burst size
1368  * and mult size.
1369  */
1370 static u32 xhci_get_max_esit_payload(struct usb_device *udev,
1371 		struct usb_host_endpoint *ep)
1372 {
1373 	int max_burst;
1374 	int max_packet;
1375 
1376 	/* Only applies for interrupt or isochronous endpoints */
1377 	if (usb_endpoint_xfer_control(&ep->desc) ||
1378 			usb_endpoint_xfer_bulk(&ep->desc))
1379 		return 0;
1380 
1381 	/* SuperSpeedPlus Isoc ep sending over 48k per esit */
1382 	if ((udev->speed >= USB_SPEED_SUPER_PLUS) &&
1383 	    USB_SS_SSP_ISOC_COMP(ep->ss_ep_comp.bmAttributes))
1384 		return le32_to_cpu(ep->ssp_isoc_ep_comp.dwBytesPerInterval);
1385 	/* SuperSpeed or SuperSpeedPlus Isoc ep with less than 48k per esit */
1386 	else if (udev->speed >= USB_SPEED_SUPER)
1387 		return le16_to_cpu(ep->ss_ep_comp.wBytesPerInterval);
1388 
1389 	max_packet = usb_endpoint_maxp(&ep->desc);
1390 	max_burst = usb_endpoint_maxp_mult(&ep->desc);
1391 	/* A 0 in max burst means 1 transfer per ESIT */
1392 	return max_packet * max_burst;
1393 }
1394 
1395 /* Set up an endpoint with one ring segment.  Do not allocate stream rings.
1396  * Drivers will have to call usb_alloc_streams() to do that.
1397  */
1398 int xhci_endpoint_init(struct xhci_hcd *xhci,
1399 		struct xhci_virt_device *virt_dev,
1400 		struct usb_device *udev,
1401 		struct usb_host_endpoint *ep,
1402 		gfp_t mem_flags)
1403 {
1404 	unsigned int ep_index;
1405 	struct xhci_ep_ctx *ep_ctx;
1406 	struct xhci_ring *ep_ring;
1407 	unsigned int max_packet;
1408 	enum xhci_ring_type ring_type;
1409 	u32 max_esit_payload;
1410 	u32 endpoint_type;
1411 	unsigned int max_burst;
1412 	unsigned int interval;
1413 	unsigned int mult;
1414 	unsigned int avg_trb_len;
1415 	unsigned int err_count = 0;
1416 
1417 	ep_index = xhci_get_endpoint_index(&ep->desc);
1418 	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1419 
1420 	endpoint_type = xhci_get_endpoint_type(ep);
1421 	if (!endpoint_type)
1422 		return -EINVAL;
1423 
1424 	ring_type = usb_endpoint_type(&ep->desc);
1425 
1426 	/*
1427 	 * Get values to fill the endpoint context, mostly from ep descriptor.
1428 	 * The average TRB buffer lengt for bulk endpoints is unclear as we
1429 	 * have no clue on scatter gather list entry size. For Isoc and Int,
1430 	 * set it to max available. See xHCI 1.1 spec 4.14.1.1 for details.
1431 	 */
1432 	max_esit_payload = xhci_get_max_esit_payload(udev, ep);
1433 	interval = xhci_get_endpoint_interval(udev, ep);
1434 
1435 	/* Periodic endpoint bInterval limit quirk */
1436 	if (usb_endpoint_xfer_int(&ep->desc) ||
1437 	    usb_endpoint_xfer_isoc(&ep->desc)) {
1438 		if ((xhci->quirks & XHCI_LIMIT_ENDPOINT_INTERVAL_7) &&
1439 		    udev->speed >= USB_SPEED_HIGH &&
1440 		    interval >= 7) {
1441 			interval = 6;
1442 		}
1443 	}
1444 
1445 	mult = xhci_get_endpoint_mult(udev, ep);
1446 	max_packet = usb_endpoint_maxp(&ep->desc);
1447 	max_burst = xhci_get_endpoint_max_burst(udev, ep);
1448 	avg_trb_len = max_esit_payload;
1449 
1450 	/* FIXME dig Mult and streams info out of ep companion desc */
1451 
1452 	/* Allow 3 retries for everything but isoc, set CErr = 3 */
1453 	if (!usb_endpoint_xfer_isoc(&ep->desc))
1454 		err_count = 3;
1455 	/* Some devices get this wrong */
1456 	if (usb_endpoint_xfer_bulk(&ep->desc) && udev->speed == USB_SPEED_HIGH)
1457 		max_packet = 512;
1458 	/* xHCI 1.0 and 1.1 indicates that ctrl ep avg TRB Length should be 8 */
1459 	if (usb_endpoint_xfer_control(&ep->desc) && xhci->hci_version >= 0x100)
1460 		avg_trb_len = 8;
1461 	/* xhci 1.1 with LEC support doesn't use mult field, use RsvdZ */
1462 	if ((xhci->hci_version > 0x100) && HCC2_LEC(xhci->hcc_params2))
1463 		mult = 0;
1464 
1465 	/* Set up the endpoint ring */
1466 	virt_dev->eps[ep_index].new_ring =
1467 		xhci_ring_alloc(xhci, 2, 1, ring_type, max_packet, mem_flags);
1468 	if (!virt_dev->eps[ep_index].new_ring)
1469 		return -ENOMEM;
1470 
1471 	virt_dev->eps[ep_index].skip = false;
1472 	ep_ring = virt_dev->eps[ep_index].new_ring;
1473 
1474 	/* Fill the endpoint context */
1475 	ep_ctx->ep_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_HI(max_esit_payload) |
1476 				      EP_INTERVAL(interval) |
1477 				      EP_MULT(mult));
1478 	ep_ctx->ep_info2 = cpu_to_le32(EP_TYPE(endpoint_type) |
1479 				       MAX_PACKET(max_packet) |
1480 				       MAX_BURST(max_burst) |
1481 				       ERROR_COUNT(err_count));
1482 	ep_ctx->deq = cpu_to_le64(ep_ring->first_seg->dma |
1483 				  ep_ring->cycle_state);
1484 
1485 	ep_ctx->tx_info = cpu_to_le32(EP_MAX_ESIT_PAYLOAD_LO(max_esit_payload) |
1486 				      EP_AVG_TRB_LENGTH(avg_trb_len));
1487 
1488 	return 0;
1489 }
1490 
1491 void xhci_endpoint_zero(struct xhci_hcd *xhci,
1492 		struct xhci_virt_device *virt_dev,
1493 		struct usb_host_endpoint *ep)
1494 {
1495 	unsigned int ep_index;
1496 	struct xhci_ep_ctx *ep_ctx;
1497 
1498 	ep_index = xhci_get_endpoint_index(&ep->desc);
1499 	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
1500 
1501 	ep_ctx->ep_info = 0;
1502 	ep_ctx->ep_info2 = 0;
1503 	ep_ctx->deq = 0;
1504 	ep_ctx->tx_info = 0;
1505 	/* Don't free the endpoint ring until the set interface or configuration
1506 	 * request succeeds.
1507 	 */
1508 }
1509 
1510 void xhci_clear_endpoint_bw_info(struct xhci_bw_info *bw_info)
1511 {
1512 	bw_info->ep_interval = 0;
1513 	bw_info->mult = 0;
1514 	bw_info->num_packets = 0;
1515 	bw_info->max_packet_size = 0;
1516 	bw_info->type = 0;
1517 	bw_info->max_esit_payload = 0;
1518 }
1519 
1520 void xhci_update_bw_info(struct xhci_hcd *xhci,
1521 		struct xhci_container_ctx *in_ctx,
1522 		struct xhci_input_control_ctx *ctrl_ctx,
1523 		struct xhci_virt_device *virt_dev)
1524 {
1525 	struct xhci_bw_info *bw_info;
1526 	struct xhci_ep_ctx *ep_ctx;
1527 	unsigned int ep_type;
1528 	int i;
1529 
1530 	for (i = 1; i < 31; i++) {
1531 		bw_info = &virt_dev->eps[i].bw_info;
1532 
1533 		/* We can't tell what endpoint type is being dropped, but
1534 		 * unconditionally clearing the bandwidth info for non-periodic
1535 		 * endpoints should be harmless because the info will never be
1536 		 * set in the first place.
1537 		 */
1538 		if (!EP_IS_ADDED(ctrl_ctx, i) && EP_IS_DROPPED(ctrl_ctx, i)) {
1539 			/* Dropped endpoint */
1540 			xhci_clear_endpoint_bw_info(bw_info);
1541 			continue;
1542 		}
1543 
1544 		if (EP_IS_ADDED(ctrl_ctx, i)) {
1545 			ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, i);
1546 			ep_type = CTX_TO_EP_TYPE(le32_to_cpu(ep_ctx->ep_info2));
1547 
1548 			/* Ignore non-periodic endpoints */
1549 			if (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
1550 					ep_type != ISOC_IN_EP &&
1551 					ep_type != INT_IN_EP)
1552 				continue;
1553 
1554 			/* Added or changed endpoint */
1555 			bw_info->ep_interval = CTX_TO_EP_INTERVAL(
1556 					le32_to_cpu(ep_ctx->ep_info));
1557 			/* Number of packets and mult are zero-based in the
1558 			 * input context, but we want one-based for the
1559 			 * interval table.
1560 			 */
1561 			bw_info->mult = CTX_TO_EP_MULT(
1562 					le32_to_cpu(ep_ctx->ep_info)) + 1;
1563 			bw_info->num_packets = CTX_TO_MAX_BURST(
1564 					le32_to_cpu(ep_ctx->ep_info2)) + 1;
1565 			bw_info->max_packet_size = MAX_PACKET_DECODED(
1566 					le32_to_cpu(ep_ctx->ep_info2));
1567 			bw_info->type = ep_type;
1568 			bw_info->max_esit_payload = CTX_TO_MAX_ESIT_PAYLOAD(
1569 					le32_to_cpu(ep_ctx->tx_info));
1570 		}
1571 	}
1572 }
1573 
1574 /* Copy output xhci_ep_ctx to the input xhci_ep_ctx copy.
1575  * Useful when you want to change one particular aspect of the endpoint and then
1576  * issue a configure endpoint command.
1577  */
1578 void xhci_endpoint_copy(struct xhci_hcd *xhci,
1579 		struct xhci_container_ctx *in_ctx,
1580 		struct xhci_container_ctx *out_ctx,
1581 		unsigned int ep_index)
1582 {
1583 	struct xhci_ep_ctx *out_ep_ctx;
1584 	struct xhci_ep_ctx *in_ep_ctx;
1585 
1586 	out_ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
1587 	in_ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
1588 
1589 	in_ep_ctx->ep_info = out_ep_ctx->ep_info;
1590 	in_ep_ctx->ep_info2 = out_ep_ctx->ep_info2;
1591 	in_ep_ctx->deq = out_ep_ctx->deq;
1592 	in_ep_ctx->tx_info = out_ep_ctx->tx_info;
1593 }
1594 
1595 /* Copy output xhci_slot_ctx to the input xhci_slot_ctx.
1596  * Useful when you want to change one particular aspect of the endpoint and then
1597  * issue a configure endpoint command.  Only the context entries field matters,
1598  * but we'll copy the whole thing anyway.
1599  */
1600 void xhci_slot_copy(struct xhci_hcd *xhci,
1601 		struct xhci_container_ctx *in_ctx,
1602 		struct xhci_container_ctx *out_ctx)
1603 {
1604 	struct xhci_slot_ctx *in_slot_ctx;
1605 	struct xhci_slot_ctx *out_slot_ctx;
1606 
1607 	in_slot_ctx = xhci_get_slot_ctx(xhci, in_ctx);
1608 	out_slot_ctx = xhci_get_slot_ctx(xhci, out_ctx);
1609 
1610 	in_slot_ctx->dev_info = out_slot_ctx->dev_info;
1611 	in_slot_ctx->dev_info2 = out_slot_ctx->dev_info2;
1612 	in_slot_ctx->tt_info = out_slot_ctx->tt_info;
1613 	in_slot_ctx->dev_state = out_slot_ctx->dev_state;
1614 }
1615 
1616 /* Set up the scratchpad buffer array and scratchpad buffers, if needed. */
1617 static int scratchpad_alloc(struct xhci_hcd *xhci, gfp_t flags)
1618 {
1619 	int i;
1620 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1621 	int num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
1622 
1623 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
1624 			"Allocating %d scratchpad buffers", num_sp);
1625 
1626 	if (!num_sp)
1627 		return 0;
1628 
1629 	xhci->scratchpad = kzalloc(sizeof(*xhci->scratchpad), flags);
1630 	if (!xhci->scratchpad)
1631 		goto fail_sp;
1632 
1633 	xhci->scratchpad->sp_array = dma_alloc_coherent(dev,
1634 				     num_sp * sizeof(u64),
1635 				     &xhci->scratchpad->sp_dma, flags);
1636 	if (!xhci->scratchpad->sp_array)
1637 		goto fail_sp2;
1638 
1639 	xhci->scratchpad->sp_buffers = kzalloc(sizeof(void *) * num_sp, flags);
1640 	if (!xhci->scratchpad->sp_buffers)
1641 		goto fail_sp3;
1642 
1643 	xhci->dcbaa->dev_context_ptrs[0] = cpu_to_le64(xhci->scratchpad->sp_dma);
1644 	for (i = 0; i < num_sp; i++) {
1645 		dma_addr_t dma;
1646 		void *buf = dma_zalloc_coherent(dev, xhci->page_size, &dma,
1647 				flags);
1648 		if (!buf)
1649 			goto fail_sp4;
1650 
1651 		xhci->scratchpad->sp_array[i] = dma;
1652 		xhci->scratchpad->sp_buffers[i] = buf;
1653 	}
1654 
1655 	return 0;
1656 
1657  fail_sp4:
1658 	for (i = i - 1; i >= 0; i--) {
1659 		dma_free_coherent(dev, xhci->page_size,
1660 				    xhci->scratchpad->sp_buffers[i],
1661 				    xhci->scratchpad->sp_array[i]);
1662 	}
1663 
1664 	kfree(xhci->scratchpad->sp_buffers);
1665 
1666  fail_sp3:
1667 	dma_free_coherent(dev, num_sp * sizeof(u64),
1668 			    xhci->scratchpad->sp_array,
1669 			    xhci->scratchpad->sp_dma);
1670 
1671  fail_sp2:
1672 	kfree(xhci->scratchpad);
1673 	xhci->scratchpad = NULL;
1674 
1675  fail_sp:
1676 	return -ENOMEM;
1677 }
1678 
1679 static void scratchpad_free(struct xhci_hcd *xhci)
1680 {
1681 	int num_sp;
1682 	int i;
1683 	struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
1684 
1685 	if (!xhci->scratchpad)
1686 		return;
1687 
1688 	num_sp = HCS_MAX_SCRATCHPAD(xhci->hcs_params2);
1689 
1690 	for (i = 0; i < num_sp; i++) {
1691 		dma_free_coherent(dev, xhci->page_size,
1692 				    xhci->scratchpad->sp_buffers[i],
1693 				    xhci->scratchpad->sp_array[i]);
1694 	}
1695 	kfree(xhci->scratchpad->sp_buffers);
1696 	dma_free_coherent(dev, num_sp * sizeof(u64),
1697 			    xhci->scratchpad->sp_array,
1698 			    xhci->scratchpad->sp_dma);
1699 	kfree(xhci->scratchpad);
1700 	xhci->scratchpad = NULL;
1701 }
1702 
1703 struct xhci_command *xhci_alloc_command(struct xhci_hcd *xhci,
1704 		bool allocate_in_ctx, bool allocate_completion,
1705 		gfp_t mem_flags)
1706 {
1707 	struct xhci_command *command;
1708 
1709 	command = kzalloc(sizeof(*command), mem_flags);
1710 	if (!command)
1711 		return NULL;
1712 
1713 	if (allocate_in_ctx) {
1714 		command->in_ctx =
1715 			xhci_alloc_container_ctx(xhci, XHCI_CTX_TYPE_INPUT,
1716 					mem_flags);
1717 		if (!command->in_ctx) {
1718 			kfree(command);
1719 			return NULL;
1720 		}
1721 	}
1722 
1723 	if (allocate_completion) {
1724 		command->completion =
1725 			kzalloc(sizeof(struct completion), mem_flags);
1726 		if (!command->completion) {
1727 			xhci_free_container_ctx(xhci, command->in_ctx);
1728 			kfree(command);
1729 			return NULL;
1730 		}
1731 		init_completion(command->completion);
1732 	}
1733 
1734 	command->status = 0;
1735 	INIT_LIST_HEAD(&command->cmd_list);
1736 	return command;
1737 }
1738 
1739 void xhci_urb_free_priv(struct urb_priv *urb_priv)
1740 {
1741 	kfree(urb_priv);
1742 }
1743 
1744 void xhci_free_command(struct xhci_hcd *xhci,
1745 		struct xhci_command *command)
1746 {
1747 	xhci_free_container_ctx(xhci,
1748 			command->in_ctx);
1749 	kfree(command->completion);
1750 	kfree(command);
1751 }
1752 
1753 void xhci_mem_cleanup(struct xhci_hcd *xhci)
1754 {
1755 	struct device	*dev = xhci_to_hcd(xhci)->self.sysdev;
1756 	int size;
1757 	int i, j, num_ports;
1758 
1759 	cancel_delayed_work_sync(&xhci->cmd_timer);
1760 
1761 	/* Free the Event Ring Segment Table and the actual Event Ring */
1762 	size = sizeof(struct xhci_erst_entry)*(xhci->erst.num_entries);
1763 	if (xhci->erst.entries)
1764 		dma_free_coherent(dev, size,
1765 				xhci->erst.entries, xhci->erst.erst_dma_addr);
1766 	xhci->erst.entries = NULL;
1767 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed ERST");
1768 	if (xhci->event_ring)
1769 		xhci_ring_free(xhci, xhci->event_ring);
1770 	xhci->event_ring = NULL;
1771 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed event ring");
1772 
1773 	if (xhci->lpm_command)
1774 		xhci_free_command(xhci, xhci->lpm_command);
1775 	xhci->lpm_command = NULL;
1776 	if (xhci->cmd_ring)
1777 		xhci_ring_free(xhci, xhci->cmd_ring);
1778 	xhci->cmd_ring = NULL;
1779 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed command ring");
1780 	xhci_cleanup_command_queue(xhci);
1781 
1782 	num_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1783 	for (i = 0; i < num_ports && xhci->rh_bw; i++) {
1784 		struct xhci_interval_bw_table *bwt = &xhci->rh_bw[i].bw_table;
1785 		for (j = 0; j < XHCI_MAX_INTERVAL; j++) {
1786 			struct list_head *ep = &bwt->interval_bw[j].endpoints;
1787 			while (!list_empty(ep))
1788 				list_del_init(ep->next);
1789 		}
1790 	}
1791 
1792 	for (i = HCS_MAX_SLOTS(xhci->hcs_params1); i > 0; i--)
1793 		xhci_free_virt_devices_depth_first(xhci, i);
1794 
1795 	dma_pool_destroy(xhci->segment_pool);
1796 	xhci->segment_pool = NULL;
1797 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed segment pool");
1798 
1799 	dma_pool_destroy(xhci->device_pool);
1800 	xhci->device_pool = NULL;
1801 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Freed device context pool");
1802 
1803 	dma_pool_destroy(xhci->small_streams_pool);
1804 	xhci->small_streams_pool = NULL;
1805 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
1806 			"Freed small stream array pool");
1807 
1808 	dma_pool_destroy(xhci->medium_streams_pool);
1809 	xhci->medium_streams_pool = NULL;
1810 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
1811 			"Freed medium stream array pool");
1812 
1813 	if (xhci->dcbaa)
1814 		dma_free_coherent(dev, sizeof(*xhci->dcbaa),
1815 				xhci->dcbaa, xhci->dcbaa->dma);
1816 	xhci->dcbaa = NULL;
1817 
1818 	scratchpad_free(xhci);
1819 
1820 	if (!xhci->rh_bw)
1821 		goto no_bw;
1822 
1823 	for (i = 0; i < num_ports; i++) {
1824 		struct xhci_tt_bw_info *tt, *n;
1825 		list_for_each_entry_safe(tt, n, &xhci->rh_bw[i].tts, tt_list) {
1826 			list_del(&tt->tt_list);
1827 			kfree(tt);
1828 		}
1829 	}
1830 
1831 no_bw:
1832 	xhci->cmd_ring_reserved_trbs = 0;
1833 	xhci->num_usb2_ports = 0;
1834 	xhci->num_usb3_ports = 0;
1835 	xhci->num_active_eps = 0;
1836 	kfree(xhci->usb2_ports);
1837 	kfree(xhci->usb3_ports);
1838 	kfree(xhci->port_array);
1839 	kfree(xhci->rh_bw);
1840 	kfree(xhci->ext_caps);
1841 
1842 	xhci->usb2_ports = NULL;
1843 	xhci->usb3_ports = NULL;
1844 	xhci->port_array = NULL;
1845 	xhci->rh_bw = NULL;
1846 	xhci->ext_caps = NULL;
1847 
1848 	xhci->page_size = 0;
1849 	xhci->page_shift = 0;
1850 	xhci->bus_state[0].bus_suspended = 0;
1851 	xhci->bus_state[1].bus_suspended = 0;
1852 }
1853 
1854 static int xhci_test_trb_in_td(struct xhci_hcd *xhci,
1855 		struct xhci_segment *input_seg,
1856 		union xhci_trb *start_trb,
1857 		union xhci_trb *end_trb,
1858 		dma_addr_t input_dma,
1859 		struct xhci_segment *result_seg,
1860 		char *test_name, int test_number)
1861 {
1862 	unsigned long long start_dma;
1863 	unsigned long long end_dma;
1864 	struct xhci_segment *seg;
1865 
1866 	start_dma = xhci_trb_virt_to_dma(input_seg, start_trb);
1867 	end_dma = xhci_trb_virt_to_dma(input_seg, end_trb);
1868 
1869 	seg = trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma, false);
1870 	if (seg != result_seg) {
1871 		xhci_warn(xhci, "WARN: %s TRB math test %d failed!\n",
1872 				test_name, test_number);
1873 		xhci_warn(xhci, "Tested TRB math w/ seg %p and "
1874 				"input DMA 0x%llx\n",
1875 				input_seg,
1876 				(unsigned long long) input_dma);
1877 		xhci_warn(xhci, "starting TRB %p (0x%llx DMA), "
1878 				"ending TRB %p (0x%llx DMA)\n",
1879 				start_trb, start_dma,
1880 				end_trb, end_dma);
1881 		xhci_warn(xhci, "Expected seg %p, got seg %p\n",
1882 				result_seg, seg);
1883 		trb_in_td(xhci, input_seg, start_trb, end_trb, input_dma,
1884 			  true);
1885 		return -1;
1886 	}
1887 	return 0;
1888 }
1889 
1890 /* TRB math checks for xhci_trb_in_td(), using the command and event rings. */
1891 static int xhci_check_trb_in_td_math(struct xhci_hcd *xhci)
1892 {
1893 	struct {
1894 		dma_addr_t		input_dma;
1895 		struct xhci_segment	*result_seg;
1896 	} simple_test_vector [] = {
1897 		/* A zeroed DMA field should fail */
1898 		{ 0, NULL },
1899 		/* One TRB before the ring start should fail */
1900 		{ xhci->event_ring->first_seg->dma - 16, NULL },
1901 		/* One byte before the ring start should fail */
1902 		{ xhci->event_ring->first_seg->dma - 1, NULL },
1903 		/* Starting TRB should succeed */
1904 		{ xhci->event_ring->first_seg->dma, xhci->event_ring->first_seg },
1905 		/* Ending TRB should succeed */
1906 		{ xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16,
1907 			xhci->event_ring->first_seg },
1908 		/* One byte after the ring end should fail */
1909 		{ xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 1)*16 + 1, NULL },
1910 		/* One TRB after the ring end should fail */
1911 		{ xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT)*16, NULL },
1912 		/* An address of all ones should fail */
1913 		{ (dma_addr_t) (~0), NULL },
1914 	};
1915 	struct {
1916 		struct xhci_segment	*input_seg;
1917 		union xhci_trb		*start_trb;
1918 		union xhci_trb		*end_trb;
1919 		dma_addr_t		input_dma;
1920 		struct xhci_segment	*result_seg;
1921 	} complex_test_vector [] = {
1922 		/* Test feeding a valid DMA address from a different ring */
1923 		{	.input_seg = xhci->event_ring->first_seg,
1924 			.start_trb = xhci->event_ring->first_seg->trbs,
1925 			.end_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
1926 			.input_dma = xhci->cmd_ring->first_seg->dma,
1927 			.result_seg = NULL,
1928 		},
1929 		/* Test feeding a valid end TRB from a different ring */
1930 		{	.input_seg = xhci->event_ring->first_seg,
1931 			.start_trb = xhci->event_ring->first_seg->trbs,
1932 			.end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
1933 			.input_dma = xhci->cmd_ring->first_seg->dma,
1934 			.result_seg = NULL,
1935 		},
1936 		/* Test feeding a valid start and end TRB from a different ring */
1937 		{	.input_seg = xhci->event_ring->first_seg,
1938 			.start_trb = xhci->cmd_ring->first_seg->trbs,
1939 			.end_trb = &xhci->cmd_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
1940 			.input_dma = xhci->cmd_ring->first_seg->dma,
1941 			.result_seg = NULL,
1942 		},
1943 		/* TRB in this ring, but after this TD */
1944 		{	.input_seg = xhci->event_ring->first_seg,
1945 			.start_trb = &xhci->event_ring->first_seg->trbs[0],
1946 			.end_trb = &xhci->event_ring->first_seg->trbs[3],
1947 			.input_dma = xhci->event_ring->first_seg->dma + 4*16,
1948 			.result_seg = NULL,
1949 		},
1950 		/* TRB in this ring, but before this TD */
1951 		{	.input_seg = xhci->event_ring->first_seg,
1952 			.start_trb = &xhci->event_ring->first_seg->trbs[3],
1953 			.end_trb = &xhci->event_ring->first_seg->trbs[6],
1954 			.input_dma = xhci->event_ring->first_seg->dma + 2*16,
1955 			.result_seg = NULL,
1956 		},
1957 		/* TRB in this ring, but after this wrapped TD */
1958 		{	.input_seg = xhci->event_ring->first_seg,
1959 			.start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
1960 			.end_trb = &xhci->event_ring->first_seg->trbs[1],
1961 			.input_dma = xhci->event_ring->first_seg->dma + 2*16,
1962 			.result_seg = NULL,
1963 		},
1964 		/* TRB in this ring, but before this wrapped TD */
1965 		{	.input_seg = xhci->event_ring->first_seg,
1966 			.start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
1967 			.end_trb = &xhci->event_ring->first_seg->trbs[1],
1968 			.input_dma = xhci->event_ring->first_seg->dma + (TRBS_PER_SEGMENT - 4)*16,
1969 			.result_seg = NULL,
1970 		},
1971 		/* TRB not in this ring, and we have a wrapped TD */
1972 		{	.input_seg = xhci->event_ring->first_seg,
1973 			.start_trb = &xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 3],
1974 			.end_trb = &xhci->event_ring->first_seg->trbs[1],
1975 			.input_dma = xhci->cmd_ring->first_seg->dma + 2*16,
1976 			.result_seg = NULL,
1977 		},
1978 	};
1979 
1980 	unsigned int num_tests;
1981 	int i, ret;
1982 
1983 	num_tests = ARRAY_SIZE(simple_test_vector);
1984 	for (i = 0; i < num_tests; i++) {
1985 		ret = xhci_test_trb_in_td(xhci,
1986 				xhci->event_ring->first_seg,
1987 				xhci->event_ring->first_seg->trbs,
1988 				&xhci->event_ring->first_seg->trbs[TRBS_PER_SEGMENT - 1],
1989 				simple_test_vector[i].input_dma,
1990 				simple_test_vector[i].result_seg,
1991 				"Simple", i);
1992 		if (ret < 0)
1993 			return ret;
1994 	}
1995 
1996 	num_tests = ARRAY_SIZE(complex_test_vector);
1997 	for (i = 0; i < num_tests; i++) {
1998 		ret = xhci_test_trb_in_td(xhci,
1999 				complex_test_vector[i].input_seg,
2000 				complex_test_vector[i].start_trb,
2001 				complex_test_vector[i].end_trb,
2002 				complex_test_vector[i].input_dma,
2003 				complex_test_vector[i].result_seg,
2004 				"Complex", i);
2005 		if (ret < 0)
2006 			return ret;
2007 	}
2008 	xhci_dbg(xhci, "TRB math tests passed.\n");
2009 	return 0;
2010 }
2011 
2012 static void xhci_set_hc_event_deq(struct xhci_hcd *xhci)
2013 {
2014 	u64 temp;
2015 	dma_addr_t deq;
2016 
2017 	deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
2018 			xhci->event_ring->dequeue);
2019 	if (deq == 0 && !in_interrupt())
2020 		xhci_warn(xhci, "WARN something wrong with SW event ring "
2021 				"dequeue ptr.\n");
2022 	/* Update HC event ring dequeue pointer */
2023 	temp = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2024 	temp &= ERST_PTR_MASK;
2025 	/* Don't clear the EHB bit (which is RW1C) because
2026 	 * there might be more events to service.
2027 	 */
2028 	temp &= ~ERST_EHB;
2029 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2030 			"// Write event ring dequeue pointer, "
2031 			"preserving EHB bit");
2032 	xhci_write_64(xhci, ((u64) deq & (u64) ~ERST_PTR_MASK) | temp,
2033 			&xhci->ir_set->erst_dequeue);
2034 }
2035 
2036 static void xhci_add_in_port(struct xhci_hcd *xhci, unsigned int num_ports,
2037 		__le32 __iomem *addr, int max_caps)
2038 {
2039 	u32 temp, port_offset, port_count;
2040 	int i;
2041 	u8 major_revision, minor_revision;
2042 	struct xhci_hub *rhub;
2043 
2044 	temp = readl(addr);
2045 	major_revision = XHCI_EXT_PORT_MAJOR(temp);
2046 	minor_revision = XHCI_EXT_PORT_MINOR(temp);
2047 
2048 	if (major_revision == 0x03) {
2049 		rhub = &xhci->usb3_rhub;
2050 	} else if (major_revision <= 0x02) {
2051 		rhub = &xhci->usb2_rhub;
2052 	} else {
2053 		xhci_warn(xhci, "Ignoring unknown port speed, "
2054 				"Ext Cap %p, revision = 0x%x\n",
2055 				addr, major_revision);
2056 		/* Ignoring port protocol we can't understand. FIXME */
2057 		return;
2058 	}
2059 	rhub->maj_rev = XHCI_EXT_PORT_MAJOR(temp);
2060 
2061 	if (rhub->min_rev < minor_revision)
2062 		rhub->min_rev = minor_revision;
2063 
2064 	/* Port offset and count in the third dword, see section 7.2 */
2065 	temp = readl(addr + 2);
2066 	port_offset = XHCI_EXT_PORT_OFF(temp);
2067 	port_count = XHCI_EXT_PORT_COUNT(temp);
2068 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2069 			"Ext Cap %p, port offset = %u, "
2070 			"count = %u, revision = 0x%x",
2071 			addr, port_offset, port_count, major_revision);
2072 	/* Port count includes the current port offset */
2073 	if (port_offset == 0 || (port_offset + port_count - 1) > num_ports)
2074 		/* WTF? "Valid values are ‘1’ to MaxPorts" */
2075 		return;
2076 
2077 	rhub->psi_count = XHCI_EXT_PORT_PSIC(temp);
2078 	if (rhub->psi_count) {
2079 		rhub->psi = kcalloc(rhub->psi_count, sizeof(*rhub->psi),
2080 				    GFP_KERNEL);
2081 		if (!rhub->psi)
2082 			rhub->psi_count = 0;
2083 
2084 		rhub->psi_uid_count++;
2085 		for (i = 0; i < rhub->psi_count; i++) {
2086 			rhub->psi[i] = readl(addr + 4 + i);
2087 
2088 			/* count unique ID values, two consecutive entries can
2089 			 * have the same ID if link is assymetric
2090 			 */
2091 			if (i && (XHCI_EXT_PORT_PSIV(rhub->psi[i]) !=
2092 				  XHCI_EXT_PORT_PSIV(rhub->psi[i - 1])))
2093 				rhub->psi_uid_count++;
2094 
2095 			xhci_dbg(xhci, "PSIV:%d PSIE:%d PLT:%d PFD:%d LP:%d PSIM:%d\n",
2096 				  XHCI_EXT_PORT_PSIV(rhub->psi[i]),
2097 				  XHCI_EXT_PORT_PSIE(rhub->psi[i]),
2098 				  XHCI_EXT_PORT_PLT(rhub->psi[i]),
2099 				  XHCI_EXT_PORT_PFD(rhub->psi[i]),
2100 				  XHCI_EXT_PORT_LP(rhub->psi[i]),
2101 				  XHCI_EXT_PORT_PSIM(rhub->psi[i]));
2102 		}
2103 	}
2104 	/* cache usb2 port capabilities */
2105 	if (major_revision < 0x03 && xhci->num_ext_caps < max_caps)
2106 		xhci->ext_caps[xhci->num_ext_caps++] = temp;
2107 
2108 	/* Check the host's USB2 LPM capability */
2109 	if ((xhci->hci_version == 0x96) && (major_revision != 0x03) &&
2110 			(temp & XHCI_L1C)) {
2111 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2112 				"xHCI 0.96: support USB2 software lpm");
2113 		xhci->sw_lpm_support = 1;
2114 	}
2115 
2116 	if ((xhci->hci_version >= 0x100) && (major_revision != 0x03)) {
2117 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2118 				"xHCI 1.0: support USB2 software lpm");
2119 		xhci->sw_lpm_support = 1;
2120 		if (temp & XHCI_HLC) {
2121 			xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2122 					"xHCI 1.0: support USB2 hardware lpm");
2123 			xhci->hw_lpm_support = 1;
2124 		}
2125 	}
2126 
2127 	port_offset--;
2128 	for (i = port_offset; i < (port_offset + port_count); i++) {
2129 		/* Duplicate entry.  Ignore the port if the revisions differ. */
2130 		if (xhci->port_array[i] != 0) {
2131 			xhci_warn(xhci, "Duplicate port entry, Ext Cap %p,"
2132 					" port %u\n", addr, i);
2133 			xhci_warn(xhci, "Port was marked as USB %u, "
2134 					"duplicated as USB %u\n",
2135 					xhci->port_array[i], major_revision);
2136 			/* Only adjust the roothub port counts if we haven't
2137 			 * found a similar duplicate.
2138 			 */
2139 			if (xhci->port_array[i] != major_revision &&
2140 				xhci->port_array[i] != DUPLICATE_ENTRY) {
2141 				if (xhci->port_array[i] == 0x03)
2142 					xhci->num_usb3_ports--;
2143 				else
2144 					xhci->num_usb2_ports--;
2145 				xhci->port_array[i] = DUPLICATE_ENTRY;
2146 			}
2147 			/* FIXME: Should we disable the port? */
2148 			continue;
2149 		}
2150 		xhci->port_array[i] = major_revision;
2151 		if (major_revision == 0x03)
2152 			xhci->num_usb3_ports++;
2153 		else
2154 			xhci->num_usb2_ports++;
2155 	}
2156 	/* FIXME: Should we disable ports not in the Extended Capabilities? */
2157 }
2158 
2159 /*
2160  * Scan the Extended Capabilities for the "Supported Protocol Capabilities" that
2161  * specify what speeds each port is supposed to be.  We can't count on the port
2162  * speed bits in the PORTSC register being correct until a device is connected,
2163  * but we need to set up the two fake roothubs with the correct number of USB
2164  * 3.0 and USB 2.0 ports at host controller initialization time.
2165  */
2166 static int xhci_setup_port_arrays(struct xhci_hcd *xhci, gfp_t flags)
2167 {
2168 	void __iomem *base;
2169 	u32 offset;
2170 	unsigned int num_ports;
2171 	int i, j, port_index;
2172 	int cap_count = 0;
2173 	u32 cap_start;
2174 
2175 	num_ports = HCS_MAX_PORTS(xhci->hcs_params1);
2176 	xhci->port_array = kzalloc(sizeof(*xhci->port_array)*num_ports, flags);
2177 	if (!xhci->port_array)
2178 		return -ENOMEM;
2179 
2180 	xhci->rh_bw = kzalloc(sizeof(*xhci->rh_bw)*num_ports, flags);
2181 	if (!xhci->rh_bw)
2182 		return -ENOMEM;
2183 	for (i = 0; i < num_ports; i++) {
2184 		struct xhci_interval_bw_table *bw_table;
2185 
2186 		INIT_LIST_HEAD(&xhci->rh_bw[i].tts);
2187 		bw_table = &xhci->rh_bw[i].bw_table;
2188 		for (j = 0; j < XHCI_MAX_INTERVAL; j++)
2189 			INIT_LIST_HEAD(&bw_table->interval_bw[j].endpoints);
2190 	}
2191 	base = &xhci->cap_regs->hc_capbase;
2192 
2193 	cap_start = xhci_find_next_ext_cap(base, 0, XHCI_EXT_CAPS_PROTOCOL);
2194 	if (!cap_start) {
2195 		xhci_err(xhci, "No Extended Capability registers, unable to set up roothub\n");
2196 		return -ENODEV;
2197 	}
2198 
2199 	offset = cap_start;
2200 	/* count extended protocol capability entries for later caching */
2201 	while (offset) {
2202 		cap_count++;
2203 		offset = xhci_find_next_ext_cap(base, offset,
2204 						      XHCI_EXT_CAPS_PROTOCOL);
2205 	}
2206 
2207 	xhci->ext_caps = kzalloc(sizeof(*xhci->ext_caps) * cap_count, flags);
2208 	if (!xhci->ext_caps)
2209 		return -ENOMEM;
2210 
2211 	offset = cap_start;
2212 
2213 	while (offset) {
2214 		xhci_add_in_port(xhci, num_ports, base + offset, cap_count);
2215 		if (xhci->num_usb2_ports + xhci->num_usb3_ports == num_ports)
2216 			break;
2217 		offset = xhci_find_next_ext_cap(base, offset,
2218 						XHCI_EXT_CAPS_PROTOCOL);
2219 	}
2220 
2221 	if (xhci->num_usb2_ports == 0 && xhci->num_usb3_ports == 0) {
2222 		xhci_warn(xhci, "No ports on the roothubs?\n");
2223 		return -ENODEV;
2224 	}
2225 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2226 			"Found %u USB 2.0 ports and %u USB 3.0 ports.",
2227 			xhci->num_usb2_ports, xhci->num_usb3_ports);
2228 
2229 	/* Place limits on the number of roothub ports so that the hub
2230 	 * descriptors aren't longer than the USB core will allocate.
2231 	 */
2232 	if (xhci->num_usb3_ports > USB_SS_MAXPORTS) {
2233 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2234 				"Limiting USB 3.0 roothub ports to %u.",
2235 				USB_SS_MAXPORTS);
2236 		xhci->num_usb3_ports = USB_SS_MAXPORTS;
2237 	}
2238 	if (xhci->num_usb2_ports > USB_MAXCHILDREN) {
2239 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2240 				"Limiting USB 2.0 roothub ports to %u.",
2241 				USB_MAXCHILDREN);
2242 		xhci->num_usb2_ports = USB_MAXCHILDREN;
2243 	}
2244 
2245 	/*
2246 	 * Note we could have all USB 3.0 ports, or all USB 2.0 ports.
2247 	 * Not sure how the USB core will handle a hub with no ports...
2248 	 */
2249 	if (xhci->num_usb2_ports) {
2250 		xhci->usb2_ports = kmalloc(sizeof(*xhci->usb2_ports)*
2251 				xhci->num_usb2_ports, flags);
2252 		if (!xhci->usb2_ports)
2253 			return -ENOMEM;
2254 
2255 		port_index = 0;
2256 		for (i = 0; i < num_ports; i++) {
2257 			if (xhci->port_array[i] == 0x03 ||
2258 					xhci->port_array[i] == 0 ||
2259 					xhci->port_array[i] == DUPLICATE_ENTRY)
2260 				continue;
2261 
2262 			xhci->usb2_ports[port_index] =
2263 				&xhci->op_regs->port_status_base +
2264 				NUM_PORT_REGS*i;
2265 			xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2266 					"USB 2.0 port at index %u, "
2267 					"addr = %p", i,
2268 					xhci->usb2_ports[port_index]);
2269 			port_index++;
2270 			if (port_index == xhci->num_usb2_ports)
2271 				break;
2272 		}
2273 	}
2274 	if (xhci->num_usb3_ports) {
2275 		xhci->usb3_ports = kmalloc(sizeof(*xhci->usb3_ports)*
2276 				xhci->num_usb3_ports, flags);
2277 		if (!xhci->usb3_ports)
2278 			return -ENOMEM;
2279 
2280 		port_index = 0;
2281 		for (i = 0; i < num_ports; i++)
2282 			if (xhci->port_array[i] == 0x03) {
2283 				xhci->usb3_ports[port_index] =
2284 					&xhci->op_regs->port_status_base +
2285 					NUM_PORT_REGS*i;
2286 				xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2287 						"USB 3.0 port at index %u, "
2288 						"addr = %p", i,
2289 						xhci->usb3_ports[port_index]);
2290 				port_index++;
2291 				if (port_index == xhci->num_usb3_ports)
2292 					break;
2293 			}
2294 	}
2295 	return 0;
2296 }
2297 
2298 int xhci_mem_init(struct xhci_hcd *xhci, gfp_t flags)
2299 {
2300 	dma_addr_t	dma;
2301 	struct device	*dev = xhci_to_hcd(xhci)->self.sysdev;
2302 	unsigned int	val, val2;
2303 	u64		val_64;
2304 	struct xhci_segment	*seg;
2305 	u32 page_size, temp;
2306 	int i;
2307 
2308 	INIT_LIST_HEAD(&xhci->cmd_list);
2309 
2310 	/* init command timeout work */
2311 	INIT_DELAYED_WORK(&xhci->cmd_timer, xhci_handle_command_timeout);
2312 	init_completion(&xhci->cmd_ring_stop_completion);
2313 
2314 	page_size = readl(&xhci->op_regs->page_size);
2315 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2316 			"Supported page size register = 0x%x", page_size);
2317 	for (i = 0; i < 16; i++) {
2318 		if ((0x1 & page_size) != 0)
2319 			break;
2320 		page_size = page_size >> 1;
2321 	}
2322 	if (i < 16)
2323 		xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2324 			"Supported page size of %iK", (1 << (i+12)) / 1024);
2325 	else
2326 		xhci_warn(xhci, "WARN: no supported page size\n");
2327 	/* Use 4K pages, since that's common and the minimum the HC supports */
2328 	xhci->page_shift = 12;
2329 	xhci->page_size = 1 << xhci->page_shift;
2330 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2331 			"HCD page size set to %iK", xhci->page_size / 1024);
2332 
2333 	/*
2334 	 * Program the Number of Device Slots Enabled field in the CONFIG
2335 	 * register with the max value of slots the HC can handle.
2336 	 */
2337 	val = HCS_MAX_SLOTS(readl(&xhci->cap_regs->hcs_params1));
2338 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2339 			"// xHC can handle at most %d device slots.", val);
2340 	val2 = readl(&xhci->op_regs->config_reg);
2341 	val |= (val2 & ~HCS_SLOTS_MASK);
2342 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2343 			"// Setting Max device slots reg = 0x%x.", val);
2344 	writel(val, &xhci->op_regs->config_reg);
2345 
2346 	/*
2347 	 * xHCI section 5.4.6 - doorbell array must be
2348 	 * "physically contiguous and 64-byte (cache line) aligned".
2349 	 */
2350 	xhci->dcbaa = dma_alloc_coherent(dev, sizeof(*xhci->dcbaa), &dma,
2351 			flags);
2352 	if (!xhci->dcbaa)
2353 		goto fail;
2354 	memset(xhci->dcbaa, 0, sizeof *(xhci->dcbaa));
2355 	xhci->dcbaa->dma = dma;
2356 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2357 			"// Device context base array address = 0x%llx (DMA), %p (virt)",
2358 			(unsigned long long)xhci->dcbaa->dma, xhci->dcbaa);
2359 	xhci_write_64(xhci, dma, &xhci->op_regs->dcbaa_ptr);
2360 
2361 	/*
2362 	 * Initialize the ring segment pool.  The ring must be a contiguous
2363 	 * structure comprised of TRBs.  The TRBs must be 16 byte aligned,
2364 	 * however, the command ring segment needs 64-byte aligned segments
2365 	 * and our use of dma addresses in the trb_address_map radix tree needs
2366 	 * TRB_SEGMENT_SIZE alignment, so we pick the greater alignment need.
2367 	 */
2368 	xhci->segment_pool = dma_pool_create("xHCI ring segments", dev,
2369 			TRB_SEGMENT_SIZE, TRB_SEGMENT_SIZE, xhci->page_size);
2370 
2371 	/* See Table 46 and Note on Figure 55 */
2372 	xhci->device_pool = dma_pool_create("xHCI input/output contexts", dev,
2373 			2112, 64, xhci->page_size);
2374 	if (!xhci->segment_pool || !xhci->device_pool)
2375 		goto fail;
2376 
2377 	/* Linear stream context arrays don't have any boundary restrictions,
2378 	 * and only need to be 16-byte aligned.
2379 	 */
2380 	xhci->small_streams_pool =
2381 		dma_pool_create("xHCI 256 byte stream ctx arrays",
2382 			dev, SMALL_STREAM_ARRAY_SIZE, 16, 0);
2383 	xhci->medium_streams_pool =
2384 		dma_pool_create("xHCI 1KB stream ctx arrays",
2385 			dev, MEDIUM_STREAM_ARRAY_SIZE, 16, 0);
2386 	/* Any stream context array bigger than MEDIUM_STREAM_ARRAY_SIZE
2387 	 * will be allocated with dma_alloc_coherent()
2388 	 */
2389 
2390 	if (!xhci->small_streams_pool || !xhci->medium_streams_pool)
2391 		goto fail;
2392 
2393 	/* Set up the command ring to have one segments for now. */
2394 	xhci->cmd_ring = xhci_ring_alloc(xhci, 1, 1, TYPE_COMMAND, 0, flags);
2395 	if (!xhci->cmd_ring)
2396 		goto fail;
2397 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2398 			"Allocated command ring at %p", xhci->cmd_ring);
2399 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "First segment DMA is 0x%llx",
2400 			(unsigned long long)xhci->cmd_ring->first_seg->dma);
2401 
2402 	/* Set the address in the Command Ring Control register */
2403 	val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
2404 	val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
2405 		(xhci->cmd_ring->first_seg->dma & (u64) ~CMD_RING_RSVD_BITS) |
2406 		xhci->cmd_ring->cycle_state;
2407 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2408 			"// Setting command ring address to 0x%016llx", val_64);
2409 	xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
2410 	xhci_dbg_cmd_ptrs(xhci);
2411 
2412 	xhci->lpm_command = xhci_alloc_command(xhci, true, true, flags);
2413 	if (!xhci->lpm_command)
2414 		goto fail;
2415 
2416 	/* Reserve one command ring TRB for disabling LPM.
2417 	 * Since the USB core grabs the shared usb_bus bandwidth mutex before
2418 	 * disabling LPM, we only need to reserve one TRB for all devices.
2419 	 */
2420 	xhci->cmd_ring_reserved_trbs++;
2421 
2422 	val = readl(&xhci->cap_regs->db_off);
2423 	val &= DBOFF_MASK;
2424 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2425 			"// Doorbell array is located at offset 0x%x"
2426 			" from cap regs base addr", val);
2427 	xhci->dba = (void __iomem *) xhci->cap_regs + val;
2428 	xhci_dbg_regs(xhci);
2429 	xhci_print_run_regs(xhci);
2430 	/* Set ir_set to interrupt register set 0 */
2431 	xhci->ir_set = &xhci->run_regs->ir_set[0];
2432 
2433 	/*
2434 	 * Event ring setup: Allocate a normal ring, but also setup
2435 	 * the event ring segment table (ERST).  Section 4.9.3.
2436 	 */
2437 	xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Allocating event ring");
2438 	xhci->event_ring = xhci_ring_alloc(xhci, ERST_NUM_SEGS, 1, TYPE_EVENT,
2439 					0, flags);
2440 	if (!xhci->event_ring)
2441 		goto fail;
2442 	if (xhci_check_trb_in_td_math(xhci) < 0)
2443 		goto fail;
2444 
2445 	xhci->erst.entries = dma_alloc_coherent(dev,
2446 			sizeof(struct xhci_erst_entry) * ERST_NUM_SEGS, &dma,
2447 			flags);
2448 	if (!xhci->erst.entries)
2449 		goto fail;
2450 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2451 			"// Allocated event ring segment table at 0x%llx",
2452 			(unsigned long long)dma);
2453 
2454 	memset(xhci->erst.entries, 0, sizeof(struct xhci_erst_entry)*ERST_NUM_SEGS);
2455 	xhci->erst.num_entries = ERST_NUM_SEGS;
2456 	xhci->erst.erst_dma_addr = dma;
2457 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2458 			"Set ERST to 0; private num segs = %i, virt addr = %p, dma addr = 0x%llx",
2459 			xhci->erst.num_entries,
2460 			xhci->erst.entries,
2461 			(unsigned long long)xhci->erst.erst_dma_addr);
2462 
2463 	/* set ring base address and size for each segment table entry */
2464 	for (val = 0, seg = xhci->event_ring->first_seg; val < ERST_NUM_SEGS; val++) {
2465 		struct xhci_erst_entry *entry = &xhci->erst.entries[val];
2466 		entry->seg_addr = cpu_to_le64(seg->dma);
2467 		entry->seg_size = cpu_to_le32(TRBS_PER_SEGMENT);
2468 		entry->rsvd = 0;
2469 		seg = seg->next;
2470 	}
2471 
2472 	/* set ERST count with the number of entries in the segment table */
2473 	val = readl(&xhci->ir_set->erst_size);
2474 	val &= ERST_SIZE_MASK;
2475 	val |= ERST_NUM_SEGS;
2476 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2477 			"// Write ERST size = %i to ir_set 0 (some bits preserved)",
2478 			val);
2479 	writel(val, &xhci->ir_set->erst_size);
2480 
2481 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2482 			"// Set ERST entries to point to event ring.");
2483 	/* set the segment table base address */
2484 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2485 			"// Set ERST base address for ir_set 0 = 0x%llx",
2486 			(unsigned long long)xhci->erst.erst_dma_addr);
2487 	val_64 = xhci_read_64(xhci, &xhci->ir_set->erst_base);
2488 	val_64 &= ERST_PTR_MASK;
2489 	val_64 |= (xhci->erst.erst_dma_addr & (u64) ~ERST_PTR_MASK);
2490 	xhci_write_64(xhci, val_64, &xhci->ir_set->erst_base);
2491 
2492 	/* Set the event ring dequeue address */
2493 	xhci_set_hc_event_deq(xhci);
2494 	xhci_dbg_trace(xhci, trace_xhci_dbg_init,
2495 			"Wrote ERST address to ir_set 0.");
2496 	xhci_print_ir_set(xhci, 0);
2497 
2498 	/*
2499 	 * XXX: Might need to set the Interrupter Moderation Register to
2500 	 * something other than the default (~1ms minimum between interrupts).
2501 	 * See section 5.5.1.2.
2502 	 */
2503 	for (i = 0; i < MAX_HC_SLOTS; i++)
2504 		xhci->devs[i] = NULL;
2505 	for (i = 0; i < USB_MAXCHILDREN; i++) {
2506 		xhci->bus_state[0].resume_done[i] = 0;
2507 		xhci->bus_state[1].resume_done[i] = 0;
2508 		/* Only the USB 2.0 completions will ever be used. */
2509 		init_completion(&xhci->bus_state[1].rexit_done[i]);
2510 	}
2511 
2512 	if (scratchpad_alloc(xhci, flags))
2513 		goto fail;
2514 	if (xhci_setup_port_arrays(xhci, flags))
2515 		goto fail;
2516 
2517 	/* Enable USB 3.0 device notifications for function remote wake, which
2518 	 * is necessary for allowing USB 3.0 devices to do remote wakeup from
2519 	 * U3 (device suspend).
2520 	 */
2521 	temp = readl(&xhci->op_regs->dev_notification);
2522 	temp &= ~DEV_NOTE_MASK;
2523 	temp |= DEV_NOTE_FWAKE;
2524 	writel(temp, &xhci->op_regs->dev_notification);
2525 
2526 	return 0;
2527 
2528 fail:
2529 	xhci_halt(xhci);
2530 	xhci_reset(xhci);
2531 	xhci_mem_cleanup(xhci);
2532 	return -ENOMEM;
2533 }
2534