xref: /openbmc/linux/drivers/usb/host/xhci-ring.c (revision 4da722ca)
1 /*
2  * xHCI host controller driver
3  *
4  * Copyright (C) 2008 Intel Corp.
5  *
6  * Author: Sarah Sharp
7  * Some code borrowed from the Linux EHCI driver.
8  *
9  * This program is free software; you can redistribute it and/or modify
10  * it under the terms of the GNU General Public License version 2 as
11  * published by the Free Software Foundation.
12  *
13  * This program is distributed in the hope that it will be useful, but
14  * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
15  * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
16  * for more details.
17  *
18  * You should have received a copy of the GNU General Public License
19  * along with this program; if not, write to the Free Software Foundation,
20  * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
21  */
22 
23 /*
24  * Ring initialization rules:
25  * 1. Each segment is initialized to zero, except for link TRBs.
26  * 2. Ring cycle state = 0.  This represents Producer Cycle State (PCS) or
27  *    Consumer Cycle State (CCS), depending on ring function.
28  * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment.
29  *
30  * Ring behavior rules:
31  * 1. A ring is empty if enqueue == dequeue.  This means there will always be at
32  *    least one free TRB in the ring.  This is useful if you want to turn that
33  *    into a link TRB and expand the ring.
34  * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a
35  *    link TRB, then load the pointer with the address in the link TRB.  If the
36  *    link TRB had its toggle bit set, you may need to update the ring cycle
37  *    state (see cycle bit rules).  You may have to do this multiple times
38  *    until you reach a non-link TRB.
39  * 3. A ring is full if enqueue++ (for the definition of increment above)
40  *    equals the dequeue pointer.
41  *
42  * Cycle bit rules:
43  * 1. When a consumer increments a dequeue pointer and encounters a toggle bit
44  *    in a link TRB, it must toggle the ring cycle state.
45  * 2. When a producer increments an enqueue pointer and encounters a toggle bit
46  *    in a link TRB, it must toggle the ring cycle state.
47  *
48  * Producer rules:
49  * 1. Check if ring is full before you enqueue.
50  * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing.
51  *    Update enqueue pointer between each write (which may update the ring
52  *    cycle state).
53  * 3. Notify consumer.  If SW is producer, it rings the doorbell for command
54  *    and endpoint rings.  If HC is the producer for the event ring,
55  *    and it generates an interrupt according to interrupt modulation rules.
56  *
57  * Consumer rules:
58  * 1. Check if TRB belongs to you.  If the cycle bit == your ring cycle state,
59  *    the TRB is owned by the consumer.
60  * 2. Update dequeue pointer (which may update the ring cycle state) and
61  *    continue processing TRBs until you reach a TRB which is not owned by you.
62  * 3. Notify the producer.  SW is the consumer for the event ring, and it
63  *   updates event ring dequeue pointer.  HC is the consumer for the command and
64  *   endpoint rings; it generates events on the event ring for these.
65  */
66 
67 #include <linux/scatterlist.h>
68 #include <linux/slab.h>
69 #include <linux/dma-mapping.h>
70 #include "xhci.h"
71 #include "xhci-trace.h"
72 #include "xhci-mtk.h"
73 
74 /*
75  * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
76  * address of the TRB.
77  */
78 dma_addr_t xhci_trb_virt_to_dma(struct xhci_segment *seg,
79 		union xhci_trb *trb)
80 {
81 	unsigned long segment_offset;
82 
83 	if (!seg || !trb || trb < seg->trbs)
84 		return 0;
85 	/* offset in TRBs */
86 	segment_offset = trb - seg->trbs;
87 	if (segment_offset >= TRBS_PER_SEGMENT)
88 		return 0;
89 	return seg->dma + (segment_offset * sizeof(*trb));
90 }
91 
92 static bool trb_is_noop(union xhci_trb *trb)
93 {
94 	return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
95 }
96 
97 static bool trb_is_link(union xhci_trb *trb)
98 {
99 	return TRB_TYPE_LINK_LE32(trb->link.control);
100 }
101 
102 static bool last_trb_on_seg(struct xhci_segment *seg, union xhci_trb *trb)
103 {
104 	return trb == &seg->trbs[TRBS_PER_SEGMENT - 1];
105 }
106 
107 static bool last_trb_on_ring(struct xhci_ring *ring,
108 			struct xhci_segment *seg, union xhci_trb *trb)
109 {
110 	return last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg);
111 }
112 
113 static bool link_trb_toggles_cycle(union xhci_trb *trb)
114 {
115 	return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
116 }
117 
118 static bool last_td_in_urb(struct xhci_td *td)
119 {
120 	struct urb_priv *urb_priv = td->urb->hcpriv;
121 
122 	return urb_priv->num_tds_done == urb_priv->num_tds;
123 }
124 
125 static void inc_td_cnt(struct urb *urb)
126 {
127 	struct urb_priv *urb_priv = urb->hcpriv;
128 
129 	urb_priv->num_tds_done++;
130 }
131 
132 static void trb_to_noop(union xhci_trb *trb, u32 noop_type)
133 {
134 	if (trb_is_link(trb)) {
135 		/* unchain chained link TRBs */
136 		trb->link.control &= cpu_to_le32(~TRB_CHAIN);
137 	} else {
138 		trb->generic.field[0] = 0;
139 		trb->generic.field[1] = 0;
140 		trb->generic.field[2] = 0;
141 		/* Preserve only the cycle bit of this TRB */
142 		trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
143 		trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type));
144 	}
145 }
146 
147 /* Updates trb to point to the next TRB in the ring, and updates seg if the next
148  * TRB is in a new segment.  This does not skip over link TRBs, and it does not
149  * effect the ring dequeue or enqueue pointers.
150  */
151 static void next_trb(struct xhci_hcd *xhci,
152 		struct xhci_ring *ring,
153 		struct xhci_segment **seg,
154 		union xhci_trb **trb)
155 {
156 	if (trb_is_link(*trb)) {
157 		*seg = (*seg)->next;
158 		*trb = ((*seg)->trbs);
159 	} else {
160 		(*trb)++;
161 	}
162 }
163 
164 /*
165  * See Cycle bit rules. SW is the consumer for the event ring only.
166  * Don't make a ring full of link TRBs.  That would be dumb and this would loop.
167  */
168 static void inc_deq(struct xhci_hcd *xhci, struct xhci_ring *ring)
169 {
170 	/* event ring doesn't have link trbs, check for last trb */
171 	if (ring->type == TYPE_EVENT) {
172 		if (!last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
173 			ring->dequeue++;
174 			return;
175 		}
176 		if (last_trb_on_ring(ring, ring->deq_seg, ring->dequeue))
177 			ring->cycle_state ^= 1;
178 		ring->deq_seg = ring->deq_seg->next;
179 		ring->dequeue = ring->deq_seg->trbs;
180 		return;
181 	}
182 
183 	/* All other rings have link trbs */
184 	if (!trb_is_link(ring->dequeue)) {
185 		ring->dequeue++;
186 		ring->num_trbs_free++;
187 	}
188 	while (trb_is_link(ring->dequeue)) {
189 		ring->deq_seg = ring->deq_seg->next;
190 		ring->dequeue = ring->deq_seg->trbs;
191 	}
192 
193 	trace_xhci_inc_deq(ring);
194 
195 	return;
196 }
197 
198 /*
199  * See Cycle bit rules. SW is the consumer for the event ring only.
200  * Don't make a ring full of link TRBs.  That would be dumb and this would loop.
201  *
202  * If we've just enqueued a TRB that is in the middle of a TD (meaning the
203  * chain bit is set), then set the chain bit in all the following link TRBs.
204  * If we've enqueued the last TRB in a TD, make sure the following link TRBs
205  * have their chain bit cleared (so that each Link TRB is a separate TD).
206  *
207  * Section 6.4.4.1 of the 0.95 spec says link TRBs cannot have the chain bit
208  * set, but other sections talk about dealing with the chain bit set.  This was
209  * fixed in the 0.96 specification errata, but we have to assume that all 0.95
210  * xHCI hardware can't handle the chain bit being cleared on a link TRB.
211  *
212  * @more_trbs_coming:	Will you enqueue more TRBs before calling
213  *			prepare_transfer()?
214  */
215 static void inc_enq(struct xhci_hcd *xhci, struct xhci_ring *ring,
216 			bool more_trbs_coming)
217 {
218 	u32 chain;
219 	union xhci_trb *next;
220 
221 	chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
222 	/* If this is not event ring, there is one less usable TRB */
223 	if (!trb_is_link(ring->enqueue))
224 		ring->num_trbs_free--;
225 	next = ++(ring->enqueue);
226 
227 	/* Update the dequeue pointer further if that was a link TRB */
228 	while (trb_is_link(next)) {
229 
230 		/*
231 		 * If the caller doesn't plan on enqueueing more TDs before
232 		 * ringing the doorbell, then we don't want to give the link TRB
233 		 * to the hardware just yet. We'll give the link TRB back in
234 		 * prepare_ring() just before we enqueue the TD at the top of
235 		 * the ring.
236 		 */
237 		if (!chain && !more_trbs_coming)
238 			break;
239 
240 		/* If we're not dealing with 0.95 hardware or isoc rings on
241 		 * AMD 0.96 host, carry over the chain bit of the previous TRB
242 		 * (which may mean the chain bit is cleared).
243 		 */
244 		if (!(ring->type == TYPE_ISOC &&
245 		      (xhci->quirks & XHCI_AMD_0x96_HOST)) &&
246 		    !xhci_link_trb_quirk(xhci)) {
247 			next->link.control &= cpu_to_le32(~TRB_CHAIN);
248 			next->link.control |= cpu_to_le32(chain);
249 		}
250 		/* Give this link TRB to the hardware */
251 		wmb();
252 		next->link.control ^= cpu_to_le32(TRB_CYCLE);
253 
254 		/* Toggle the cycle bit after the last ring segment. */
255 		if (link_trb_toggles_cycle(next))
256 			ring->cycle_state ^= 1;
257 
258 		ring->enq_seg = ring->enq_seg->next;
259 		ring->enqueue = ring->enq_seg->trbs;
260 		next = ring->enqueue;
261 	}
262 
263 	trace_xhci_inc_enq(ring);
264 }
265 
266 /*
267  * Check to see if there's room to enqueue num_trbs on the ring and make sure
268  * enqueue pointer will not advance into dequeue segment. See rules above.
269  */
270 static inline int room_on_ring(struct xhci_hcd *xhci, struct xhci_ring *ring,
271 		unsigned int num_trbs)
272 {
273 	int num_trbs_in_deq_seg;
274 
275 	if (ring->num_trbs_free < num_trbs)
276 		return 0;
277 
278 	if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) {
279 		num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs;
280 		if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg)
281 			return 0;
282 	}
283 
284 	return 1;
285 }
286 
287 /* Ring the host controller doorbell after placing a command on the ring */
288 void xhci_ring_cmd_db(struct xhci_hcd *xhci)
289 {
290 	if (!(xhci->cmd_ring_state & CMD_RING_STATE_RUNNING))
291 		return;
292 
293 	xhci_dbg(xhci, "// Ding dong!\n");
294 	writel(DB_VALUE_HOST, &xhci->dba->doorbell[0]);
295 	/* Flush PCI posted writes */
296 	readl(&xhci->dba->doorbell[0]);
297 }
298 
299 static bool xhci_mod_cmd_timer(struct xhci_hcd *xhci, unsigned long delay)
300 {
301 	return mod_delayed_work(system_wq, &xhci->cmd_timer, delay);
302 }
303 
304 static struct xhci_command *xhci_next_queued_cmd(struct xhci_hcd *xhci)
305 {
306 	return list_first_entry_or_null(&xhci->cmd_list, struct xhci_command,
307 					cmd_list);
308 }
309 
310 /*
311  * Turn all commands on command ring with status set to "aborted" to no-op trbs.
312  * If there are other commands waiting then restart the ring and kick the timer.
313  * This must be called with command ring stopped and xhci->lock held.
314  */
315 static void xhci_handle_stopped_cmd_ring(struct xhci_hcd *xhci,
316 					 struct xhci_command *cur_cmd)
317 {
318 	struct xhci_command *i_cmd;
319 
320 	/* Turn all aborted commands in list to no-ops, then restart */
321 	list_for_each_entry(i_cmd, &xhci->cmd_list, cmd_list) {
322 
323 		if (i_cmd->status != COMP_COMMAND_ABORTED)
324 			continue;
325 
326 		i_cmd->status = COMP_COMMAND_RING_STOPPED;
327 
328 		xhci_dbg(xhci, "Turn aborted command %p to no-op\n",
329 			 i_cmd->command_trb);
330 
331 		trb_to_noop(i_cmd->command_trb, TRB_CMD_NOOP);
332 
333 		/*
334 		 * caller waiting for completion is called when command
335 		 *  completion event is received for these no-op commands
336 		 */
337 	}
338 
339 	xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
340 
341 	/* ring command ring doorbell to restart the command ring */
342 	if ((xhci->cmd_ring->dequeue != xhci->cmd_ring->enqueue) &&
343 	    !(xhci->xhc_state & XHCI_STATE_DYING)) {
344 		xhci->current_cmd = cur_cmd;
345 		xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
346 		xhci_ring_cmd_db(xhci);
347 	}
348 }
349 
350 /* Must be called with xhci->lock held, releases and aquires lock back */
351 static int xhci_abort_cmd_ring(struct xhci_hcd *xhci, unsigned long flags)
352 {
353 	u64 temp_64;
354 	int ret;
355 
356 	xhci_dbg(xhci, "Abort command ring\n");
357 
358 	reinit_completion(&xhci->cmd_ring_stop_completion);
359 
360 	temp_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
361 	xhci_write_64(xhci, temp_64 | CMD_RING_ABORT,
362 			&xhci->op_regs->cmd_ring);
363 
364 	/* Section 4.6.1.2 of xHCI 1.0 spec says software should also time the
365 	 * completion of the Command Abort operation. If CRR is not negated in 5
366 	 * seconds then driver handles it as if host died (-ENODEV).
367 	 * In the future we should distinguish between -ENODEV and -ETIMEDOUT
368 	 * and try to recover a -ETIMEDOUT with a host controller reset.
369 	 */
370 	ret = xhci_handshake(&xhci->op_regs->cmd_ring,
371 			CMD_RING_RUNNING, 0, 5 * 1000 * 1000);
372 	if (ret < 0) {
373 		xhci_err(xhci, "Abort failed to stop command ring: %d\n", ret);
374 		xhci_halt(xhci);
375 		xhci_hc_died(xhci);
376 		return ret;
377 	}
378 	/*
379 	 * Writing the CMD_RING_ABORT bit should cause a cmd completion event,
380 	 * however on some host hw the CMD_RING_RUNNING bit is correctly cleared
381 	 * but the completion event in never sent. Wait 2 secs (arbitrary
382 	 * number) to handle those cases after negation of CMD_RING_RUNNING.
383 	 */
384 	spin_unlock_irqrestore(&xhci->lock, flags);
385 	ret = wait_for_completion_timeout(&xhci->cmd_ring_stop_completion,
386 					  msecs_to_jiffies(2000));
387 	spin_lock_irqsave(&xhci->lock, flags);
388 	if (!ret) {
389 		xhci_dbg(xhci, "No stop event for abort, ring start fail?\n");
390 		xhci_cleanup_command_queue(xhci);
391 	} else {
392 		xhci_handle_stopped_cmd_ring(xhci, xhci_next_queued_cmd(xhci));
393 	}
394 	return 0;
395 }
396 
397 void xhci_ring_ep_doorbell(struct xhci_hcd *xhci,
398 		unsigned int slot_id,
399 		unsigned int ep_index,
400 		unsigned int stream_id)
401 {
402 	__le32 __iomem *db_addr = &xhci->dba->doorbell[slot_id];
403 	struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
404 	unsigned int ep_state = ep->ep_state;
405 
406 	/* Don't ring the doorbell for this endpoint if there are pending
407 	 * cancellations because we don't want to interrupt processing.
408 	 * We don't want to restart any stream rings if there's a set dequeue
409 	 * pointer command pending because the device can choose to start any
410 	 * stream once the endpoint is on the HW schedule.
411 	 */
412 	if ((ep_state & EP_STOP_CMD_PENDING) || (ep_state & SET_DEQ_PENDING) ||
413 	    (ep_state & EP_HALTED))
414 		return;
415 	writel(DB_VALUE(ep_index, stream_id), db_addr);
416 	/* The CPU has better things to do at this point than wait for a
417 	 * write-posting flush.  It'll get there soon enough.
418 	 */
419 }
420 
421 /* Ring the doorbell for any rings with pending URBs */
422 static void ring_doorbell_for_active_rings(struct xhci_hcd *xhci,
423 		unsigned int slot_id,
424 		unsigned int ep_index)
425 {
426 	unsigned int stream_id;
427 	struct xhci_virt_ep *ep;
428 
429 	ep = &xhci->devs[slot_id]->eps[ep_index];
430 
431 	/* A ring has pending URBs if its TD list is not empty */
432 	if (!(ep->ep_state & EP_HAS_STREAMS)) {
433 		if (ep->ring && !(list_empty(&ep->ring->td_list)))
434 			xhci_ring_ep_doorbell(xhci, slot_id, ep_index, 0);
435 		return;
436 	}
437 
438 	for (stream_id = 1; stream_id < ep->stream_info->num_streams;
439 			stream_id++) {
440 		struct xhci_stream_info *stream_info = ep->stream_info;
441 		if (!list_empty(&stream_info->stream_rings[stream_id]->td_list))
442 			xhci_ring_ep_doorbell(xhci, slot_id, ep_index,
443 						stream_id);
444 	}
445 }
446 
447 /* Get the right ring for the given slot_id, ep_index and stream_id.
448  * If the endpoint supports streams, boundary check the URB's stream ID.
449  * If the endpoint doesn't support streams, return the singular endpoint ring.
450  */
451 struct xhci_ring *xhci_triad_to_transfer_ring(struct xhci_hcd *xhci,
452 		unsigned int slot_id, unsigned int ep_index,
453 		unsigned int stream_id)
454 {
455 	struct xhci_virt_ep *ep;
456 
457 	ep = &xhci->devs[slot_id]->eps[ep_index];
458 	/* Common case: no streams */
459 	if (!(ep->ep_state & EP_HAS_STREAMS))
460 		return ep->ring;
461 
462 	if (stream_id == 0) {
463 		xhci_warn(xhci,
464 				"WARN: Slot ID %u, ep index %u has streams, "
465 				"but URB has no stream ID.\n",
466 				slot_id, ep_index);
467 		return NULL;
468 	}
469 
470 	if (stream_id < ep->stream_info->num_streams)
471 		return ep->stream_info->stream_rings[stream_id];
472 
473 	xhci_warn(xhci,
474 			"WARN: Slot ID %u, ep index %u has "
475 			"stream IDs 1 to %u allocated, "
476 			"but stream ID %u is requested.\n",
477 			slot_id, ep_index,
478 			ep->stream_info->num_streams - 1,
479 			stream_id);
480 	return NULL;
481 }
482 
483 
484 /*
485  * Get the hw dequeue pointer xHC stopped on, either directly from the
486  * endpoint context, or if streams are in use from the stream context.
487  * The returned hw_dequeue contains the lowest four bits with cycle state
488  * and possbile stream context type.
489  */
490 static u64 xhci_get_hw_deq(struct xhci_hcd *xhci, struct xhci_virt_device *vdev,
491 			   unsigned int ep_index, unsigned int stream_id)
492 {
493 	struct xhci_ep_ctx *ep_ctx;
494 	struct xhci_stream_ctx *st_ctx;
495 	struct xhci_virt_ep *ep;
496 
497 	ep = &vdev->eps[ep_index];
498 
499 	if (ep->ep_state & EP_HAS_STREAMS) {
500 		st_ctx = &ep->stream_info->stream_ctx_array[stream_id];
501 		return le64_to_cpu(st_ctx->stream_ring);
502 	}
503 	ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
504 	return le64_to_cpu(ep_ctx->deq);
505 }
506 
507 /*
508  * Move the xHC's endpoint ring dequeue pointer past cur_td.
509  * Record the new state of the xHC's endpoint ring dequeue segment,
510  * dequeue pointer, stream id, and new consumer cycle state in state.
511  * Update our internal representation of the ring's dequeue pointer.
512  *
513  * We do this in three jumps:
514  *  - First we update our new ring state to be the same as when the xHC stopped.
515  *  - Then we traverse the ring to find the segment that contains
516  *    the last TRB in the TD.  We toggle the xHC's new cycle state when we pass
517  *    any link TRBs with the toggle cycle bit set.
518  *  - Finally we move the dequeue state one TRB further, toggling the cycle bit
519  *    if we've moved it past a link TRB with the toggle cycle bit set.
520  *
521  * Some of the uses of xhci_generic_trb are grotty, but if they're done
522  * with correct __le32 accesses they should work fine.  Only users of this are
523  * in here.
524  */
525 void xhci_find_new_dequeue_state(struct xhci_hcd *xhci,
526 		unsigned int slot_id, unsigned int ep_index,
527 		unsigned int stream_id, struct xhci_td *cur_td,
528 		struct xhci_dequeue_state *state)
529 {
530 	struct xhci_virt_device *dev = xhci->devs[slot_id];
531 	struct xhci_virt_ep *ep = &dev->eps[ep_index];
532 	struct xhci_ring *ep_ring;
533 	struct xhci_segment *new_seg;
534 	union xhci_trb *new_deq;
535 	dma_addr_t addr;
536 	u64 hw_dequeue;
537 	bool cycle_found = false;
538 	bool td_last_trb_found = false;
539 
540 	ep_ring = xhci_triad_to_transfer_ring(xhci, slot_id,
541 			ep_index, stream_id);
542 	if (!ep_ring) {
543 		xhci_warn(xhci, "WARN can't find new dequeue state "
544 				"for invalid stream ID %u.\n",
545 				stream_id);
546 		return;
547 	}
548 	/* Dig out the cycle state saved by the xHC during the stop ep cmd */
549 	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
550 			"Finding endpoint context");
551 
552 	hw_dequeue = xhci_get_hw_deq(xhci, dev, ep_index, stream_id);
553 	new_seg = ep_ring->deq_seg;
554 	new_deq = ep_ring->dequeue;
555 	state->new_cycle_state = hw_dequeue & 0x1;
556 	state->stream_id = stream_id;
557 
558 	/*
559 	 * We want to find the pointer, segment and cycle state of the new trb
560 	 * (the one after current TD's last_trb). We know the cycle state at
561 	 * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are
562 	 * found.
563 	 */
564 	do {
565 		if (!cycle_found && xhci_trb_virt_to_dma(new_seg, new_deq)
566 		    == (dma_addr_t)(hw_dequeue & ~0xf)) {
567 			cycle_found = true;
568 			if (td_last_trb_found)
569 				break;
570 		}
571 		if (new_deq == cur_td->last_trb)
572 			td_last_trb_found = true;
573 
574 		if (cycle_found && trb_is_link(new_deq) &&
575 		    link_trb_toggles_cycle(new_deq))
576 			state->new_cycle_state ^= 0x1;
577 
578 		next_trb(xhci, ep_ring, &new_seg, &new_deq);
579 
580 		/* Search wrapped around, bail out */
581 		if (new_deq == ep->ring->dequeue) {
582 			xhci_err(xhci, "Error: Failed finding new dequeue state\n");
583 			state->new_deq_seg = NULL;
584 			state->new_deq_ptr = NULL;
585 			return;
586 		}
587 
588 	} while (!cycle_found || !td_last_trb_found);
589 
590 	state->new_deq_seg = new_seg;
591 	state->new_deq_ptr = new_deq;
592 
593 	/* Don't update the ring cycle state for the producer (us). */
594 	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
595 			"Cycle state = 0x%x", state->new_cycle_state);
596 
597 	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
598 			"New dequeue segment = %p (virtual)",
599 			state->new_deq_seg);
600 	addr = xhci_trb_virt_to_dma(state->new_deq_seg, state->new_deq_ptr);
601 	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
602 			"New dequeue pointer = 0x%llx (DMA)",
603 			(unsigned long long) addr);
604 }
605 
606 /* flip_cycle means flip the cycle bit of all but the first and last TRB.
607  * (The last TRB actually points to the ring enqueue pointer, which is not part
608  * of this TD.)  This is used to remove partially enqueued isoc TDs from a ring.
609  */
610 static void td_to_noop(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
611 		       struct xhci_td *td, bool flip_cycle)
612 {
613 	struct xhci_segment *seg	= td->start_seg;
614 	union xhci_trb *trb		= td->first_trb;
615 
616 	while (1) {
617 		trb_to_noop(trb, TRB_TR_NOOP);
618 
619 		/* flip cycle if asked to */
620 		if (flip_cycle && trb != td->first_trb && trb != td->last_trb)
621 			trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
622 
623 		if (trb == td->last_trb)
624 			break;
625 
626 		next_trb(xhci, ep_ring, &seg, &trb);
627 	}
628 }
629 
630 static void xhci_stop_watchdog_timer_in_irq(struct xhci_hcd *xhci,
631 		struct xhci_virt_ep *ep)
632 {
633 	ep->ep_state &= ~EP_STOP_CMD_PENDING;
634 	/* Can't del_timer_sync in interrupt */
635 	del_timer(&ep->stop_cmd_timer);
636 }
637 
638 /*
639  * Must be called with xhci->lock held in interrupt context,
640  * releases and re-acquires xhci->lock
641  */
642 static void xhci_giveback_urb_in_irq(struct xhci_hcd *xhci,
643 				     struct xhci_td *cur_td, int status)
644 {
645 	struct urb	*urb		= cur_td->urb;
646 	struct urb_priv	*urb_priv	= urb->hcpriv;
647 	struct usb_hcd	*hcd		= bus_to_hcd(urb->dev->bus);
648 
649 	if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
650 		xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs--;
651 		if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs	== 0) {
652 			if (xhci->quirks & XHCI_AMD_PLL_FIX)
653 				usb_amd_quirk_pll_enable();
654 		}
655 	}
656 	xhci_urb_free_priv(urb_priv);
657 	usb_hcd_unlink_urb_from_ep(hcd, urb);
658 	spin_unlock(&xhci->lock);
659 	trace_xhci_urb_giveback(urb);
660 	usb_hcd_giveback_urb(hcd, urb, status);
661 	spin_lock(&xhci->lock);
662 }
663 
664 static void xhci_unmap_td_bounce_buffer(struct xhci_hcd *xhci,
665 		struct xhci_ring *ring, struct xhci_td *td)
666 {
667 	struct device *dev = xhci_to_hcd(xhci)->self.controller;
668 	struct xhci_segment *seg = td->bounce_seg;
669 	struct urb *urb = td->urb;
670 
671 	if (!ring || !seg || !urb)
672 		return;
673 
674 	if (usb_urb_dir_out(urb)) {
675 		dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
676 				 DMA_TO_DEVICE);
677 		return;
678 	}
679 
680 	/* for in tranfers we need to copy the data from bounce to sg */
681 	sg_pcopy_from_buffer(urb->sg, urb->num_mapped_sgs, seg->bounce_buf,
682 			     seg->bounce_len, seg->bounce_offs);
683 	dma_unmap_single(dev, seg->bounce_dma, ring->bounce_buf_len,
684 			 DMA_FROM_DEVICE);
685 	seg->bounce_len = 0;
686 	seg->bounce_offs = 0;
687 }
688 
689 /*
690  * When we get a command completion for a Stop Endpoint Command, we need to
691  * unlink any cancelled TDs from the ring.  There are two ways to do that:
692  *
693  *  1. If the HW was in the middle of processing the TD that needs to be
694  *     cancelled, then we must move the ring's dequeue pointer past the last TRB
695  *     in the TD with a Set Dequeue Pointer Command.
696  *  2. Otherwise, we turn all the TRBs in the TD into No-op TRBs (with the chain
697  *     bit cleared) so that the HW will skip over them.
698  */
699 static void xhci_handle_cmd_stop_ep(struct xhci_hcd *xhci, int slot_id,
700 		union xhci_trb *trb, struct xhci_event_cmd *event)
701 {
702 	unsigned int ep_index;
703 	struct xhci_ring *ep_ring;
704 	struct xhci_virt_ep *ep;
705 	struct xhci_td *cur_td = NULL;
706 	struct xhci_td *last_unlinked_td;
707 	struct xhci_ep_ctx *ep_ctx;
708 	struct xhci_virt_device *vdev;
709 	u64 hw_deq;
710 	struct xhci_dequeue_state deq_state;
711 
712 	if (unlikely(TRB_TO_SUSPEND_PORT(le32_to_cpu(trb->generic.field[3])))) {
713 		if (!xhci->devs[slot_id])
714 			xhci_warn(xhci, "Stop endpoint command "
715 				"completion for disabled slot %u\n",
716 				slot_id);
717 		return;
718 	}
719 
720 	memset(&deq_state, 0, sizeof(deq_state));
721 	ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
722 
723 	vdev = xhci->devs[slot_id];
724 	ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
725 	trace_xhci_handle_cmd_stop_ep(ep_ctx);
726 
727 	ep = &xhci->devs[slot_id]->eps[ep_index];
728 	last_unlinked_td = list_last_entry(&ep->cancelled_td_list,
729 			struct xhci_td, cancelled_td_list);
730 
731 	if (list_empty(&ep->cancelled_td_list)) {
732 		xhci_stop_watchdog_timer_in_irq(xhci, ep);
733 		ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
734 		return;
735 	}
736 
737 	/* Fix up the ep ring first, so HW stops executing cancelled TDs.
738 	 * We have the xHCI lock, so nothing can modify this list until we drop
739 	 * it.  We're also in the event handler, so we can't get re-interrupted
740 	 * if another Stop Endpoint command completes
741 	 */
742 	list_for_each_entry(cur_td, &ep->cancelled_td_list, cancelled_td_list) {
743 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
744 				"Removing canceled TD starting at 0x%llx (dma).",
745 				(unsigned long long)xhci_trb_virt_to_dma(
746 					cur_td->start_seg, cur_td->first_trb));
747 		ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
748 		if (!ep_ring) {
749 			/* This shouldn't happen unless a driver is mucking
750 			 * with the stream ID after submission.  This will
751 			 * leave the TD on the hardware ring, and the hardware
752 			 * will try to execute it, and may access a buffer
753 			 * that has already been freed.  In the best case, the
754 			 * hardware will execute it, and the event handler will
755 			 * ignore the completion event for that TD, since it was
756 			 * removed from the td_list for that endpoint.  In
757 			 * short, don't muck with the stream ID after
758 			 * submission.
759 			 */
760 			xhci_warn(xhci, "WARN Cancelled URB %p "
761 					"has invalid stream ID %u.\n",
762 					cur_td->urb,
763 					cur_td->urb->stream_id);
764 			goto remove_finished_td;
765 		}
766 		/*
767 		 * If we stopped on the TD we need to cancel, then we have to
768 		 * move the xHC endpoint ring dequeue pointer past this TD.
769 		 */
770 		hw_deq = xhci_get_hw_deq(xhci, vdev, ep_index,
771 					 cur_td->urb->stream_id);
772 		hw_deq &= ~0xf;
773 
774 		if (trb_in_td(xhci, cur_td->start_seg, cur_td->first_trb,
775 			      cur_td->last_trb, hw_deq, false)) {
776 			xhci_find_new_dequeue_state(xhci, slot_id, ep_index,
777 						    cur_td->urb->stream_id,
778 						    cur_td, &deq_state);
779 		} else {
780 			td_to_noop(xhci, ep_ring, cur_td, false);
781 		}
782 
783 remove_finished_td:
784 		/*
785 		 * The event handler won't see a completion for this TD anymore,
786 		 * so remove it from the endpoint ring's TD list.  Keep it in
787 		 * the cancelled TD list for URB completion later.
788 		 */
789 		list_del_init(&cur_td->td_list);
790 	}
791 
792 	xhci_stop_watchdog_timer_in_irq(xhci, ep);
793 
794 	/* If necessary, queue a Set Transfer Ring Dequeue Pointer command */
795 	if (deq_state.new_deq_ptr && deq_state.new_deq_seg) {
796 		xhci_queue_new_dequeue_state(xhci, slot_id, ep_index,
797 					     &deq_state);
798 		xhci_ring_cmd_db(xhci);
799 	} else {
800 		/* Otherwise ring the doorbell(s) to restart queued transfers */
801 		ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
802 	}
803 
804 	/*
805 	 * Drop the lock and complete the URBs in the cancelled TD list.
806 	 * New TDs to be cancelled might be added to the end of the list before
807 	 * we can complete all the URBs for the TDs we already unlinked.
808 	 * So stop when we've completed the URB for the last TD we unlinked.
809 	 */
810 	do {
811 		cur_td = list_first_entry(&ep->cancelled_td_list,
812 				struct xhci_td, cancelled_td_list);
813 		list_del_init(&cur_td->cancelled_td_list);
814 
815 		/* Clean up the cancelled URB */
816 		/* Doesn't matter what we pass for status, since the core will
817 		 * just overwrite it (because the URB has been unlinked).
818 		 */
819 		ep_ring = xhci_urb_to_transfer_ring(xhci, cur_td->urb);
820 		xhci_unmap_td_bounce_buffer(xhci, ep_ring, cur_td);
821 		inc_td_cnt(cur_td->urb);
822 		if (last_td_in_urb(cur_td))
823 			xhci_giveback_urb_in_irq(xhci, cur_td, 0);
824 
825 		/* Stop processing the cancelled list if the watchdog timer is
826 		 * running.
827 		 */
828 		if (xhci->xhc_state & XHCI_STATE_DYING)
829 			return;
830 	} while (cur_td != last_unlinked_td);
831 
832 	/* Return to the event handler with xhci->lock re-acquired */
833 }
834 
835 static void xhci_kill_ring_urbs(struct xhci_hcd *xhci, struct xhci_ring *ring)
836 {
837 	struct xhci_td *cur_td;
838 	struct xhci_td *tmp;
839 
840 	list_for_each_entry_safe(cur_td, tmp, &ring->td_list, td_list) {
841 		list_del_init(&cur_td->td_list);
842 
843 		if (!list_empty(&cur_td->cancelled_td_list))
844 			list_del_init(&cur_td->cancelled_td_list);
845 
846 		xhci_unmap_td_bounce_buffer(xhci, ring, cur_td);
847 
848 		inc_td_cnt(cur_td->urb);
849 		if (last_td_in_urb(cur_td))
850 			xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
851 	}
852 }
853 
854 static void xhci_kill_endpoint_urbs(struct xhci_hcd *xhci,
855 		int slot_id, int ep_index)
856 {
857 	struct xhci_td *cur_td;
858 	struct xhci_td *tmp;
859 	struct xhci_virt_ep *ep;
860 	struct xhci_ring *ring;
861 
862 	ep = &xhci->devs[slot_id]->eps[ep_index];
863 	if ((ep->ep_state & EP_HAS_STREAMS) ||
864 			(ep->ep_state & EP_GETTING_NO_STREAMS)) {
865 		int stream_id;
866 
867 		for (stream_id = 0; stream_id < ep->stream_info->num_streams;
868 				stream_id++) {
869 			xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
870 					"Killing URBs for slot ID %u, ep index %u, stream %u",
871 					slot_id, ep_index, stream_id + 1);
872 			xhci_kill_ring_urbs(xhci,
873 					ep->stream_info->stream_rings[stream_id]);
874 		}
875 	} else {
876 		ring = ep->ring;
877 		if (!ring)
878 			return;
879 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
880 				"Killing URBs for slot ID %u, ep index %u",
881 				slot_id, ep_index);
882 		xhci_kill_ring_urbs(xhci, ring);
883 	}
884 
885 	list_for_each_entry_safe(cur_td, tmp, &ep->cancelled_td_list,
886 			cancelled_td_list) {
887 		list_del_init(&cur_td->cancelled_td_list);
888 		inc_td_cnt(cur_td->urb);
889 
890 		if (last_td_in_urb(cur_td))
891 			xhci_giveback_urb_in_irq(xhci, cur_td, -ESHUTDOWN);
892 	}
893 }
894 
895 /*
896  * host controller died, register read returns 0xffffffff
897  * Complete pending commands, mark them ABORTED.
898  * URBs need to be given back as usb core might be waiting with device locks
899  * held for the URBs to finish during device disconnect, blocking host remove.
900  *
901  * Call with xhci->lock held.
902  * lock is relased and re-acquired while giving back urb.
903  */
904 void xhci_hc_died(struct xhci_hcd *xhci)
905 {
906 	int i, j;
907 
908 	if (xhci->xhc_state & XHCI_STATE_DYING)
909 		return;
910 
911 	xhci_err(xhci, "xHCI host controller not responding, assume dead\n");
912 	xhci->xhc_state |= XHCI_STATE_DYING;
913 
914 	xhci_cleanup_command_queue(xhci);
915 
916 	/* return any pending urbs, remove may be waiting for them */
917 	for (i = 0; i <= HCS_MAX_SLOTS(xhci->hcs_params1); i++) {
918 		if (!xhci->devs[i])
919 			continue;
920 		for (j = 0; j < 31; j++)
921 			xhci_kill_endpoint_urbs(xhci, i, j);
922 	}
923 
924 	/* inform usb core hc died if PCI remove isn't already handling it */
925 	if (!(xhci->xhc_state & XHCI_STATE_REMOVING))
926 		usb_hc_died(xhci_to_hcd(xhci));
927 }
928 
929 /* Watchdog timer function for when a stop endpoint command fails to complete.
930  * In this case, we assume the host controller is broken or dying or dead.  The
931  * host may still be completing some other events, so we have to be careful to
932  * let the event ring handler and the URB dequeueing/enqueueing functions know
933  * through xhci->state.
934  *
935  * The timer may also fire if the host takes a very long time to respond to the
936  * command, and the stop endpoint command completion handler cannot delete the
937  * timer before the timer function is called.  Another endpoint cancellation may
938  * sneak in before the timer function can grab the lock, and that may queue
939  * another stop endpoint command and add the timer back.  So we cannot use a
940  * simple flag to say whether there is a pending stop endpoint command for a
941  * particular endpoint.
942  *
943  * Instead we use a combination of that flag and checking if a new timer is
944  * pending.
945  */
946 void xhci_stop_endpoint_command_watchdog(unsigned long arg)
947 {
948 	struct xhci_hcd *xhci;
949 	struct xhci_virt_ep *ep;
950 	unsigned long flags;
951 
952 	ep = (struct xhci_virt_ep *) arg;
953 	xhci = ep->xhci;
954 
955 	spin_lock_irqsave(&xhci->lock, flags);
956 
957 	/* bail out if cmd completed but raced with stop ep watchdog timer.*/
958 	if (!(ep->ep_state & EP_STOP_CMD_PENDING) ||
959 	    timer_pending(&ep->stop_cmd_timer)) {
960 		spin_unlock_irqrestore(&xhci->lock, flags);
961 		xhci_dbg(xhci, "Stop EP timer raced with cmd completion, exit");
962 		return;
963 	}
964 
965 	xhci_warn(xhci, "xHCI host not responding to stop endpoint command.\n");
966 	ep->ep_state &= ~EP_STOP_CMD_PENDING;
967 
968 	xhci_halt(xhci);
969 
970 	/*
971 	 * handle a stop endpoint cmd timeout as if host died (-ENODEV).
972 	 * In the future we could distinguish between -ENODEV and -ETIMEDOUT
973 	 * and try to recover a -ETIMEDOUT with a host controller reset
974 	 */
975 	xhci_hc_died(xhci);
976 
977 	spin_unlock_irqrestore(&xhci->lock, flags);
978 	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
979 			"xHCI host controller is dead.");
980 }
981 
982 static void update_ring_for_set_deq_completion(struct xhci_hcd *xhci,
983 		struct xhci_virt_device *dev,
984 		struct xhci_ring *ep_ring,
985 		unsigned int ep_index)
986 {
987 	union xhci_trb *dequeue_temp;
988 	int num_trbs_free_temp;
989 	bool revert = false;
990 
991 	num_trbs_free_temp = ep_ring->num_trbs_free;
992 	dequeue_temp = ep_ring->dequeue;
993 
994 	/* If we get two back-to-back stalls, and the first stalled transfer
995 	 * ends just before a link TRB, the dequeue pointer will be left on
996 	 * the link TRB by the code in the while loop.  So we have to update
997 	 * the dequeue pointer one segment further, or we'll jump off
998 	 * the segment into la-la-land.
999 	 */
1000 	if (trb_is_link(ep_ring->dequeue)) {
1001 		ep_ring->deq_seg = ep_ring->deq_seg->next;
1002 		ep_ring->dequeue = ep_ring->deq_seg->trbs;
1003 	}
1004 
1005 	while (ep_ring->dequeue != dev->eps[ep_index].queued_deq_ptr) {
1006 		/* We have more usable TRBs */
1007 		ep_ring->num_trbs_free++;
1008 		ep_ring->dequeue++;
1009 		if (trb_is_link(ep_ring->dequeue)) {
1010 			if (ep_ring->dequeue ==
1011 					dev->eps[ep_index].queued_deq_ptr)
1012 				break;
1013 			ep_ring->deq_seg = ep_ring->deq_seg->next;
1014 			ep_ring->dequeue = ep_ring->deq_seg->trbs;
1015 		}
1016 		if (ep_ring->dequeue == dequeue_temp) {
1017 			revert = true;
1018 			break;
1019 		}
1020 	}
1021 
1022 	if (revert) {
1023 		xhci_dbg(xhci, "Unable to find new dequeue pointer\n");
1024 		ep_ring->num_trbs_free = num_trbs_free_temp;
1025 	}
1026 }
1027 
1028 /*
1029  * When we get a completion for a Set Transfer Ring Dequeue Pointer command,
1030  * we need to clear the set deq pending flag in the endpoint ring state, so that
1031  * the TD queueing code can ring the doorbell again.  We also need to ring the
1032  * endpoint doorbell to restart the ring, but only if there aren't more
1033  * cancellations pending.
1034  */
1035 static void xhci_handle_cmd_set_deq(struct xhci_hcd *xhci, int slot_id,
1036 		union xhci_trb *trb, u32 cmd_comp_code)
1037 {
1038 	unsigned int ep_index;
1039 	unsigned int stream_id;
1040 	struct xhci_ring *ep_ring;
1041 	struct xhci_virt_device *dev;
1042 	struct xhci_virt_ep *ep;
1043 	struct xhci_ep_ctx *ep_ctx;
1044 	struct xhci_slot_ctx *slot_ctx;
1045 
1046 	ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1047 	stream_id = TRB_TO_STREAM_ID(le32_to_cpu(trb->generic.field[2]));
1048 	dev = xhci->devs[slot_id];
1049 	ep = &dev->eps[ep_index];
1050 
1051 	ep_ring = xhci_stream_id_to_ring(dev, ep_index, stream_id);
1052 	if (!ep_ring) {
1053 		xhci_warn(xhci, "WARN Set TR deq ptr command for freed stream ID %u\n",
1054 				stream_id);
1055 		/* XXX: Harmless??? */
1056 		goto cleanup;
1057 	}
1058 
1059 	ep_ctx = xhci_get_ep_ctx(xhci, dev->out_ctx, ep_index);
1060 	slot_ctx = xhci_get_slot_ctx(xhci, dev->out_ctx);
1061 	trace_xhci_handle_cmd_set_deq(slot_ctx);
1062 	trace_xhci_handle_cmd_set_deq_ep(ep_ctx);
1063 
1064 	if (cmd_comp_code != COMP_SUCCESS) {
1065 		unsigned int ep_state;
1066 		unsigned int slot_state;
1067 
1068 		switch (cmd_comp_code) {
1069 		case COMP_TRB_ERROR:
1070 			xhci_warn(xhci, "WARN Set TR Deq Ptr cmd invalid because of stream ID configuration\n");
1071 			break;
1072 		case COMP_CONTEXT_STATE_ERROR:
1073 			xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed due to incorrect slot or ep state.\n");
1074 			ep_state = GET_EP_CTX_STATE(ep_ctx);
1075 			slot_state = le32_to_cpu(slot_ctx->dev_state);
1076 			slot_state = GET_SLOT_STATE(slot_state);
1077 			xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1078 					"Slot state = %u, EP state = %u",
1079 					slot_state, ep_state);
1080 			break;
1081 		case COMP_SLOT_NOT_ENABLED_ERROR:
1082 			xhci_warn(xhci, "WARN Set TR Deq Ptr cmd failed because slot %u was not enabled.\n",
1083 					slot_id);
1084 			break;
1085 		default:
1086 			xhci_warn(xhci, "WARN Set TR Deq Ptr cmd with unknown completion code of %u.\n",
1087 					cmd_comp_code);
1088 			break;
1089 		}
1090 		/* OK what do we do now?  The endpoint state is hosed, and we
1091 		 * should never get to this point if the synchronization between
1092 		 * queueing, and endpoint state are correct.  This might happen
1093 		 * if the device gets disconnected after we've finished
1094 		 * cancelling URBs, which might not be an error...
1095 		 */
1096 	} else {
1097 		u64 deq;
1098 		/* 4.6.10 deq ptr is written to the stream ctx for streams */
1099 		if (ep->ep_state & EP_HAS_STREAMS) {
1100 			struct xhci_stream_ctx *ctx =
1101 				&ep->stream_info->stream_ctx_array[stream_id];
1102 			deq = le64_to_cpu(ctx->stream_ring) & SCTX_DEQ_MASK;
1103 		} else {
1104 			deq = le64_to_cpu(ep_ctx->deq) & ~EP_CTX_CYCLE_MASK;
1105 		}
1106 		xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
1107 			"Successful Set TR Deq Ptr cmd, deq = @%08llx", deq);
1108 		if (xhci_trb_virt_to_dma(ep->queued_deq_seg,
1109 					 ep->queued_deq_ptr) == deq) {
1110 			/* Update the ring's dequeue segment and dequeue pointer
1111 			 * to reflect the new position.
1112 			 */
1113 			update_ring_for_set_deq_completion(xhci, dev,
1114 				ep_ring, ep_index);
1115 		} else {
1116 			xhci_warn(xhci, "Mismatch between completed Set TR Deq Ptr command & xHCI internal state.\n");
1117 			xhci_warn(xhci, "ep deq seg = %p, deq ptr = %p\n",
1118 				  ep->queued_deq_seg, ep->queued_deq_ptr);
1119 		}
1120 	}
1121 
1122 cleanup:
1123 	dev->eps[ep_index].ep_state &= ~SET_DEQ_PENDING;
1124 	dev->eps[ep_index].queued_deq_seg = NULL;
1125 	dev->eps[ep_index].queued_deq_ptr = NULL;
1126 	/* Restart any rings with pending URBs */
1127 	ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1128 }
1129 
1130 static void xhci_handle_cmd_reset_ep(struct xhci_hcd *xhci, int slot_id,
1131 		union xhci_trb *trb, u32 cmd_comp_code)
1132 {
1133 	struct xhci_virt_device *vdev;
1134 	struct xhci_ep_ctx *ep_ctx;
1135 	unsigned int ep_index;
1136 
1137 	ep_index = TRB_TO_EP_INDEX(le32_to_cpu(trb->generic.field[3]));
1138 	vdev = xhci->devs[slot_id];
1139 	ep_ctx = xhci_get_ep_ctx(xhci, vdev->out_ctx, ep_index);
1140 	trace_xhci_handle_cmd_reset_ep(ep_ctx);
1141 
1142 	/* This command will only fail if the endpoint wasn't halted,
1143 	 * but we don't care.
1144 	 */
1145 	xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
1146 		"Ignoring reset ep completion code of %u", cmd_comp_code);
1147 
1148 	/* HW with the reset endpoint quirk needs to have a configure endpoint
1149 	 * command complete before the endpoint can be used.  Queue that here
1150 	 * because the HW can't handle two commands being queued in a row.
1151 	 */
1152 	if (xhci->quirks & XHCI_RESET_EP_QUIRK) {
1153 		struct xhci_command *command;
1154 
1155 		command = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
1156 		if (!command)
1157 			return;
1158 
1159 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1160 				"Queueing configure endpoint command");
1161 		xhci_queue_configure_endpoint(xhci, command,
1162 				xhci->devs[slot_id]->in_ctx->dma, slot_id,
1163 				false);
1164 		xhci_ring_cmd_db(xhci);
1165 	} else {
1166 		/* Clear our internal halted state */
1167 		xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_HALTED;
1168 	}
1169 }
1170 
1171 static void xhci_handle_cmd_enable_slot(struct xhci_hcd *xhci, int slot_id,
1172 		struct xhci_command *command, u32 cmd_comp_code)
1173 {
1174 	if (cmd_comp_code == COMP_SUCCESS)
1175 		command->slot_id = slot_id;
1176 	else
1177 		command->slot_id = 0;
1178 }
1179 
1180 static void xhci_handle_cmd_disable_slot(struct xhci_hcd *xhci, int slot_id)
1181 {
1182 	struct xhci_virt_device *virt_dev;
1183 	struct xhci_slot_ctx *slot_ctx;
1184 
1185 	virt_dev = xhci->devs[slot_id];
1186 	if (!virt_dev)
1187 		return;
1188 
1189 	slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
1190 	trace_xhci_handle_cmd_disable_slot(slot_ctx);
1191 
1192 	if (xhci->quirks & XHCI_EP_LIMIT_QUIRK)
1193 		/* Delete default control endpoint resources */
1194 		xhci_free_device_endpoint_resources(xhci, virt_dev, true);
1195 	xhci_free_virt_device(xhci, slot_id);
1196 }
1197 
1198 static void xhci_handle_cmd_config_ep(struct xhci_hcd *xhci, int slot_id,
1199 		struct xhci_event_cmd *event, u32 cmd_comp_code)
1200 {
1201 	struct xhci_virt_device *virt_dev;
1202 	struct xhci_input_control_ctx *ctrl_ctx;
1203 	struct xhci_ep_ctx *ep_ctx;
1204 	unsigned int ep_index;
1205 	unsigned int ep_state;
1206 	u32 add_flags, drop_flags;
1207 
1208 	/*
1209 	 * Configure endpoint commands can come from the USB core
1210 	 * configuration or alt setting changes, or because the HW
1211 	 * needed an extra configure endpoint command after a reset
1212 	 * endpoint command or streams were being configured.
1213 	 * If the command was for a halted endpoint, the xHCI driver
1214 	 * is not waiting on the configure endpoint command.
1215 	 */
1216 	virt_dev = xhci->devs[slot_id];
1217 	ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
1218 	if (!ctrl_ctx) {
1219 		xhci_warn(xhci, "Could not get input context, bad type.\n");
1220 		return;
1221 	}
1222 
1223 	add_flags = le32_to_cpu(ctrl_ctx->add_flags);
1224 	drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
1225 	/* Input ctx add_flags are the endpoint index plus one */
1226 	ep_index = xhci_last_valid_endpoint(add_flags) - 1;
1227 
1228 	ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->out_ctx, ep_index);
1229 	trace_xhci_handle_cmd_config_ep(ep_ctx);
1230 
1231 	/* A usb_set_interface() call directly after clearing a halted
1232 	 * condition may race on this quirky hardware.  Not worth
1233 	 * worrying about, since this is prototype hardware.  Not sure
1234 	 * if this will work for streams, but streams support was
1235 	 * untested on this prototype.
1236 	 */
1237 	if (xhci->quirks & XHCI_RESET_EP_QUIRK &&
1238 			ep_index != (unsigned int) -1 &&
1239 			add_flags - SLOT_FLAG == drop_flags) {
1240 		ep_state = virt_dev->eps[ep_index].ep_state;
1241 		if (!(ep_state & EP_HALTED))
1242 			return;
1243 		xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1244 				"Completed config ep cmd - "
1245 				"last ep index = %d, state = %d",
1246 				ep_index, ep_state);
1247 		/* Clear internal halted state and restart ring(s) */
1248 		virt_dev->eps[ep_index].ep_state &= ~EP_HALTED;
1249 		ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
1250 		return;
1251 	}
1252 	return;
1253 }
1254 
1255 static void xhci_handle_cmd_addr_dev(struct xhci_hcd *xhci, int slot_id)
1256 {
1257 	struct xhci_virt_device *vdev;
1258 	struct xhci_slot_ctx *slot_ctx;
1259 
1260 	vdev = xhci->devs[slot_id];
1261 	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1262 	trace_xhci_handle_cmd_addr_dev(slot_ctx);
1263 }
1264 
1265 static void xhci_handle_cmd_reset_dev(struct xhci_hcd *xhci, int slot_id,
1266 		struct xhci_event_cmd *event)
1267 {
1268 	struct xhci_virt_device *vdev;
1269 	struct xhci_slot_ctx *slot_ctx;
1270 
1271 	vdev = xhci->devs[slot_id];
1272 	slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
1273 	trace_xhci_handle_cmd_reset_dev(slot_ctx);
1274 
1275 	xhci_dbg(xhci, "Completed reset device command.\n");
1276 	if (!xhci->devs[slot_id])
1277 		xhci_warn(xhci, "Reset device command completion "
1278 				"for disabled slot %u\n", slot_id);
1279 }
1280 
1281 static void xhci_handle_cmd_nec_get_fw(struct xhci_hcd *xhci,
1282 		struct xhci_event_cmd *event)
1283 {
1284 	if (!(xhci->quirks & XHCI_NEC_HOST)) {
1285 		xhci_warn(xhci, "WARN NEC_GET_FW command on non-NEC host\n");
1286 		return;
1287 	}
1288 	xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
1289 			"NEC firmware version %2x.%02x",
1290 			NEC_FW_MAJOR(le32_to_cpu(event->status)),
1291 			NEC_FW_MINOR(le32_to_cpu(event->status)));
1292 }
1293 
1294 static void xhci_complete_del_and_free_cmd(struct xhci_command *cmd, u32 status)
1295 {
1296 	list_del(&cmd->cmd_list);
1297 
1298 	if (cmd->completion) {
1299 		cmd->status = status;
1300 		complete(cmd->completion);
1301 	} else {
1302 		kfree(cmd);
1303 	}
1304 }
1305 
1306 void xhci_cleanup_command_queue(struct xhci_hcd *xhci)
1307 {
1308 	struct xhci_command *cur_cmd, *tmp_cmd;
1309 	list_for_each_entry_safe(cur_cmd, tmp_cmd, &xhci->cmd_list, cmd_list)
1310 		xhci_complete_del_and_free_cmd(cur_cmd, COMP_COMMAND_ABORTED);
1311 }
1312 
1313 void xhci_handle_command_timeout(struct work_struct *work)
1314 {
1315 	struct xhci_hcd *xhci;
1316 	unsigned long flags;
1317 	u64 hw_ring_state;
1318 
1319 	xhci = container_of(to_delayed_work(work), struct xhci_hcd, cmd_timer);
1320 
1321 	spin_lock_irqsave(&xhci->lock, flags);
1322 
1323 	/*
1324 	 * If timeout work is pending, or current_cmd is NULL, it means we
1325 	 * raced with command completion. Command is handled so just return.
1326 	 */
1327 	if (!xhci->current_cmd || delayed_work_pending(&xhci->cmd_timer)) {
1328 		spin_unlock_irqrestore(&xhci->lock, flags);
1329 		return;
1330 	}
1331 	/* mark this command to be cancelled */
1332 	xhci->current_cmd->status = COMP_COMMAND_ABORTED;
1333 
1334 	/* Make sure command ring is running before aborting it */
1335 	hw_ring_state = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
1336 	if (hw_ring_state == ~(u64)0) {
1337 		xhci_hc_died(xhci);
1338 		goto time_out_completed;
1339 	}
1340 
1341 	if ((xhci->cmd_ring_state & CMD_RING_STATE_RUNNING) &&
1342 	    (hw_ring_state & CMD_RING_RUNNING))  {
1343 		/* Prevent new doorbell, and start command abort */
1344 		xhci->cmd_ring_state = CMD_RING_STATE_ABORTED;
1345 		xhci_dbg(xhci, "Command timeout\n");
1346 		xhci_abort_cmd_ring(xhci, flags);
1347 		goto time_out_completed;
1348 	}
1349 
1350 	/* host removed. Bail out */
1351 	if (xhci->xhc_state & XHCI_STATE_REMOVING) {
1352 		xhci_dbg(xhci, "host removed, ring start fail?\n");
1353 		xhci_cleanup_command_queue(xhci);
1354 
1355 		goto time_out_completed;
1356 	}
1357 
1358 	/* command timeout on stopped ring, ring can't be aborted */
1359 	xhci_dbg(xhci, "Command timeout on stopped ring\n");
1360 	xhci_handle_stopped_cmd_ring(xhci, xhci->current_cmd);
1361 
1362 time_out_completed:
1363 	spin_unlock_irqrestore(&xhci->lock, flags);
1364 	return;
1365 }
1366 
1367 static void handle_cmd_completion(struct xhci_hcd *xhci,
1368 		struct xhci_event_cmd *event)
1369 {
1370 	int slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1371 	u64 cmd_dma;
1372 	dma_addr_t cmd_dequeue_dma;
1373 	u32 cmd_comp_code;
1374 	union xhci_trb *cmd_trb;
1375 	struct xhci_command *cmd;
1376 	u32 cmd_type;
1377 
1378 	cmd_dma = le64_to_cpu(event->cmd_trb);
1379 	cmd_trb = xhci->cmd_ring->dequeue;
1380 
1381 	trace_xhci_handle_command(xhci->cmd_ring, &cmd_trb->generic);
1382 
1383 	cmd_dequeue_dma = xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
1384 			cmd_trb);
1385 	/*
1386 	 * Check whether the completion event is for our internal kept
1387 	 * command.
1388 	 */
1389 	if (!cmd_dequeue_dma || cmd_dma != (u64)cmd_dequeue_dma) {
1390 		xhci_warn(xhci,
1391 			  "ERROR mismatched command completion event\n");
1392 		return;
1393 	}
1394 
1395 	cmd = list_first_entry(&xhci->cmd_list, struct xhci_command, cmd_list);
1396 
1397 	cancel_delayed_work(&xhci->cmd_timer);
1398 
1399 	cmd_comp_code = GET_COMP_CODE(le32_to_cpu(event->status));
1400 
1401 	/* If CMD ring stopped we own the trbs between enqueue and dequeue */
1402 	if (cmd_comp_code == COMP_COMMAND_RING_STOPPED) {
1403 		complete_all(&xhci->cmd_ring_stop_completion);
1404 		return;
1405 	}
1406 
1407 	if (cmd->command_trb != xhci->cmd_ring->dequeue) {
1408 		xhci_err(xhci,
1409 			 "Command completion event does not match command\n");
1410 		return;
1411 	}
1412 
1413 	/*
1414 	 * Host aborted the command ring, check if the current command was
1415 	 * supposed to be aborted, otherwise continue normally.
1416 	 * The command ring is stopped now, but the xHC will issue a Command
1417 	 * Ring Stopped event which will cause us to restart it.
1418 	 */
1419 	if (cmd_comp_code == COMP_COMMAND_ABORTED) {
1420 		xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
1421 		if (cmd->status == COMP_COMMAND_ABORTED) {
1422 			if (xhci->current_cmd == cmd)
1423 				xhci->current_cmd = NULL;
1424 			goto event_handled;
1425 		}
1426 	}
1427 
1428 	cmd_type = TRB_FIELD_TO_TYPE(le32_to_cpu(cmd_trb->generic.field[3]));
1429 	switch (cmd_type) {
1430 	case TRB_ENABLE_SLOT:
1431 		xhci_handle_cmd_enable_slot(xhci, slot_id, cmd, cmd_comp_code);
1432 		break;
1433 	case TRB_DISABLE_SLOT:
1434 		xhci_handle_cmd_disable_slot(xhci, slot_id);
1435 		break;
1436 	case TRB_CONFIG_EP:
1437 		if (!cmd->completion)
1438 			xhci_handle_cmd_config_ep(xhci, slot_id, event,
1439 						  cmd_comp_code);
1440 		break;
1441 	case TRB_EVAL_CONTEXT:
1442 		break;
1443 	case TRB_ADDR_DEV:
1444 		xhci_handle_cmd_addr_dev(xhci, slot_id);
1445 		break;
1446 	case TRB_STOP_RING:
1447 		WARN_ON(slot_id != TRB_TO_SLOT_ID(
1448 				le32_to_cpu(cmd_trb->generic.field[3])));
1449 		xhci_handle_cmd_stop_ep(xhci, slot_id, cmd_trb, event);
1450 		break;
1451 	case TRB_SET_DEQ:
1452 		WARN_ON(slot_id != TRB_TO_SLOT_ID(
1453 				le32_to_cpu(cmd_trb->generic.field[3])));
1454 		xhci_handle_cmd_set_deq(xhci, slot_id, cmd_trb, cmd_comp_code);
1455 		break;
1456 	case TRB_CMD_NOOP:
1457 		/* Is this an aborted command turned to NO-OP? */
1458 		if (cmd->status == COMP_COMMAND_RING_STOPPED)
1459 			cmd_comp_code = COMP_COMMAND_RING_STOPPED;
1460 		break;
1461 	case TRB_RESET_EP:
1462 		WARN_ON(slot_id != TRB_TO_SLOT_ID(
1463 				le32_to_cpu(cmd_trb->generic.field[3])));
1464 		xhci_handle_cmd_reset_ep(xhci, slot_id, cmd_trb, cmd_comp_code);
1465 		break;
1466 	case TRB_RESET_DEV:
1467 		/* SLOT_ID field in reset device cmd completion event TRB is 0.
1468 		 * Use the SLOT_ID from the command TRB instead (xhci 4.6.11)
1469 		 */
1470 		slot_id = TRB_TO_SLOT_ID(
1471 				le32_to_cpu(cmd_trb->generic.field[3]));
1472 		xhci_handle_cmd_reset_dev(xhci, slot_id, event);
1473 		break;
1474 	case TRB_NEC_GET_FW:
1475 		xhci_handle_cmd_nec_get_fw(xhci, event);
1476 		break;
1477 	default:
1478 		/* Skip over unknown commands on the event ring */
1479 		xhci_info(xhci, "INFO unknown command type %d\n", cmd_type);
1480 		break;
1481 	}
1482 
1483 	/* restart timer if this wasn't the last command */
1484 	if (!list_is_singular(&xhci->cmd_list)) {
1485 		xhci->current_cmd = list_first_entry(&cmd->cmd_list,
1486 						struct xhci_command, cmd_list);
1487 		xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
1488 	} else if (xhci->current_cmd == cmd) {
1489 		xhci->current_cmd = NULL;
1490 	}
1491 
1492 event_handled:
1493 	xhci_complete_del_and_free_cmd(cmd, cmd_comp_code);
1494 
1495 	inc_deq(xhci, xhci->cmd_ring);
1496 }
1497 
1498 static void handle_vendor_event(struct xhci_hcd *xhci,
1499 		union xhci_trb *event)
1500 {
1501 	u32 trb_type;
1502 
1503 	trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event->generic.field[3]));
1504 	xhci_dbg(xhci, "Vendor specific event TRB type = %u\n", trb_type);
1505 	if (trb_type == TRB_NEC_CMD_COMP && (xhci->quirks & XHCI_NEC_HOST))
1506 		handle_cmd_completion(xhci, &event->event_cmd);
1507 }
1508 
1509 /* @port_id: the one-based port ID from the hardware (indexed from array of all
1510  * port registers -- USB 3.0 and USB 2.0).
1511  *
1512  * Returns a zero-based port number, which is suitable for indexing into each of
1513  * the split roothubs' port arrays and bus state arrays.
1514  * Add one to it in order to call xhci_find_slot_id_by_port.
1515  */
1516 static unsigned int find_faked_portnum_from_hw_portnum(struct usb_hcd *hcd,
1517 		struct xhci_hcd *xhci, u32 port_id)
1518 {
1519 	unsigned int i;
1520 	unsigned int num_similar_speed_ports = 0;
1521 
1522 	/* port_id from the hardware is 1-based, but port_array[], usb3_ports[],
1523 	 * and usb2_ports are 0-based indexes.  Count the number of similar
1524 	 * speed ports, up to 1 port before this port.
1525 	 */
1526 	for (i = 0; i < (port_id - 1); i++) {
1527 		u8 port_speed = xhci->port_array[i];
1528 
1529 		/*
1530 		 * Skip ports that don't have known speeds, or have duplicate
1531 		 * Extended Capabilities port speed entries.
1532 		 */
1533 		if (port_speed == 0 || port_speed == DUPLICATE_ENTRY)
1534 			continue;
1535 
1536 		/*
1537 		 * USB 3.0 ports are always under a USB 3.0 hub.  USB 2.0 and
1538 		 * 1.1 ports are under the USB 2.0 hub.  If the port speed
1539 		 * matches the device speed, it's a similar speed port.
1540 		 */
1541 		if ((port_speed == 0x03) == (hcd->speed >= HCD_USB3))
1542 			num_similar_speed_ports++;
1543 	}
1544 	return num_similar_speed_ports;
1545 }
1546 
1547 static void handle_device_notification(struct xhci_hcd *xhci,
1548 		union xhci_trb *event)
1549 {
1550 	u32 slot_id;
1551 	struct usb_device *udev;
1552 
1553 	slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->generic.field[3]));
1554 	if (!xhci->devs[slot_id]) {
1555 		xhci_warn(xhci, "Device Notification event for "
1556 				"unused slot %u\n", slot_id);
1557 		return;
1558 	}
1559 
1560 	xhci_dbg(xhci, "Device Wake Notification event for slot ID %u\n",
1561 			slot_id);
1562 	udev = xhci->devs[slot_id]->udev;
1563 	if (udev && udev->parent)
1564 		usb_wakeup_notification(udev->parent, udev->portnum);
1565 }
1566 
1567 static void handle_port_status(struct xhci_hcd *xhci,
1568 		union xhci_trb *event)
1569 {
1570 	struct usb_hcd *hcd;
1571 	u32 port_id;
1572 	u32 temp, temp1;
1573 	int max_ports;
1574 	int slot_id;
1575 	unsigned int faked_port_index;
1576 	u8 major_revision;
1577 	struct xhci_bus_state *bus_state;
1578 	__le32 __iomem **port_array;
1579 	bool bogus_port_status = false;
1580 
1581 	/* Port status change events always have a successful completion code */
1582 	if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS)
1583 		xhci_warn(xhci,
1584 			  "WARN: xHC returned failed port status event\n");
1585 
1586 	port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
1587 	xhci_dbg(xhci, "Port Status Change Event for port %d\n", port_id);
1588 
1589 	max_ports = HCS_MAX_PORTS(xhci->hcs_params1);
1590 	if ((port_id <= 0) || (port_id > max_ports)) {
1591 		xhci_warn(xhci, "Invalid port id %d\n", port_id);
1592 		inc_deq(xhci, xhci->event_ring);
1593 		return;
1594 	}
1595 
1596 	/* Figure out which usb_hcd this port is attached to:
1597 	 * is it a USB 3.0 port or a USB 2.0/1.1 port?
1598 	 */
1599 	major_revision = xhci->port_array[port_id - 1];
1600 
1601 	/* Find the right roothub. */
1602 	hcd = xhci_to_hcd(xhci);
1603 	if ((major_revision == 0x03) != (hcd->speed >= HCD_USB3))
1604 		hcd = xhci->shared_hcd;
1605 
1606 	if (major_revision == 0) {
1607 		xhci_warn(xhci, "Event for port %u not in "
1608 				"Extended Capabilities, ignoring.\n",
1609 				port_id);
1610 		bogus_port_status = true;
1611 		goto cleanup;
1612 	}
1613 	if (major_revision == DUPLICATE_ENTRY) {
1614 		xhci_warn(xhci, "Event for port %u duplicated in"
1615 				"Extended Capabilities, ignoring.\n",
1616 				port_id);
1617 		bogus_port_status = true;
1618 		goto cleanup;
1619 	}
1620 
1621 	/*
1622 	 * Hardware port IDs reported by a Port Status Change Event include USB
1623 	 * 3.0 and USB 2.0 ports.  We want to check if the port has reported a
1624 	 * resume event, but we first need to translate the hardware port ID
1625 	 * into the index into the ports on the correct split roothub, and the
1626 	 * correct bus_state structure.
1627 	 */
1628 	bus_state = &xhci->bus_state[hcd_index(hcd)];
1629 	if (hcd->speed >= HCD_USB3)
1630 		port_array = xhci->usb3_ports;
1631 	else
1632 		port_array = xhci->usb2_ports;
1633 	/* Find the faked port hub number */
1634 	faked_port_index = find_faked_portnum_from_hw_portnum(hcd, xhci,
1635 			port_id);
1636 
1637 	temp = readl(port_array[faked_port_index]);
1638 	if (hcd->state == HC_STATE_SUSPENDED) {
1639 		xhci_dbg(xhci, "resume root hub\n");
1640 		usb_hcd_resume_root_hub(hcd);
1641 	}
1642 
1643 	if (hcd->speed >= HCD_USB3 && (temp & PORT_PLS_MASK) == XDEV_INACTIVE)
1644 		bus_state->port_remote_wakeup &= ~(1 << faked_port_index);
1645 
1646 	if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_RESUME) {
1647 		xhci_dbg(xhci, "port resume event for port %d\n", port_id);
1648 
1649 		temp1 = readl(&xhci->op_regs->command);
1650 		if (!(temp1 & CMD_RUN)) {
1651 			xhci_warn(xhci, "xHC is not running.\n");
1652 			goto cleanup;
1653 		}
1654 
1655 		if (DEV_SUPERSPEED_ANY(temp)) {
1656 			xhci_dbg(xhci, "remote wake SS port %d\n", port_id);
1657 			/* Set a flag to say the port signaled remote wakeup,
1658 			 * so we can tell the difference between the end of
1659 			 * device and host initiated resume.
1660 			 */
1661 			bus_state->port_remote_wakeup |= 1 << faked_port_index;
1662 			xhci_test_and_clear_bit(xhci, port_array,
1663 					faked_port_index, PORT_PLC);
1664 			xhci_set_link_state(xhci, port_array, faked_port_index,
1665 						XDEV_U0);
1666 			/* Need to wait until the next link state change
1667 			 * indicates the device is actually in U0.
1668 			 */
1669 			bogus_port_status = true;
1670 			goto cleanup;
1671 		} else if (!test_bit(faked_port_index,
1672 				     &bus_state->resuming_ports)) {
1673 			xhci_dbg(xhci, "resume HS port %d\n", port_id);
1674 			bus_state->resume_done[faked_port_index] = jiffies +
1675 				msecs_to_jiffies(USB_RESUME_TIMEOUT);
1676 			set_bit(faked_port_index, &bus_state->resuming_ports);
1677 			mod_timer(&hcd->rh_timer,
1678 				  bus_state->resume_done[faked_port_index]);
1679 			/* Do the rest in GetPortStatus */
1680 		}
1681 	}
1682 
1683 	if ((temp & PORT_PLC) && (temp & PORT_PLS_MASK) == XDEV_U0 &&
1684 			DEV_SUPERSPEED_ANY(temp)) {
1685 		xhci_dbg(xhci, "resume SS port %d finished\n", port_id);
1686 		/* We've just brought the device into U0 through either the
1687 		 * Resume state after a device remote wakeup, or through the
1688 		 * U3Exit state after a host-initiated resume.  If it's a device
1689 		 * initiated remote wake, don't pass up the link state change,
1690 		 * so the roothub behavior is consistent with external
1691 		 * USB 3.0 hub behavior.
1692 		 */
1693 		slot_id = xhci_find_slot_id_by_port(hcd, xhci,
1694 				faked_port_index + 1);
1695 		if (slot_id && xhci->devs[slot_id])
1696 			xhci_ring_device(xhci, slot_id);
1697 		if (bus_state->port_remote_wakeup & (1 << faked_port_index)) {
1698 			bus_state->port_remote_wakeup &=
1699 				~(1 << faked_port_index);
1700 			xhci_test_and_clear_bit(xhci, port_array,
1701 					faked_port_index, PORT_PLC);
1702 			usb_wakeup_notification(hcd->self.root_hub,
1703 					faked_port_index + 1);
1704 			bogus_port_status = true;
1705 			goto cleanup;
1706 		}
1707 	}
1708 
1709 	/*
1710 	 * Check to see if xhci-hub.c is waiting on RExit to U0 transition (or
1711 	 * RExit to a disconnect state).  If so, let the the driver know it's
1712 	 * out of the RExit state.
1713 	 */
1714 	if (!DEV_SUPERSPEED_ANY(temp) &&
1715 			test_and_clear_bit(faked_port_index,
1716 				&bus_state->rexit_ports)) {
1717 		complete(&bus_state->rexit_done[faked_port_index]);
1718 		bogus_port_status = true;
1719 		goto cleanup;
1720 	}
1721 
1722 	if (hcd->speed < HCD_USB3)
1723 		xhci_test_and_clear_bit(xhci, port_array, faked_port_index,
1724 					PORT_PLC);
1725 
1726 cleanup:
1727 	/* Update event ring dequeue pointer before dropping the lock */
1728 	inc_deq(xhci, xhci->event_ring);
1729 
1730 	/* Don't make the USB core poll the roothub if we got a bad port status
1731 	 * change event.  Besides, at that point we can't tell which roothub
1732 	 * (USB 2.0 or USB 3.0) to kick.
1733 	 */
1734 	if (bogus_port_status)
1735 		return;
1736 
1737 	/*
1738 	 * xHCI port-status-change events occur when the "or" of all the
1739 	 * status-change bits in the portsc register changes from 0 to 1.
1740 	 * New status changes won't cause an event if any other change
1741 	 * bits are still set.  When an event occurs, switch over to
1742 	 * polling to avoid losing status changes.
1743 	 */
1744 	xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
1745 	set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
1746 	spin_unlock(&xhci->lock);
1747 	/* Pass this up to the core */
1748 	usb_hcd_poll_rh_status(hcd);
1749 	spin_lock(&xhci->lock);
1750 }
1751 
1752 /*
1753  * This TD is defined by the TRBs starting at start_trb in start_seg and ending
1754  * at end_trb, which may be in another segment.  If the suspect DMA address is a
1755  * TRB in this TD, this function returns that TRB's segment.  Otherwise it
1756  * returns 0.
1757  */
1758 struct xhci_segment *trb_in_td(struct xhci_hcd *xhci,
1759 		struct xhci_segment *start_seg,
1760 		union xhci_trb	*start_trb,
1761 		union xhci_trb	*end_trb,
1762 		dma_addr_t	suspect_dma,
1763 		bool		debug)
1764 {
1765 	dma_addr_t start_dma;
1766 	dma_addr_t end_seg_dma;
1767 	dma_addr_t end_trb_dma;
1768 	struct xhci_segment *cur_seg;
1769 
1770 	start_dma = xhci_trb_virt_to_dma(start_seg, start_trb);
1771 	cur_seg = start_seg;
1772 
1773 	do {
1774 		if (start_dma == 0)
1775 			return NULL;
1776 		/* We may get an event for a Link TRB in the middle of a TD */
1777 		end_seg_dma = xhci_trb_virt_to_dma(cur_seg,
1778 				&cur_seg->trbs[TRBS_PER_SEGMENT - 1]);
1779 		/* If the end TRB isn't in this segment, this is set to 0 */
1780 		end_trb_dma = xhci_trb_virt_to_dma(cur_seg, end_trb);
1781 
1782 		if (debug)
1783 			xhci_warn(xhci,
1784 				"Looking for event-dma %016llx trb-start %016llx trb-end %016llx seg-start %016llx seg-end %016llx\n",
1785 				(unsigned long long)suspect_dma,
1786 				(unsigned long long)start_dma,
1787 				(unsigned long long)end_trb_dma,
1788 				(unsigned long long)cur_seg->dma,
1789 				(unsigned long long)end_seg_dma);
1790 
1791 		if (end_trb_dma > 0) {
1792 			/* The end TRB is in this segment, so suspect should be here */
1793 			if (start_dma <= end_trb_dma) {
1794 				if (suspect_dma >= start_dma && suspect_dma <= end_trb_dma)
1795 					return cur_seg;
1796 			} else {
1797 				/* Case for one segment with
1798 				 * a TD wrapped around to the top
1799 				 */
1800 				if ((suspect_dma >= start_dma &&
1801 							suspect_dma <= end_seg_dma) ||
1802 						(suspect_dma >= cur_seg->dma &&
1803 						 suspect_dma <= end_trb_dma))
1804 					return cur_seg;
1805 			}
1806 			return NULL;
1807 		} else {
1808 			/* Might still be somewhere in this segment */
1809 			if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma)
1810 				return cur_seg;
1811 		}
1812 		cur_seg = cur_seg->next;
1813 		start_dma = xhci_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]);
1814 	} while (cur_seg != start_seg);
1815 
1816 	return NULL;
1817 }
1818 
1819 static void xhci_cleanup_halted_endpoint(struct xhci_hcd *xhci,
1820 		unsigned int slot_id, unsigned int ep_index,
1821 		unsigned int stream_id,
1822 		struct xhci_td *td, union xhci_trb *ep_trb,
1823 		enum xhci_ep_reset_type reset_type)
1824 {
1825 	struct xhci_virt_ep *ep = &xhci->devs[slot_id]->eps[ep_index];
1826 	struct xhci_command *command;
1827 	command = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
1828 	if (!command)
1829 		return;
1830 
1831 	ep->ep_state |= EP_HALTED;
1832 
1833 	xhci_queue_reset_ep(xhci, command, slot_id, ep_index, reset_type);
1834 
1835 	if (reset_type == EP_HARD_RESET)
1836 		xhci_cleanup_stalled_ring(xhci, ep_index, stream_id, td);
1837 
1838 	xhci_ring_cmd_db(xhci);
1839 }
1840 
1841 /* Check if an error has halted the endpoint ring.  The class driver will
1842  * cleanup the halt for a non-default control endpoint if we indicate a stall.
1843  * However, a babble and other errors also halt the endpoint ring, and the class
1844  * driver won't clear the halt in that case, so we need to issue a Set Transfer
1845  * Ring Dequeue Pointer command manually.
1846  */
1847 static int xhci_requires_manual_halt_cleanup(struct xhci_hcd *xhci,
1848 		struct xhci_ep_ctx *ep_ctx,
1849 		unsigned int trb_comp_code)
1850 {
1851 	/* TRB completion codes that may require a manual halt cleanup */
1852 	if (trb_comp_code == COMP_USB_TRANSACTION_ERROR ||
1853 			trb_comp_code == COMP_BABBLE_DETECTED_ERROR ||
1854 			trb_comp_code == COMP_SPLIT_TRANSACTION_ERROR)
1855 		/* The 0.95 spec says a babbling control endpoint
1856 		 * is not halted. The 0.96 spec says it is.  Some HW
1857 		 * claims to be 0.95 compliant, but it halts the control
1858 		 * endpoint anyway.  Check if a babble halted the
1859 		 * endpoint.
1860 		 */
1861 		if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_HALTED)
1862 			return 1;
1863 
1864 	return 0;
1865 }
1866 
1867 int xhci_is_vendor_info_code(struct xhci_hcd *xhci, unsigned int trb_comp_code)
1868 {
1869 	if (trb_comp_code >= 224 && trb_comp_code <= 255) {
1870 		/* Vendor defined "informational" completion code,
1871 		 * treat as not-an-error.
1872 		 */
1873 		xhci_dbg(xhci, "Vendor defined info completion code %u\n",
1874 				trb_comp_code);
1875 		xhci_dbg(xhci, "Treating code as success.\n");
1876 		return 1;
1877 	}
1878 	return 0;
1879 }
1880 
1881 static int xhci_td_cleanup(struct xhci_hcd *xhci, struct xhci_td *td,
1882 		struct xhci_ring *ep_ring, int *status)
1883 {
1884 	struct urb_priv	*urb_priv;
1885 	struct urb *urb = NULL;
1886 
1887 	/* Clean up the endpoint's TD list */
1888 	urb = td->urb;
1889 	urb_priv = urb->hcpriv;
1890 
1891 	/* if a bounce buffer was used to align this td then unmap it */
1892 	xhci_unmap_td_bounce_buffer(xhci, ep_ring, td);
1893 
1894 	/* Do one last check of the actual transfer length.
1895 	 * If the host controller said we transferred more data than the buffer
1896 	 * length, urb->actual_length will be a very big number (since it's
1897 	 * unsigned).  Play it safe and say we didn't transfer anything.
1898 	 */
1899 	if (urb->actual_length > urb->transfer_buffer_length) {
1900 		xhci_warn(xhci, "URB req %u and actual %u transfer length mismatch\n",
1901 			  urb->transfer_buffer_length, urb->actual_length);
1902 		urb->actual_length = 0;
1903 		*status = 0;
1904 	}
1905 	list_del_init(&td->td_list);
1906 	/* Was this TD slated to be cancelled but completed anyway? */
1907 	if (!list_empty(&td->cancelled_td_list))
1908 		list_del_init(&td->cancelled_td_list);
1909 
1910 	inc_td_cnt(urb);
1911 	/* Giveback the urb when all the tds are completed */
1912 	if (last_td_in_urb(td)) {
1913 		if ((urb->actual_length != urb->transfer_buffer_length &&
1914 		     (urb->transfer_flags & URB_SHORT_NOT_OK)) ||
1915 		    (*status != 0 && !usb_endpoint_xfer_isoc(&urb->ep->desc)))
1916 			xhci_dbg(xhci, "Giveback URB %p, len = %d, expected = %d, status = %d\n",
1917 				 urb, urb->actual_length,
1918 				 urb->transfer_buffer_length, *status);
1919 
1920 		/* set isoc urb status to 0 just as EHCI, UHCI, and OHCI */
1921 		if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS)
1922 			*status = 0;
1923 		xhci_giveback_urb_in_irq(xhci, td, *status);
1924 	}
1925 
1926 	return 0;
1927 }
1928 
1929 static int finish_td(struct xhci_hcd *xhci, struct xhci_td *td,
1930 	union xhci_trb *ep_trb, struct xhci_transfer_event *event,
1931 	struct xhci_virt_ep *ep, int *status)
1932 {
1933 	struct xhci_virt_device *xdev;
1934 	struct xhci_ep_ctx *ep_ctx;
1935 	struct xhci_ring *ep_ring;
1936 	unsigned int slot_id;
1937 	u32 trb_comp_code;
1938 	int ep_index;
1939 
1940 	slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
1941 	xdev = xhci->devs[slot_id];
1942 	ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1943 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1944 	ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
1945 	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1946 
1947 	if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
1948 			trb_comp_code == COMP_STOPPED ||
1949 			trb_comp_code == COMP_STOPPED_SHORT_PACKET) {
1950 		/* The Endpoint Stop Command completion will take care of any
1951 		 * stopped TDs.  A stopped TD may be restarted, so don't update
1952 		 * the ring dequeue pointer or take this TD off any lists yet.
1953 		 */
1954 		return 0;
1955 	}
1956 	if (trb_comp_code == COMP_STALL_ERROR ||
1957 		xhci_requires_manual_halt_cleanup(xhci, ep_ctx,
1958 						trb_comp_code)) {
1959 		/* Issue a reset endpoint command to clear the host side
1960 		 * halt, followed by a set dequeue command to move the
1961 		 * dequeue pointer past the TD.
1962 		 * The class driver clears the device side halt later.
1963 		 */
1964 		xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index,
1965 					ep_ring->stream_id, td, ep_trb,
1966 					EP_HARD_RESET);
1967 	} else {
1968 		/* Update ring dequeue pointer */
1969 		while (ep_ring->dequeue != td->last_trb)
1970 			inc_deq(xhci, ep_ring);
1971 		inc_deq(xhci, ep_ring);
1972 	}
1973 
1974 	return xhci_td_cleanup(xhci, td, ep_ring, status);
1975 }
1976 
1977 /* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */
1978 static int sum_trb_lengths(struct xhci_hcd *xhci, struct xhci_ring *ring,
1979 			   union xhci_trb *stop_trb)
1980 {
1981 	u32 sum;
1982 	union xhci_trb *trb = ring->dequeue;
1983 	struct xhci_segment *seg = ring->deq_seg;
1984 
1985 	for (sum = 0; trb != stop_trb; next_trb(xhci, ring, &seg, &trb)) {
1986 		if (!trb_is_noop(trb) && !trb_is_link(trb))
1987 			sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
1988 	}
1989 	return sum;
1990 }
1991 
1992 /*
1993  * Process control tds, update urb status and actual_length.
1994  */
1995 static int process_ctrl_td(struct xhci_hcd *xhci, struct xhci_td *td,
1996 	union xhci_trb *ep_trb, struct xhci_transfer_event *event,
1997 	struct xhci_virt_ep *ep, int *status)
1998 {
1999 	struct xhci_virt_device *xdev;
2000 	struct xhci_ring *ep_ring;
2001 	unsigned int slot_id;
2002 	int ep_index;
2003 	struct xhci_ep_ctx *ep_ctx;
2004 	u32 trb_comp_code;
2005 	u32 remaining, requested;
2006 	u32 trb_type;
2007 
2008 	trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(ep_trb->generic.field[3]));
2009 	slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2010 	xdev = xhci->devs[slot_id];
2011 	ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2012 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2013 	ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2014 	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2015 	requested = td->urb->transfer_buffer_length;
2016 	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2017 
2018 	switch (trb_comp_code) {
2019 	case COMP_SUCCESS:
2020 		if (trb_type != TRB_STATUS) {
2021 			xhci_warn(xhci, "WARN: Success on ctrl %s TRB without IOC set?\n",
2022 				  (trb_type == TRB_DATA) ? "data" : "setup");
2023 			*status = -ESHUTDOWN;
2024 			break;
2025 		}
2026 		*status = 0;
2027 		break;
2028 	case COMP_SHORT_PACKET:
2029 		*status = 0;
2030 		break;
2031 	case COMP_STOPPED_SHORT_PACKET:
2032 		if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
2033 			td->urb->actual_length = remaining;
2034 		else
2035 			xhci_warn(xhci, "WARN: Stopped Short Packet on ctrl setup or status TRB\n");
2036 		goto finish_td;
2037 	case COMP_STOPPED:
2038 		switch (trb_type) {
2039 		case TRB_SETUP:
2040 			td->urb->actual_length = 0;
2041 			goto finish_td;
2042 		case TRB_DATA:
2043 		case TRB_NORMAL:
2044 			td->urb->actual_length = requested - remaining;
2045 			goto finish_td;
2046 		case TRB_STATUS:
2047 			td->urb->actual_length = requested;
2048 			goto finish_td;
2049 		default:
2050 			xhci_warn(xhci, "WARN: unexpected TRB Type %d\n",
2051 				  trb_type);
2052 			goto finish_td;
2053 		}
2054 	case COMP_STOPPED_LENGTH_INVALID:
2055 		goto finish_td;
2056 	default:
2057 		if (!xhci_requires_manual_halt_cleanup(xhci,
2058 						       ep_ctx, trb_comp_code))
2059 			break;
2060 		xhci_dbg(xhci, "TRB error %u, halted endpoint index = %u\n",
2061 			 trb_comp_code, ep_index);
2062 		/* else fall through */
2063 	case COMP_STALL_ERROR:
2064 		/* Did we transfer part of the data (middle) phase? */
2065 		if (trb_type == TRB_DATA || trb_type == TRB_NORMAL)
2066 			td->urb->actual_length = requested - remaining;
2067 		else if (!td->urb_length_set)
2068 			td->urb->actual_length = 0;
2069 		goto finish_td;
2070 	}
2071 
2072 	/* stopped at setup stage, no data transferred */
2073 	if (trb_type == TRB_SETUP)
2074 		goto finish_td;
2075 
2076 	/*
2077 	 * if on data stage then update the actual_length of the URB and flag it
2078 	 * as set, so it won't be overwritten in the event for the last TRB.
2079 	 */
2080 	if (trb_type == TRB_DATA ||
2081 		trb_type == TRB_NORMAL) {
2082 		td->urb_length_set = true;
2083 		td->urb->actual_length = requested - remaining;
2084 		xhci_dbg(xhci, "Waiting for status stage event\n");
2085 		return 0;
2086 	}
2087 
2088 	/* at status stage */
2089 	if (!td->urb_length_set)
2090 		td->urb->actual_length = requested;
2091 
2092 finish_td:
2093 	return finish_td(xhci, td, ep_trb, event, ep, status);
2094 }
2095 
2096 /*
2097  * Process isochronous tds, update urb packet status and actual_length.
2098  */
2099 static int process_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2100 	union xhci_trb *ep_trb, struct xhci_transfer_event *event,
2101 	struct xhci_virt_ep *ep, int *status)
2102 {
2103 	struct xhci_ring *ep_ring;
2104 	struct urb_priv *urb_priv;
2105 	int idx;
2106 	struct usb_iso_packet_descriptor *frame;
2107 	u32 trb_comp_code;
2108 	bool sum_trbs_for_length = false;
2109 	u32 remaining, requested, ep_trb_len;
2110 	int short_framestatus;
2111 
2112 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2113 	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2114 	urb_priv = td->urb->hcpriv;
2115 	idx = urb_priv->num_tds_done;
2116 	frame = &td->urb->iso_frame_desc[idx];
2117 	requested = frame->length;
2118 	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2119 	ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
2120 	short_framestatus = td->urb->transfer_flags & URB_SHORT_NOT_OK ?
2121 		-EREMOTEIO : 0;
2122 
2123 	/* handle completion code */
2124 	switch (trb_comp_code) {
2125 	case COMP_SUCCESS:
2126 		if (remaining) {
2127 			frame->status = short_framestatus;
2128 			if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
2129 				sum_trbs_for_length = true;
2130 			break;
2131 		}
2132 		frame->status = 0;
2133 		break;
2134 	case COMP_SHORT_PACKET:
2135 		frame->status = short_framestatus;
2136 		sum_trbs_for_length = true;
2137 		break;
2138 	case COMP_BANDWIDTH_OVERRUN_ERROR:
2139 		frame->status = -ECOMM;
2140 		break;
2141 	case COMP_ISOCH_BUFFER_OVERRUN:
2142 	case COMP_BABBLE_DETECTED_ERROR:
2143 		frame->status = -EOVERFLOW;
2144 		break;
2145 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
2146 	case COMP_STALL_ERROR:
2147 		frame->status = -EPROTO;
2148 		break;
2149 	case COMP_USB_TRANSACTION_ERROR:
2150 		frame->status = -EPROTO;
2151 		if (ep_trb != td->last_trb)
2152 			return 0;
2153 		break;
2154 	case COMP_STOPPED:
2155 		sum_trbs_for_length = true;
2156 		break;
2157 	case COMP_STOPPED_SHORT_PACKET:
2158 		/* field normally containing residue now contains tranferred */
2159 		frame->status = short_framestatus;
2160 		requested = remaining;
2161 		break;
2162 	case COMP_STOPPED_LENGTH_INVALID:
2163 		requested = 0;
2164 		remaining = 0;
2165 		break;
2166 	default:
2167 		sum_trbs_for_length = true;
2168 		frame->status = -1;
2169 		break;
2170 	}
2171 
2172 	if (sum_trbs_for_length)
2173 		frame->actual_length = sum_trb_lengths(xhci, ep_ring, ep_trb) +
2174 			ep_trb_len - remaining;
2175 	else
2176 		frame->actual_length = requested;
2177 
2178 	td->urb->actual_length += frame->actual_length;
2179 
2180 	return finish_td(xhci, td, ep_trb, event, ep, status);
2181 }
2182 
2183 static int skip_isoc_td(struct xhci_hcd *xhci, struct xhci_td *td,
2184 			struct xhci_transfer_event *event,
2185 			struct xhci_virt_ep *ep, int *status)
2186 {
2187 	struct xhci_ring *ep_ring;
2188 	struct urb_priv *urb_priv;
2189 	struct usb_iso_packet_descriptor *frame;
2190 	int idx;
2191 
2192 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2193 	urb_priv = td->urb->hcpriv;
2194 	idx = urb_priv->num_tds_done;
2195 	frame = &td->urb->iso_frame_desc[idx];
2196 
2197 	/* The transfer is partly done. */
2198 	frame->status = -EXDEV;
2199 
2200 	/* calc actual length */
2201 	frame->actual_length = 0;
2202 
2203 	/* Update ring dequeue pointer */
2204 	while (ep_ring->dequeue != td->last_trb)
2205 		inc_deq(xhci, ep_ring);
2206 	inc_deq(xhci, ep_ring);
2207 
2208 	return xhci_td_cleanup(xhci, td, ep_ring, status);
2209 }
2210 
2211 /*
2212  * Process bulk and interrupt tds, update urb status and actual_length.
2213  */
2214 static int process_bulk_intr_td(struct xhci_hcd *xhci, struct xhci_td *td,
2215 	union xhci_trb *ep_trb, struct xhci_transfer_event *event,
2216 	struct xhci_virt_ep *ep, int *status)
2217 {
2218 	struct xhci_ring *ep_ring;
2219 	u32 trb_comp_code;
2220 	u32 remaining, requested, ep_trb_len;
2221 
2222 	ep_ring = xhci_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
2223 	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2224 	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
2225 	ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
2226 	requested = td->urb->transfer_buffer_length;
2227 
2228 	switch (trb_comp_code) {
2229 	case COMP_SUCCESS:
2230 		/* handle success with untransferred data as short packet */
2231 		if (ep_trb != td->last_trb || remaining) {
2232 			xhci_warn(xhci, "WARN Successful completion on short TX\n");
2233 			xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2234 				 td->urb->ep->desc.bEndpointAddress,
2235 				 requested, remaining);
2236 		}
2237 		*status = 0;
2238 		break;
2239 	case COMP_SHORT_PACKET:
2240 		xhci_dbg(xhci, "ep %#x - asked for %d bytes, %d bytes untransferred\n",
2241 			 td->urb->ep->desc.bEndpointAddress,
2242 			 requested, remaining);
2243 		*status = 0;
2244 		break;
2245 	case COMP_STOPPED_SHORT_PACKET:
2246 		td->urb->actual_length = remaining;
2247 		goto finish_td;
2248 	case COMP_STOPPED_LENGTH_INVALID:
2249 		/* stopped on ep trb with invalid length, exclude it */
2250 		ep_trb_len	= 0;
2251 		remaining	= 0;
2252 		break;
2253 	default:
2254 		/* do nothing */
2255 		break;
2256 	}
2257 
2258 	if (ep_trb == td->last_trb)
2259 		td->urb->actual_length = requested - remaining;
2260 	else
2261 		td->urb->actual_length =
2262 			sum_trb_lengths(xhci, ep_ring, ep_trb) +
2263 			ep_trb_len - remaining;
2264 finish_td:
2265 	if (remaining > requested) {
2266 		xhci_warn(xhci, "bad transfer trb length %d in event trb\n",
2267 			  remaining);
2268 		td->urb->actual_length = 0;
2269 	}
2270 	return finish_td(xhci, td, ep_trb, event, ep, status);
2271 }
2272 
2273 /*
2274  * If this function returns an error condition, it means it got a Transfer
2275  * event with a corrupted Slot ID, Endpoint ID, or TRB DMA address.
2276  * At this point, the host controller is probably hosed and should be reset.
2277  */
2278 static int handle_tx_event(struct xhci_hcd *xhci,
2279 		struct xhci_transfer_event *event)
2280 {
2281 	struct xhci_virt_device *xdev;
2282 	struct xhci_virt_ep *ep;
2283 	struct xhci_ring *ep_ring;
2284 	unsigned int slot_id;
2285 	int ep_index;
2286 	struct xhci_td *td = NULL;
2287 	dma_addr_t ep_trb_dma;
2288 	struct xhci_segment *ep_seg;
2289 	union xhci_trb *ep_trb;
2290 	int status = -EINPROGRESS;
2291 	struct xhci_ep_ctx *ep_ctx;
2292 	struct list_head *tmp;
2293 	u32 trb_comp_code;
2294 	int td_num = 0;
2295 	bool handling_skipped_tds = false;
2296 
2297 	slot_id = TRB_TO_SLOT_ID(le32_to_cpu(event->flags));
2298 	ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
2299 	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
2300 	ep_trb_dma = le64_to_cpu(event->buffer);
2301 
2302 	xdev = xhci->devs[slot_id];
2303 	if (!xdev) {
2304 		xhci_err(xhci, "ERROR Transfer event pointed to bad slot %u\n",
2305 			 slot_id);
2306 		goto err_out;
2307 	}
2308 
2309 	ep = &xdev->eps[ep_index];
2310 	ep_ring = xhci_dma_to_transfer_ring(ep, ep_trb_dma);
2311 	ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2312 
2313 	if (GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) {
2314 		xhci_err(xhci,
2315 			 "ERROR Transfer event for disabled endpoint slot %u ep %u\n",
2316 			  slot_id, ep_index);
2317 		goto err_out;
2318 	}
2319 
2320 	/* Some transfer events don't always point to a trb, see xhci 4.17.4 */
2321 	if (!ep_ring) {
2322 		switch (trb_comp_code) {
2323 		case COMP_STALL_ERROR:
2324 		case COMP_USB_TRANSACTION_ERROR:
2325 		case COMP_INVALID_STREAM_TYPE_ERROR:
2326 		case COMP_INVALID_STREAM_ID_ERROR:
2327 			xhci_cleanup_halted_endpoint(xhci, slot_id, ep_index, 0,
2328 						     NULL, NULL, EP_SOFT_RESET);
2329 			goto cleanup;
2330 		case COMP_RING_UNDERRUN:
2331 		case COMP_RING_OVERRUN:
2332 			goto cleanup;
2333 		default:
2334 			xhci_err(xhci, "ERROR Transfer event for unknown stream ring slot %u ep %u\n",
2335 				 slot_id, ep_index);
2336 			goto err_out;
2337 		}
2338 	}
2339 
2340 	/* Count current td numbers if ep->skip is set */
2341 	if (ep->skip) {
2342 		list_for_each(tmp, &ep_ring->td_list)
2343 			td_num++;
2344 	}
2345 
2346 	/* Look for common error cases */
2347 	switch (trb_comp_code) {
2348 	/* Skip codes that require special handling depending on
2349 	 * transfer type
2350 	 */
2351 	case COMP_SUCCESS:
2352 		if (EVENT_TRB_LEN(le32_to_cpu(event->transfer_len)) == 0)
2353 			break;
2354 		if (xhci->quirks & XHCI_TRUST_TX_LENGTH)
2355 			trb_comp_code = COMP_SHORT_PACKET;
2356 		else
2357 			xhci_warn_ratelimited(xhci,
2358 					      "WARN Successful completion on short TX for slot %u ep %u: needs XHCI_TRUST_TX_LENGTH quirk?\n",
2359 					      slot_id, ep_index);
2360 	case COMP_SHORT_PACKET:
2361 		break;
2362 	/* Completion codes for endpoint stopped state */
2363 	case COMP_STOPPED:
2364 		xhci_dbg(xhci, "Stopped on Transfer TRB for slot %u ep %u\n",
2365 			 slot_id, ep_index);
2366 		break;
2367 	case COMP_STOPPED_LENGTH_INVALID:
2368 		xhci_dbg(xhci,
2369 			 "Stopped on No-op or Link TRB for slot %u ep %u\n",
2370 			 slot_id, ep_index);
2371 		break;
2372 	case COMP_STOPPED_SHORT_PACKET:
2373 		xhci_dbg(xhci,
2374 			 "Stopped with short packet transfer detected for slot %u ep %u\n",
2375 			 slot_id, ep_index);
2376 		break;
2377 	/* Completion codes for endpoint halted state */
2378 	case COMP_STALL_ERROR:
2379 		xhci_dbg(xhci, "Stalled endpoint for slot %u ep %u\n", slot_id,
2380 			 ep_index);
2381 		ep->ep_state |= EP_HALTED;
2382 		status = -EPIPE;
2383 		break;
2384 	case COMP_SPLIT_TRANSACTION_ERROR:
2385 	case COMP_USB_TRANSACTION_ERROR:
2386 		xhci_dbg(xhci, "Transfer error for slot %u ep %u on endpoint\n",
2387 			 slot_id, ep_index);
2388 		status = -EPROTO;
2389 		break;
2390 	case COMP_BABBLE_DETECTED_ERROR:
2391 		xhci_dbg(xhci, "Babble error for slot %u ep %u on endpoint\n",
2392 			 slot_id, ep_index);
2393 		status = -EOVERFLOW;
2394 		break;
2395 	/* Completion codes for endpoint error state */
2396 	case COMP_TRB_ERROR:
2397 		xhci_warn(xhci,
2398 			  "WARN: TRB error for slot %u ep %u on endpoint\n",
2399 			  slot_id, ep_index);
2400 		status = -EILSEQ;
2401 		break;
2402 	/* completion codes not indicating endpoint state change */
2403 	case COMP_DATA_BUFFER_ERROR:
2404 		xhci_warn(xhci,
2405 			  "WARN: HC couldn't access mem fast enough for slot %u ep %u\n",
2406 			  slot_id, ep_index);
2407 		status = -ENOSR;
2408 		break;
2409 	case COMP_BANDWIDTH_OVERRUN_ERROR:
2410 		xhci_warn(xhci,
2411 			  "WARN: bandwidth overrun event for slot %u ep %u on endpoint\n",
2412 			  slot_id, ep_index);
2413 		break;
2414 	case COMP_ISOCH_BUFFER_OVERRUN:
2415 		xhci_warn(xhci,
2416 			  "WARN: buffer overrun event for slot %u ep %u on endpoint",
2417 			  slot_id, ep_index);
2418 		break;
2419 	case COMP_RING_UNDERRUN:
2420 		/*
2421 		 * When the Isoch ring is empty, the xHC will generate
2422 		 * a Ring Overrun Event for IN Isoch endpoint or Ring
2423 		 * Underrun Event for OUT Isoch endpoint.
2424 		 */
2425 		xhci_dbg(xhci, "underrun event on endpoint\n");
2426 		if (!list_empty(&ep_ring->td_list))
2427 			xhci_dbg(xhci, "Underrun Event for slot %d ep %d "
2428 					"still with TDs queued?\n",
2429 				 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2430 				 ep_index);
2431 		goto cleanup;
2432 	case COMP_RING_OVERRUN:
2433 		xhci_dbg(xhci, "overrun event on endpoint\n");
2434 		if (!list_empty(&ep_ring->td_list))
2435 			xhci_dbg(xhci, "Overrun Event for slot %d ep %d "
2436 					"still with TDs queued?\n",
2437 				 TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2438 				 ep_index);
2439 		goto cleanup;
2440 	case COMP_MISSED_SERVICE_ERROR:
2441 		/*
2442 		 * When encounter missed service error, one or more isoc tds
2443 		 * may be missed by xHC.
2444 		 * Set skip flag of the ep_ring; Complete the missed tds as
2445 		 * short transfer when process the ep_ring next time.
2446 		 */
2447 		ep->skip = true;
2448 		xhci_dbg(xhci,
2449 			 "Miss service interval error for slot %u ep %u, set skip flag\n",
2450 			 slot_id, ep_index);
2451 		goto cleanup;
2452 	case COMP_NO_PING_RESPONSE_ERROR:
2453 		ep->skip = true;
2454 		xhci_dbg(xhci,
2455 			 "No Ping response error for slot %u ep %u, Skip one Isoc TD\n",
2456 			 slot_id, ep_index);
2457 		goto cleanup;
2458 
2459 	case COMP_INCOMPATIBLE_DEVICE_ERROR:
2460 		/* needs disable slot command to recover */
2461 		xhci_warn(xhci,
2462 			  "WARN: detect an incompatible device for slot %u ep %u",
2463 			  slot_id, ep_index);
2464 		status = -EPROTO;
2465 		break;
2466 	default:
2467 		if (xhci_is_vendor_info_code(xhci, trb_comp_code)) {
2468 			status = 0;
2469 			break;
2470 		}
2471 		xhci_warn(xhci,
2472 			  "ERROR Unknown event condition %u for slot %u ep %u , HC probably busted\n",
2473 			  trb_comp_code, slot_id, ep_index);
2474 		goto cleanup;
2475 	}
2476 
2477 	do {
2478 		/* This TRB should be in the TD at the head of this ring's
2479 		 * TD list.
2480 		 */
2481 		if (list_empty(&ep_ring->td_list)) {
2482 			/*
2483 			 * A stopped endpoint may generate an extra completion
2484 			 * event if the device was suspended.  Don't print
2485 			 * warnings.
2486 			 */
2487 			if (!(trb_comp_code == COMP_STOPPED ||
2488 				trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) {
2489 				xhci_warn(xhci, "WARN Event TRB for slot %d ep %d with no TDs queued?\n",
2490 						TRB_TO_SLOT_ID(le32_to_cpu(event->flags)),
2491 						ep_index);
2492 			}
2493 			if (ep->skip) {
2494 				ep->skip = false;
2495 				xhci_dbg(xhci, "td_list is empty while skip flag set. Clear skip flag for slot %u ep %u.\n",
2496 					 slot_id, ep_index);
2497 			}
2498 			goto cleanup;
2499 		}
2500 
2501 		/* We've skipped all the TDs on the ep ring when ep->skip set */
2502 		if (ep->skip && td_num == 0) {
2503 			ep->skip = false;
2504 			xhci_dbg(xhci, "All tds on the ep_ring skipped. Clear skip flag for slot %u ep %u.\n",
2505 				 slot_id, ep_index);
2506 			goto cleanup;
2507 		}
2508 
2509 		td = list_first_entry(&ep_ring->td_list, struct xhci_td,
2510 				      td_list);
2511 		if (ep->skip)
2512 			td_num--;
2513 
2514 		/* Is this a TRB in the currently executing TD? */
2515 		ep_seg = trb_in_td(xhci, ep_ring->deq_seg, ep_ring->dequeue,
2516 				td->last_trb, ep_trb_dma, false);
2517 
2518 		/*
2519 		 * Skip the Force Stopped Event. The event_trb(event_dma) of FSE
2520 		 * is not in the current TD pointed by ep_ring->dequeue because
2521 		 * that the hardware dequeue pointer still at the previous TRB
2522 		 * of the current TD. The previous TRB maybe a Link TD or the
2523 		 * last TRB of the previous TD. The command completion handle
2524 		 * will take care the rest.
2525 		 */
2526 		if (!ep_seg && (trb_comp_code == COMP_STOPPED ||
2527 			   trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) {
2528 			goto cleanup;
2529 		}
2530 
2531 		if (!ep_seg) {
2532 			if (!ep->skip ||
2533 			    !usb_endpoint_xfer_isoc(&td->urb->ep->desc)) {
2534 				/* Some host controllers give a spurious
2535 				 * successful event after a short transfer.
2536 				 * Ignore it.
2537 				 */
2538 				if ((xhci->quirks & XHCI_SPURIOUS_SUCCESS) &&
2539 						ep_ring->last_td_was_short) {
2540 					ep_ring->last_td_was_short = false;
2541 					goto cleanup;
2542 				}
2543 				/* HC is busted, give up! */
2544 				xhci_err(xhci,
2545 					"ERROR Transfer event TRB DMA ptr not "
2546 					"part of current TD ep_index %d "
2547 					"comp_code %u\n", ep_index,
2548 					trb_comp_code);
2549 				trb_in_td(xhci, ep_ring->deq_seg,
2550 					  ep_ring->dequeue, td->last_trb,
2551 					  ep_trb_dma, true);
2552 				return -ESHUTDOWN;
2553 			}
2554 
2555 			skip_isoc_td(xhci, td, event, ep, &status);
2556 			goto cleanup;
2557 		}
2558 		if (trb_comp_code == COMP_SHORT_PACKET)
2559 			ep_ring->last_td_was_short = true;
2560 		else
2561 			ep_ring->last_td_was_short = false;
2562 
2563 		if (ep->skip) {
2564 			xhci_dbg(xhci,
2565 				 "Found td. Clear skip flag for slot %u ep %u.\n",
2566 				 slot_id, ep_index);
2567 			ep->skip = false;
2568 		}
2569 
2570 		ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma) /
2571 						sizeof(*ep_trb)];
2572 
2573 		trace_xhci_handle_transfer(ep_ring,
2574 				(struct xhci_generic_trb *) ep_trb);
2575 
2576 		/*
2577 		 * No-op TRB should not trigger interrupts.
2578 		 * If ep_trb is a no-op TRB, it means the
2579 		 * corresponding TD has been cancelled. Just ignore
2580 		 * the TD.
2581 		 */
2582 		if (trb_is_noop(ep_trb)) {
2583 			xhci_dbg(xhci,
2584 				 "ep_trb is a no-op TRB. Skip it for slot %u ep %u\n",
2585 				 slot_id, ep_index);
2586 			goto cleanup;
2587 		}
2588 
2589 		/* update the urb's actual_length and give back to the core */
2590 		if (usb_endpoint_xfer_control(&td->urb->ep->desc))
2591 			process_ctrl_td(xhci, td, ep_trb, event, ep, &status);
2592 		else if (usb_endpoint_xfer_isoc(&td->urb->ep->desc))
2593 			process_isoc_td(xhci, td, ep_trb, event, ep, &status);
2594 		else
2595 			process_bulk_intr_td(xhci, td, ep_trb, event, ep,
2596 					     &status);
2597 cleanup:
2598 		handling_skipped_tds = ep->skip &&
2599 			trb_comp_code != COMP_MISSED_SERVICE_ERROR &&
2600 			trb_comp_code != COMP_NO_PING_RESPONSE_ERROR;
2601 
2602 		/*
2603 		 * Do not update event ring dequeue pointer if we're in a loop
2604 		 * processing missed tds.
2605 		 */
2606 		if (!handling_skipped_tds)
2607 			inc_deq(xhci, xhci->event_ring);
2608 
2609 	/*
2610 	 * If ep->skip is set, it means there are missed tds on the
2611 	 * endpoint ring need to take care of.
2612 	 * Process them as short transfer until reach the td pointed by
2613 	 * the event.
2614 	 */
2615 	} while (handling_skipped_tds);
2616 
2617 	return 0;
2618 
2619 err_out:
2620 	xhci_err(xhci, "@%016llx %08x %08x %08x %08x\n",
2621 		 (unsigned long long) xhci_trb_virt_to_dma(
2622 			 xhci->event_ring->deq_seg,
2623 			 xhci->event_ring->dequeue),
2624 		 lower_32_bits(le64_to_cpu(event->buffer)),
2625 		 upper_32_bits(le64_to_cpu(event->buffer)),
2626 		 le32_to_cpu(event->transfer_len),
2627 		 le32_to_cpu(event->flags));
2628 	return -ENODEV;
2629 }
2630 
2631 /*
2632  * This function handles all OS-owned events on the event ring.  It may drop
2633  * xhci->lock between event processing (e.g. to pass up port status changes).
2634  * Returns >0 for "possibly more events to process" (caller should call again),
2635  * otherwise 0 if done.  In future, <0 returns should indicate error code.
2636  */
2637 static int xhci_handle_event(struct xhci_hcd *xhci)
2638 {
2639 	union xhci_trb *event;
2640 	int update_ptrs = 1;
2641 	int ret;
2642 
2643 	/* Event ring hasn't been allocated yet. */
2644 	if (!xhci->event_ring || !xhci->event_ring->dequeue) {
2645 		xhci_err(xhci, "ERROR event ring not ready\n");
2646 		return -ENOMEM;
2647 	}
2648 
2649 	event = xhci->event_ring->dequeue;
2650 	/* Does the HC or OS own the TRB? */
2651 	if ((le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE) !=
2652 	    xhci->event_ring->cycle_state)
2653 		return 0;
2654 
2655 	trace_xhci_handle_event(xhci->event_ring, &event->generic);
2656 
2657 	/*
2658 	 * Barrier between reading the TRB_CYCLE (valid) flag above and any
2659 	 * speculative reads of the event's flags/data below.
2660 	 */
2661 	rmb();
2662 	/* FIXME: Handle more event types. */
2663 	switch (le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) {
2664 	case TRB_TYPE(TRB_COMPLETION):
2665 		handle_cmd_completion(xhci, &event->event_cmd);
2666 		break;
2667 	case TRB_TYPE(TRB_PORT_STATUS):
2668 		handle_port_status(xhci, event);
2669 		update_ptrs = 0;
2670 		break;
2671 	case TRB_TYPE(TRB_TRANSFER):
2672 		ret = handle_tx_event(xhci, &event->trans_event);
2673 		if (ret >= 0)
2674 			update_ptrs = 0;
2675 		break;
2676 	case TRB_TYPE(TRB_DEV_NOTE):
2677 		handle_device_notification(xhci, event);
2678 		break;
2679 	default:
2680 		if ((le32_to_cpu(event->event_cmd.flags) & TRB_TYPE_BITMASK) >=
2681 		    TRB_TYPE(48))
2682 			handle_vendor_event(xhci, event);
2683 		else
2684 			xhci_warn(xhci, "ERROR unknown event type %d\n",
2685 				  TRB_FIELD_TO_TYPE(
2686 				  le32_to_cpu(event->event_cmd.flags)));
2687 	}
2688 	/* Any of the above functions may drop and re-acquire the lock, so check
2689 	 * to make sure a watchdog timer didn't mark the host as non-responsive.
2690 	 */
2691 	if (xhci->xhc_state & XHCI_STATE_DYING) {
2692 		xhci_dbg(xhci, "xHCI host dying, returning from "
2693 				"event handler.\n");
2694 		return 0;
2695 	}
2696 
2697 	if (update_ptrs)
2698 		/* Update SW event ring dequeue pointer */
2699 		inc_deq(xhci, xhci->event_ring);
2700 
2701 	/* Are there more items on the event ring?  Caller will call us again to
2702 	 * check.
2703 	 */
2704 	return 1;
2705 }
2706 
2707 /*
2708  * xHCI spec says we can get an interrupt, and if the HC has an error condition,
2709  * we might get bad data out of the event ring.  Section 4.10.2.7 has a list of
2710  * indicators of an event TRB error, but we check the status *first* to be safe.
2711  */
2712 irqreturn_t xhci_irq(struct usb_hcd *hcd)
2713 {
2714 	struct xhci_hcd *xhci = hcd_to_xhci(hcd);
2715 	union xhci_trb *event_ring_deq;
2716 	irqreturn_t ret = IRQ_NONE;
2717 	unsigned long flags;
2718 	dma_addr_t deq;
2719 	u64 temp_64;
2720 	u32 status;
2721 
2722 	spin_lock_irqsave(&xhci->lock, flags);
2723 	/* Check if the xHC generated the interrupt, or the irq is shared */
2724 	status = readl(&xhci->op_regs->status);
2725 	if (status == ~(u32)0) {
2726 		xhci_hc_died(xhci);
2727 		ret = IRQ_HANDLED;
2728 		goto out;
2729 	}
2730 
2731 	if (!(status & STS_EINT))
2732 		goto out;
2733 
2734 	if (status & STS_FATAL) {
2735 		xhci_warn(xhci, "WARNING: Host System Error\n");
2736 		xhci_halt(xhci);
2737 		ret = IRQ_HANDLED;
2738 		goto out;
2739 	}
2740 
2741 	/*
2742 	 * Clear the op reg interrupt status first,
2743 	 * so we can receive interrupts from other MSI-X interrupters.
2744 	 * Write 1 to clear the interrupt status.
2745 	 */
2746 	status |= STS_EINT;
2747 	writel(status, &xhci->op_regs->status);
2748 
2749 	if (!hcd->msi_enabled) {
2750 		u32 irq_pending;
2751 		irq_pending = readl(&xhci->ir_set->irq_pending);
2752 		irq_pending |= IMAN_IP;
2753 		writel(irq_pending, &xhci->ir_set->irq_pending);
2754 	}
2755 
2756 	if (xhci->xhc_state & XHCI_STATE_DYING ||
2757 	    xhci->xhc_state & XHCI_STATE_HALTED) {
2758 		xhci_dbg(xhci, "xHCI dying, ignoring interrupt. "
2759 				"Shouldn't IRQs be disabled?\n");
2760 		/* Clear the event handler busy flag (RW1C);
2761 		 * the event ring should be empty.
2762 		 */
2763 		temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2764 		xhci_write_64(xhci, temp_64 | ERST_EHB,
2765 				&xhci->ir_set->erst_dequeue);
2766 		ret = IRQ_HANDLED;
2767 		goto out;
2768 	}
2769 
2770 	event_ring_deq = xhci->event_ring->dequeue;
2771 	/* FIXME this should be a delayed service routine
2772 	 * that clears the EHB.
2773 	 */
2774 	while (xhci_handle_event(xhci) > 0) {}
2775 
2776 	temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
2777 	/* If necessary, update the HW's version of the event ring deq ptr. */
2778 	if (event_ring_deq != xhci->event_ring->dequeue) {
2779 		deq = xhci_trb_virt_to_dma(xhci->event_ring->deq_seg,
2780 				xhci->event_ring->dequeue);
2781 		if (deq == 0)
2782 			xhci_warn(xhci, "WARN something wrong with SW event "
2783 					"ring dequeue ptr.\n");
2784 		/* Update HC event ring dequeue pointer */
2785 		temp_64 &= ERST_PTR_MASK;
2786 		temp_64 |= ((u64) deq & (u64) ~ERST_PTR_MASK);
2787 	}
2788 
2789 	/* Clear the event handler busy flag (RW1C); event ring is empty. */
2790 	temp_64 |= ERST_EHB;
2791 	xhci_write_64(xhci, temp_64, &xhci->ir_set->erst_dequeue);
2792 	ret = IRQ_HANDLED;
2793 
2794 out:
2795 	spin_unlock_irqrestore(&xhci->lock, flags);
2796 
2797 	return ret;
2798 }
2799 
2800 irqreturn_t xhci_msi_irq(int irq, void *hcd)
2801 {
2802 	return xhci_irq(hcd);
2803 }
2804 
2805 /****		Endpoint Ring Operations	****/
2806 
2807 /*
2808  * Generic function for queueing a TRB on a ring.
2809  * The caller must have checked to make sure there's room on the ring.
2810  *
2811  * @more_trbs_coming:	Will you enqueue more TRBs before calling
2812  *			prepare_transfer()?
2813  */
2814 static void queue_trb(struct xhci_hcd *xhci, struct xhci_ring *ring,
2815 		bool more_trbs_coming,
2816 		u32 field1, u32 field2, u32 field3, u32 field4)
2817 {
2818 	struct xhci_generic_trb *trb;
2819 
2820 	trb = &ring->enqueue->generic;
2821 	trb->field[0] = cpu_to_le32(field1);
2822 	trb->field[1] = cpu_to_le32(field2);
2823 	trb->field[2] = cpu_to_le32(field3);
2824 	trb->field[3] = cpu_to_le32(field4);
2825 
2826 	trace_xhci_queue_trb(ring, trb);
2827 
2828 	inc_enq(xhci, ring, more_trbs_coming);
2829 }
2830 
2831 /*
2832  * Does various checks on the endpoint ring, and makes it ready to queue num_trbs.
2833  * FIXME allocate segments if the ring is full.
2834  */
2835 static int prepare_ring(struct xhci_hcd *xhci, struct xhci_ring *ep_ring,
2836 		u32 ep_state, unsigned int num_trbs, gfp_t mem_flags)
2837 {
2838 	unsigned int num_trbs_needed;
2839 
2840 	/* Make sure the endpoint has been added to xHC schedule */
2841 	switch (ep_state) {
2842 	case EP_STATE_DISABLED:
2843 		/*
2844 		 * USB core changed config/interfaces without notifying us,
2845 		 * or hardware is reporting the wrong state.
2846 		 */
2847 		xhci_warn(xhci, "WARN urb submitted to disabled ep\n");
2848 		return -ENOENT;
2849 	case EP_STATE_ERROR:
2850 		xhci_warn(xhci, "WARN waiting for error on ep to be cleared\n");
2851 		/* FIXME event handling code for error needs to clear it */
2852 		/* XXX not sure if this should be -ENOENT or not */
2853 		return -EINVAL;
2854 	case EP_STATE_HALTED:
2855 		xhci_dbg(xhci, "WARN halted endpoint, queueing URB anyway.\n");
2856 	case EP_STATE_STOPPED:
2857 	case EP_STATE_RUNNING:
2858 		break;
2859 	default:
2860 		xhci_err(xhci, "ERROR unknown endpoint state for ep\n");
2861 		/*
2862 		 * FIXME issue Configure Endpoint command to try to get the HC
2863 		 * back into a known state.
2864 		 */
2865 		return -EINVAL;
2866 	}
2867 
2868 	while (1) {
2869 		if (room_on_ring(xhci, ep_ring, num_trbs))
2870 			break;
2871 
2872 		if (ep_ring == xhci->cmd_ring) {
2873 			xhci_err(xhci, "Do not support expand command ring\n");
2874 			return -ENOMEM;
2875 		}
2876 
2877 		xhci_dbg_trace(xhci, trace_xhci_dbg_ring_expansion,
2878 				"ERROR no room on ep ring, try ring expansion");
2879 		num_trbs_needed = num_trbs - ep_ring->num_trbs_free;
2880 		if (xhci_ring_expansion(xhci, ep_ring, num_trbs_needed,
2881 					mem_flags)) {
2882 			xhci_err(xhci, "Ring expansion failed\n");
2883 			return -ENOMEM;
2884 		}
2885 	}
2886 
2887 	while (trb_is_link(ep_ring->enqueue)) {
2888 		/* If we're not dealing with 0.95 hardware or isoc rings
2889 		 * on AMD 0.96 host, clear the chain bit.
2890 		 */
2891 		if (!xhci_link_trb_quirk(xhci) &&
2892 		    !(ep_ring->type == TYPE_ISOC &&
2893 		      (xhci->quirks & XHCI_AMD_0x96_HOST)))
2894 			ep_ring->enqueue->link.control &=
2895 				cpu_to_le32(~TRB_CHAIN);
2896 		else
2897 			ep_ring->enqueue->link.control |=
2898 				cpu_to_le32(TRB_CHAIN);
2899 
2900 		wmb();
2901 		ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);
2902 
2903 		/* Toggle the cycle bit after the last ring segment. */
2904 		if (link_trb_toggles_cycle(ep_ring->enqueue))
2905 			ep_ring->cycle_state ^= 1;
2906 
2907 		ep_ring->enq_seg = ep_ring->enq_seg->next;
2908 		ep_ring->enqueue = ep_ring->enq_seg->trbs;
2909 	}
2910 	return 0;
2911 }
2912 
2913 static int prepare_transfer(struct xhci_hcd *xhci,
2914 		struct xhci_virt_device *xdev,
2915 		unsigned int ep_index,
2916 		unsigned int stream_id,
2917 		unsigned int num_trbs,
2918 		struct urb *urb,
2919 		unsigned int td_index,
2920 		gfp_t mem_flags)
2921 {
2922 	int ret;
2923 	struct urb_priv *urb_priv;
2924 	struct xhci_td	*td;
2925 	struct xhci_ring *ep_ring;
2926 	struct xhci_ep_ctx *ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
2927 
2928 	ep_ring = xhci_stream_id_to_ring(xdev, ep_index, stream_id);
2929 	if (!ep_ring) {
2930 		xhci_dbg(xhci, "Can't prepare ring for bad stream ID %u\n",
2931 				stream_id);
2932 		return -EINVAL;
2933 	}
2934 
2935 	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
2936 			   num_trbs, mem_flags);
2937 	if (ret)
2938 		return ret;
2939 
2940 	urb_priv = urb->hcpriv;
2941 	td = &urb_priv->td[td_index];
2942 
2943 	INIT_LIST_HEAD(&td->td_list);
2944 	INIT_LIST_HEAD(&td->cancelled_td_list);
2945 
2946 	if (td_index == 0) {
2947 		ret = usb_hcd_link_urb_to_ep(bus_to_hcd(urb->dev->bus), urb);
2948 		if (unlikely(ret))
2949 			return ret;
2950 	}
2951 
2952 	td->urb = urb;
2953 	/* Add this TD to the tail of the endpoint ring's TD list */
2954 	list_add_tail(&td->td_list, &ep_ring->td_list);
2955 	td->start_seg = ep_ring->enq_seg;
2956 	td->first_trb = ep_ring->enqueue;
2957 
2958 	return 0;
2959 }
2960 
2961 static unsigned int count_trbs(u64 addr, u64 len)
2962 {
2963 	unsigned int num_trbs;
2964 
2965 	num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)),
2966 			TRB_MAX_BUFF_SIZE);
2967 	if (num_trbs == 0)
2968 		num_trbs++;
2969 
2970 	return num_trbs;
2971 }
2972 
2973 static inline unsigned int count_trbs_needed(struct urb *urb)
2974 {
2975 	return count_trbs(urb->transfer_dma, urb->transfer_buffer_length);
2976 }
2977 
2978 static unsigned int count_sg_trbs_needed(struct urb *urb)
2979 {
2980 	struct scatterlist *sg;
2981 	unsigned int i, len, full_len, num_trbs = 0;
2982 
2983 	full_len = urb->transfer_buffer_length;
2984 
2985 	for_each_sg(urb->sg, sg, urb->num_mapped_sgs, i) {
2986 		len = sg_dma_len(sg);
2987 		num_trbs += count_trbs(sg_dma_address(sg), len);
2988 		len = min_t(unsigned int, len, full_len);
2989 		full_len -= len;
2990 		if (full_len == 0)
2991 			break;
2992 	}
2993 
2994 	return num_trbs;
2995 }
2996 
2997 static unsigned int count_isoc_trbs_needed(struct urb *urb, int i)
2998 {
2999 	u64 addr, len;
3000 
3001 	addr = (u64) (urb->transfer_dma + urb->iso_frame_desc[i].offset);
3002 	len = urb->iso_frame_desc[i].length;
3003 
3004 	return count_trbs(addr, len);
3005 }
3006 
3007 static void check_trb_math(struct urb *urb, int running_total)
3008 {
3009 	if (unlikely(running_total != urb->transfer_buffer_length))
3010 		dev_err(&urb->dev->dev, "%s - ep %#x - Miscalculated tx length, "
3011 				"queued %#x (%d), asked for %#x (%d)\n",
3012 				__func__,
3013 				urb->ep->desc.bEndpointAddress,
3014 				running_total, running_total,
3015 				urb->transfer_buffer_length,
3016 				urb->transfer_buffer_length);
3017 }
3018 
3019 static void giveback_first_trb(struct xhci_hcd *xhci, int slot_id,
3020 		unsigned int ep_index, unsigned int stream_id, int start_cycle,
3021 		struct xhci_generic_trb *start_trb)
3022 {
3023 	/*
3024 	 * Pass all the TRBs to the hardware at once and make sure this write
3025 	 * isn't reordered.
3026 	 */
3027 	wmb();
3028 	if (start_cycle)
3029 		start_trb->field[3] |= cpu_to_le32(start_cycle);
3030 	else
3031 		start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
3032 	xhci_ring_ep_doorbell(xhci, slot_id, ep_index, stream_id);
3033 }
3034 
3035 static void check_interval(struct xhci_hcd *xhci, struct urb *urb,
3036 						struct xhci_ep_ctx *ep_ctx)
3037 {
3038 	int xhci_interval;
3039 	int ep_interval;
3040 
3041 	xhci_interval = EP_INTERVAL_TO_UFRAMES(le32_to_cpu(ep_ctx->ep_info));
3042 	ep_interval = urb->interval;
3043 
3044 	/* Convert to microframes */
3045 	if (urb->dev->speed == USB_SPEED_LOW ||
3046 			urb->dev->speed == USB_SPEED_FULL)
3047 		ep_interval *= 8;
3048 
3049 	/* FIXME change this to a warning and a suggestion to use the new API
3050 	 * to set the polling interval (once the API is added).
3051 	 */
3052 	if (xhci_interval != ep_interval) {
3053 		dev_dbg_ratelimited(&urb->dev->dev,
3054 				"Driver uses different interval (%d microframe%s) than xHCI (%d microframe%s)\n",
3055 				ep_interval, ep_interval == 1 ? "" : "s",
3056 				xhci_interval, xhci_interval == 1 ? "" : "s");
3057 		urb->interval = xhci_interval;
3058 		/* Convert back to frames for LS/FS devices */
3059 		if (urb->dev->speed == USB_SPEED_LOW ||
3060 				urb->dev->speed == USB_SPEED_FULL)
3061 			urb->interval /= 8;
3062 	}
3063 }
3064 
3065 /*
3066  * xHCI uses normal TRBs for both bulk and interrupt.  When the interrupt
3067  * endpoint is to be serviced, the xHC will consume (at most) one TD.  A TD
3068  * (comprised of sg list entries) can take several service intervals to
3069  * transmit.
3070  */
3071 int xhci_queue_intr_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3072 		struct urb *urb, int slot_id, unsigned int ep_index)
3073 {
3074 	struct xhci_ep_ctx *ep_ctx;
3075 
3076 	ep_ctx = xhci_get_ep_ctx(xhci, xhci->devs[slot_id]->out_ctx, ep_index);
3077 	check_interval(xhci, urb, ep_ctx);
3078 
3079 	return xhci_queue_bulk_tx(xhci, mem_flags, urb, slot_id, ep_index);
3080 }
3081 
3082 /*
3083  * For xHCI 1.0 host controllers, TD size is the number of max packet sized
3084  * packets remaining in the TD (*not* including this TRB).
3085  *
3086  * Total TD packet count = total_packet_count =
3087  *     DIV_ROUND_UP(TD size in bytes / wMaxPacketSize)
3088  *
3089  * Packets transferred up to and including this TRB = packets_transferred =
3090  *     rounddown(total bytes transferred including this TRB / wMaxPacketSize)
3091  *
3092  * TD size = total_packet_count - packets_transferred
3093  *
3094  * For xHCI 0.96 and older, TD size field should be the remaining bytes
3095  * including this TRB, right shifted by 10
3096  *
3097  * For all hosts it must fit in bits 21:17, so it can't be bigger than 31.
3098  * This is taken care of in the TRB_TD_SIZE() macro
3099  *
3100  * The last TRB in a TD must have the TD size set to zero.
3101  */
3102 static u32 xhci_td_remainder(struct xhci_hcd *xhci, int transferred,
3103 			      int trb_buff_len, unsigned int td_total_len,
3104 			      struct urb *urb, bool more_trbs_coming)
3105 {
3106 	u32 maxp, total_packet_count;
3107 
3108 	/* MTK xHCI is mostly 0.97 but contains some features from 1.0 */
3109 	if (xhci->hci_version < 0x100 && !(xhci->quirks & XHCI_MTK_HOST))
3110 		return ((td_total_len - transferred) >> 10);
3111 
3112 	/* One TRB with a zero-length data packet. */
3113 	if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
3114 	    trb_buff_len == td_total_len)
3115 		return 0;
3116 
3117 	/* for MTK xHCI, TD size doesn't include this TRB */
3118 	if (xhci->quirks & XHCI_MTK_HOST)
3119 		trb_buff_len = 0;
3120 
3121 	maxp = usb_endpoint_maxp(&urb->ep->desc);
3122 	total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
3123 
3124 	/* Queueing functions don't count the current TRB into transferred */
3125 	return (total_packet_count - ((transferred + trb_buff_len) / maxp));
3126 }
3127 
3128 
3129 static int xhci_align_td(struct xhci_hcd *xhci, struct urb *urb, u32 enqd_len,
3130 			 u32 *trb_buff_len, struct xhci_segment *seg)
3131 {
3132 	struct device *dev = xhci_to_hcd(xhci)->self.controller;
3133 	unsigned int unalign;
3134 	unsigned int max_pkt;
3135 	u32 new_buff_len;
3136 
3137 	max_pkt = usb_endpoint_maxp(&urb->ep->desc);
3138 	unalign = (enqd_len + *trb_buff_len) % max_pkt;
3139 
3140 	/* we got lucky, last normal TRB data on segment is packet aligned */
3141 	if (unalign == 0)
3142 		return 0;
3143 
3144 	xhci_dbg(xhci, "Unaligned %d bytes, buff len %d\n",
3145 		 unalign, *trb_buff_len);
3146 
3147 	/* is the last nornal TRB alignable by splitting it */
3148 	if (*trb_buff_len > unalign) {
3149 		*trb_buff_len -= unalign;
3150 		xhci_dbg(xhci, "split align, new buff len %d\n", *trb_buff_len);
3151 		return 0;
3152 	}
3153 
3154 	/*
3155 	 * We want enqd_len + trb_buff_len to sum up to a number aligned to
3156 	 * number which is divisible by the endpoint's wMaxPacketSize. IOW:
3157 	 * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0.
3158 	 */
3159 	new_buff_len = max_pkt - (enqd_len % max_pkt);
3160 
3161 	if (new_buff_len > (urb->transfer_buffer_length - enqd_len))
3162 		new_buff_len = (urb->transfer_buffer_length - enqd_len);
3163 
3164 	/* create a max max_pkt sized bounce buffer pointed to by last trb */
3165 	if (usb_urb_dir_out(urb)) {
3166 		sg_pcopy_to_buffer(urb->sg, urb->num_mapped_sgs,
3167 				   seg->bounce_buf, new_buff_len, enqd_len);
3168 		seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
3169 						 max_pkt, DMA_TO_DEVICE);
3170 	} else {
3171 		seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
3172 						 max_pkt, DMA_FROM_DEVICE);
3173 	}
3174 
3175 	if (dma_mapping_error(dev, seg->bounce_dma)) {
3176 		/* try without aligning. Some host controllers survive */
3177 		xhci_warn(xhci, "Failed mapping bounce buffer, not aligning\n");
3178 		return 0;
3179 	}
3180 	*trb_buff_len = new_buff_len;
3181 	seg->bounce_len = new_buff_len;
3182 	seg->bounce_offs = enqd_len;
3183 
3184 	xhci_dbg(xhci, "Bounce align, new buff len %d\n", *trb_buff_len);
3185 
3186 	return 1;
3187 }
3188 
3189 /* This is very similar to what ehci-q.c qtd_fill() does */
3190 int xhci_queue_bulk_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3191 		struct urb *urb, int slot_id, unsigned int ep_index)
3192 {
3193 	struct xhci_ring *ring;
3194 	struct urb_priv *urb_priv;
3195 	struct xhci_td *td;
3196 	struct xhci_generic_trb *start_trb;
3197 	struct scatterlist *sg = NULL;
3198 	bool more_trbs_coming = true;
3199 	bool need_zero_pkt = false;
3200 	bool first_trb = true;
3201 	unsigned int num_trbs;
3202 	unsigned int start_cycle, num_sgs = 0;
3203 	unsigned int enqd_len, block_len, trb_buff_len, full_len;
3204 	int sent_len, ret;
3205 	u32 field, length_field, remainder;
3206 	u64 addr, send_addr;
3207 
3208 	ring = xhci_urb_to_transfer_ring(xhci, urb);
3209 	if (!ring)
3210 		return -EINVAL;
3211 
3212 	full_len = urb->transfer_buffer_length;
3213 	/* If we have scatter/gather list, we use it. */
3214 	if (urb->num_sgs) {
3215 		num_sgs = urb->num_mapped_sgs;
3216 		sg = urb->sg;
3217 		addr = (u64) sg_dma_address(sg);
3218 		block_len = sg_dma_len(sg);
3219 		num_trbs = count_sg_trbs_needed(urb);
3220 	} else {
3221 		num_trbs = count_trbs_needed(urb);
3222 		addr = (u64) urb->transfer_dma;
3223 		block_len = full_len;
3224 	}
3225 	ret = prepare_transfer(xhci, xhci->devs[slot_id],
3226 			ep_index, urb->stream_id,
3227 			num_trbs, urb, 0, mem_flags);
3228 	if (unlikely(ret < 0))
3229 		return ret;
3230 
3231 	urb_priv = urb->hcpriv;
3232 
3233 	/* Deal with URB_ZERO_PACKET - need one more td/trb */
3234 	if (urb->transfer_flags & URB_ZERO_PACKET && urb_priv->num_tds > 1)
3235 		need_zero_pkt = true;
3236 
3237 	td = &urb_priv->td[0];
3238 
3239 	/*
3240 	 * Don't give the first TRB to the hardware (by toggling the cycle bit)
3241 	 * until we've finished creating all the other TRBs.  The ring's cycle
3242 	 * state may change as we enqueue the other TRBs, so save it too.
3243 	 */
3244 	start_trb = &ring->enqueue->generic;
3245 	start_cycle = ring->cycle_state;
3246 	send_addr = addr;
3247 
3248 	/* Queue the TRBs, even if they are zero-length */
3249 	for (enqd_len = 0; first_trb || enqd_len < full_len;
3250 			enqd_len += trb_buff_len) {
3251 		field = TRB_TYPE(TRB_NORMAL);
3252 
3253 		/* TRB buffer should not cross 64KB boundaries */
3254 		trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
3255 		trb_buff_len = min_t(unsigned int, trb_buff_len, block_len);
3256 
3257 		if (enqd_len + trb_buff_len > full_len)
3258 			trb_buff_len = full_len - enqd_len;
3259 
3260 		/* Don't change the cycle bit of the first TRB until later */
3261 		if (first_trb) {
3262 			first_trb = false;
3263 			if (start_cycle == 0)
3264 				field |= TRB_CYCLE;
3265 		} else
3266 			field |= ring->cycle_state;
3267 
3268 		/* Chain all the TRBs together; clear the chain bit in the last
3269 		 * TRB to indicate it's the last TRB in the chain.
3270 		 */
3271 		if (enqd_len + trb_buff_len < full_len) {
3272 			field |= TRB_CHAIN;
3273 			if (trb_is_link(ring->enqueue + 1)) {
3274 				if (xhci_align_td(xhci, urb, enqd_len,
3275 						  &trb_buff_len,
3276 						  ring->enq_seg)) {
3277 					send_addr = ring->enq_seg->bounce_dma;
3278 					/* assuming TD won't span 2 segs */
3279 					td->bounce_seg = ring->enq_seg;
3280 				}
3281 			}
3282 		}
3283 		if (enqd_len + trb_buff_len >= full_len) {
3284 			field &= ~TRB_CHAIN;
3285 			field |= TRB_IOC;
3286 			more_trbs_coming = false;
3287 			td->last_trb = ring->enqueue;
3288 		}
3289 
3290 		/* Only set interrupt on short packet for IN endpoints */
3291 		if (usb_urb_dir_in(urb))
3292 			field |= TRB_ISP;
3293 
3294 		/* Set the TRB length, TD size, and interrupter fields. */
3295 		remainder = xhci_td_remainder(xhci, enqd_len, trb_buff_len,
3296 					      full_len, urb, more_trbs_coming);
3297 
3298 		length_field = TRB_LEN(trb_buff_len) |
3299 			TRB_TD_SIZE(remainder) |
3300 			TRB_INTR_TARGET(0);
3301 
3302 		queue_trb(xhci, ring, more_trbs_coming | need_zero_pkt,
3303 				lower_32_bits(send_addr),
3304 				upper_32_bits(send_addr),
3305 				length_field,
3306 				field);
3307 
3308 		addr += trb_buff_len;
3309 		sent_len = trb_buff_len;
3310 
3311 		while (sg && sent_len >= block_len) {
3312 			/* New sg entry */
3313 			--num_sgs;
3314 			sent_len -= block_len;
3315 			if (num_sgs != 0) {
3316 				sg = sg_next(sg);
3317 				block_len = sg_dma_len(sg);
3318 				addr = (u64) sg_dma_address(sg);
3319 				addr += sent_len;
3320 			}
3321 		}
3322 		block_len -= sent_len;
3323 		send_addr = addr;
3324 	}
3325 
3326 	if (need_zero_pkt) {
3327 		ret = prepare_transfer(xhci, xhci->devs[slot_id],
3328 				       ep_index, urb->stream_id,
3329 				       1, urb, 1, mem_flags);
3330 		urb_priv->td[1].last_trb = ring->enqueue;
3331 		field = TRB_TYPE(TRB_NORMAL) | ring->cycle_state | TRB_IOC;
3332 		queue_trb(xhci, ring, 0, 0, 0, TRB_INTR_TARGET(0), field);
3333 	}
3334 
3335 	check_trb_math(urb, enqd_len);
3336 	giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
3337 			start_cycle, start_trb);
3338 	return 0;
3339 }
3340 
3341 /* Caller must have locked xhci->lock */
3342 int xhci_queue_ctrl_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3343 		struct urb *urb, int slot_id, unsigned int ep_index)
3344 {
3345 	struct xhci_ring *ep_ring;
3346 	int num_trbs;
3347 	int ret;
3348 	struct usb_ctrlrequest *setup;
3349 	struct xhci_generic_trb *start_trb;
3350 	int start_cycle;
3351 	u32 field;
3352 	struct urb_priv *urb_priv;
3353 	struct xhci_td *td;
3354 
3355 	ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
3356 	if (!ep_ring)
3357 		return -EINVAL;
3358 
3359 	/*
3360 	 * Need to copy setup packet into setup TRB, so we can't use the setup
3361 	 * DMA address.
3362 	 */
3363 	if (!urb->setup_packet)
3364 		return -EINVAL;
3365 
3366 	/* 1 TRB for setup, 1 for status */
3367 	num_trbs = 2;
3368 	/*
3369 	 * Don't need to check if we need additional event data and normal TRBs,
3370 	 * since data in control transfers will never get bigger than 16MB
3371 	 * XXX: can we get a buffer that crosses 64KB boundaries?
3372 	 */
3373 	if (urb->transfer_buffer_length > 0)
3374 		num_trbs++;
3375 	ret = prepare_transfer(xhci, xhci->devs[slot_id],
3376 			ep_index, urb->stream_id,
3377 			num_trbs, urb, 0, mem_flags);
3378 	if (ret < 0)
3379 		return ret;
3380 
3381 	urb_priv = urb->hcpriv;
3382 	td = &urb_priv->td[0];
3383 
3384 	/*
3385 	 * Don't give the first TRB to the hardware (by toggling the cycle bit)
3386 	 * until we've finished creating all the other TRBs.  The ring's cycle
3387 	 * state may change as we enqueue the other TRBs, so save it too.
3388 	 */
3389 	start_trb = &ep_ring->enqueue->generic;
3390 	start_cycle = ep_ring->cycle_state;
3391 
3392 	/* Queue setup TRB - see section 6.4.1.2.1 */
3393 	/* FIXME better way to translate setup_packet into two u32 fields? */
3394 	setup = (struct usb_ctrlrequest *) urb->setup_packet;
3395 	field = 0;
3396 	field |= TRB_IDT | TRB_TYPE(TRB_SETUP);
3397 	if (start_cycle == 0)
3398 		field |= 0x1;
3399 
3400 	/* xHCI 1.0/1.1 6.4.1.2.1: Transfer Type field */
3401 	if ((xhci->hci_version >= 0x100) || (xhci->quirks & XHCI_MTK_HOST)) {
3402 		if (urb->transfer_buffer_length > 0) {
3403 			if (setup->bRequestType & USB_DIR_IN)
3404 				field |= TRB_TX_TYPE(TRB_DATA_IN);
3405 			else
3406 				field |= TRB_TX_TYPE(TRB_DATA_OUT);
3407 		}
3408 	}
3409 
3410 	queue_trb(xhci, ep_ring, true,
3411 		  setup->bRequestType | setup->bRequest << 8 | le16_to_cpu(setup->wValue) << 16,
3412 		  le16_to_cpu(setup->wIndex) | le16_to_cpu(setup->wLength) << 16,
3413 		  TRB_LEN(8) | TRB_INTR_TARGET(0),
3414 		  /* Immediate data in pointer */
3415 		  field);
3416 
3417 	/* If there's data, queue data TRBs */
3418 	/* Only set interrupt on short packet for IN endpoints */
3419 	if (usb_urb_dir_in(urb))
3420 		field = TRB_ISP | TRB_TYPE(TRB_DATA);
3421 	else
3422 		field = TRB_TYPE(TRB_DATA);
3423 
3424 	if (urb->transfer_buffer_length > 0) {
3425 		u32 length_field, remainder;
3426 
3427 		remainder = xhci_td_remainder(xhci, 0,
3428 				urb->transfer_buffer_length,
3429 				urb->transfer_buffer_length,
3430 				urb, 1);
3431 		length_field = TRB_LEN(urb->transfer_buffer_length) |
3432 				TRB_TD_SIZE(remainder) |
3433 				TRB_INTR_TARGET(0);
3434 		if (setup->bRequestType & USB_DIR_IN)
3435 			field |= TRB_DIR_IN;
3436 		queue_trb(xhci, ep_ring, true,
3437 				lower_32_bits(urb->transfer_dma),
3438 				upper_32_bits(urb->transfer_dma),
3439 				length_field,
3440 				field | ep_ring->cycle_state);
3441 	}
3442 
3443 	/* Save the DMA address of the last TRB in the TD */
3444 	td->last_trb = ep_ring->enqueue;
3445 
3446 	/* Queue status TRB - see Table 7 and sections 4.11.2.2 and 6.4.1.2.3 */
3447 	/* If the device sent data, the status stage is an OUT transfer */
3448 	if (urb->transfer_buffer_length > 0 && setup->bRequestType & USB_DIR_IN)
3449 		field = 0;
3450 	else
3451 		field = TRB_DIR_IN;
3452 	queue_trb(xhci, ep_ring, false,
3453 			0,
3454 			0,
3455 			TRB_INTR_TARGET(0),
3456 			/* Event on completion */
3457 			field | TRB_IOC | TRB_TYPE(TRB_STATUS) | ep_ring->cycle_state);
3458 
3459 	giveback_first_trb(xhci, slot_id, ep_index, 0,
3460 			start_cycle, start_trb);
3461 	return 0;
3462 }
3463 
3464 /*
3465  * The transfer burst count field of the isochronous TRB defines the number of
3466  * bursts that are required to move all packets in this TD.  Only SuperSpeed
3467  * devices can burst up to bMaxBurst number of packets per service interval.
3468  * This field is zero based, meaning a value of zero in the field means one
3469  * burst.  Basically, for everything but SuperSpeed devices, this field will be
3470  * zero.  Only xHCI 1.0 host controllers support this field.
3471  */
3472 static unsigned int xhci_get_burst_count(struct xhci_hcd *xhci,
3473 		struct urb *urb, unsigned int total_packet_count)
3474 {
3475 	unsigned int max_burst;
3476 
3477 	if (xhci->hci_version < 0x100 || urb->dev->speed < USB_SPEED_SUPER)
3478 		return 0;
3479 
3480 	max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3481 	return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1;
3482 }
3483 
3484 /*
3485  * Returns the number of packets in the last "burst" of packets.  This field is
3486  * valid for all speeds of devices.  USB 2.0 devices can only do one "burst", so
3487  * the last burst packet count is equal to the total number of packets in the
3488  * TD.  SuperSpeed endpoints can have up to 3 bursts.  All but the last burst
3489  * must contain (bMaxBurst + 1) number of packets, but the last burst can
3490  * contain 1 to (bMaxBurst + 1) packets.
3491  */
3492 static unsigned int xhci_get_last_burst_packet_count(struct xhci_hcd *xhci,
3493 		struct urb *urb, unsigned int total_packet_count)
3494 {
3495 	unsigned int max_burst;
3496 	unsigned int residue;
3497 
3498 	if (xhci->hci_version < 0x100)
3499 		return 0;
3500 
3501 	if (urb->dev->speed >= USB_SPEED_SUPER) {
3502 		/* bMaxBurst is zero based: 0 means 1 packet per burst */
3503 		max_burst = urb->ep->ss_ep_comp.bMaxBurst;
3504 		residue = total_packet_count % (max_burst + 1);
3505 		/* If residue is zero, the last burst contains (max_burst + 1)
3506 		 * number of packets, but the TLBPC field is zero-based.
3507 		 */
3508 		if (residue == 0)
3509 			return max_burst;
3510 		return residue - 1;
3511 	}
3512 	if (total_packet_count == 0)
3513 		return 0;
3514 	return total_packet_count - 1;
3515 }
3516 
3517 /*
3518  * Calculates Frame ID field of the isochronous TRB identifies the
3519  * target frame that the Interval associated with this Isochronous
3520  * Transfer Descriptor will start on. Refer to 4.11.2.5 in 1.1 spec.
3521  *
3522  * Returns actual frame id on success, negative value on error.
3523  */
3524 static int xhci_get_isoc_frame_id(struct xhci_hcd *xhci,
3525 		struct urb *urb, int index)
3526 {
3527 	int start_frame, ist, ret = 0;
3528 	int start_frame_id, end_frame_id, current_frame_id;
3529 
3530 	if (urb->dev->speed == USB_SPEED_LOW ||
3531 			urb->dev->speed == USB_SPEED_FULL)
3532 		start_frame = urb->start_frame + index * urb->interval;
3533 	else
3534 		start_frame = (urb->start_frame + index * urb->interval) >> 3;
3535 
3536 	/* Isochronous Scheduling Threshold (IST, bits 0~3 in HCSPARAMS2):
3537 	 *
3538 	 * If bit [3] of IST is cleared to '0', software can add a TRB no
3539 	 * later than IST[2:0] Microframes before that TRB is scheduled to
3540 	 * be executed.
3541 	 * If bit [3] of IST is set to '1', software can add a TRB no later
3542 	 * than IST[2:0] Frames before that TRB is scheduled to be executed.
3543 	 */
3544 	ist = HCS_IST(xhci->hcs_params2) & 0x7;
3545 	if (HCS_IST(xhci->hcs_params2) & (1 << 3))
3546 		ist <<= 3;
3547 
3548 	/* Software shall not schedule an Isoch TD with a Frame ID value that
3549 	 * is less than the Start Frame ID or greater than the End Frame ID,
3550 	 * where:
3551 	 *
3552 	 * End Frame ID = (Current MFINDEX register value + 895 ms.) MOD 2048
3553 	 * Start Frame ID = (Current MFINDEX register value + IST + 1) MOD 2048
3554 	 *
3555 	 * Both the End Frame ID and Start Frame ID values are calculated
3556 	 * in microframes. When software determines the valid Frame ID value;
3557 	 * The End Frame ID value should be rounded down to the nearest Frame
3558 	 * boundary, and the Start Frame ID value should be rounded up to the
3559 	 * nearest Frame boundary.
3560 	 */
3561 	current_frame_id = readl(&xhci->run_regs->microframe_index);
3562 	start_frame_id = roundup(current_frame_id + ist + 1, 8);
3563 	end_frame_id = rounddown(current_frame_id + 895 * 8, 8);
3564 
3565 	start_frame &= 0x7ff;
3566 	start_frame_id = (start_frame_id >> 3) & 0x7ff;
3567 	end_frame_id = (end_frame_id >> 3) & 0x7ff;
3568 
3569 	xhci_dbg(xhci, "%s: index %d, reg 0x%x start_frame_id 0x%x, end_frame_id 0x%x, start_frame 0x%x\n",
3570 		 __func__, index, readl(&xhci->run_regs->microframe_index),
3571 		 start_frame_id, end_frame_id, start_frame);
3572 
3573 	if (start_frame_id < end_frame_id) {
3574 		if (start_frame > end_frame_id ||
3575 				start_frame < start_frame_id)
3576 			ret = -EINVAL;
3577 	} else if (start_frame_id > end_frame_id) {
3578 		if ((start_frame > end_frame_id &&
3579 				start_frame < start_frame_id))
3580 			ret = -EINVAL;
3581 	} else {
3582 			ret = -EINVAL;
3583 	}
3584 
3585 	if (index == 0) {
3586 		if (ret == -EINVAL || start_frame == start_frame_id) {
3587 			start_frame = start_frame_id + 1;
3588 			if (urb->dev->speed == USB_SPEED_LOW ||
3589 					urb->dev->speed == USB_SPEED_FULL)
3590 				urb->start_frame = start_frame;
3591 			else
3592 				urb->start_frame = start_frame << 3;
3593 			ret = 0;
3594 		}
3595 	}
3596 
3597 	if (ret) {
3598 		xhci_warn(xhci, "Frame ID %d (reg %d, index %d) beyond range (%d, %d)\n",
3599 				start_frame, current_frame_id, index,
3600 				start_frame_id, end_frame_id);
3601 		xhci_warn(xhci, "Ignore frame ID field, use SIA bit instead\n");
3602 		return ret;
3603 	}
3604 
3605 	return start_frame;
3606 }
3607 
3608 /* This is for isoc transfer */
3609 static int xhci_queue_isoc_tx(struct xhci_hcd *xhci, gfp_t mem_flags,
3610 		struct urb *urb, int slot_id, unsigned int ep_index)
3611 {
3612 	struct xhci_ring *ep_ring;
3613 	struct urb_priv *urb_priv;
3614 	struct xhci_td *td;
3615 	int num_tds, trbs_per_td;
3616 	struct xhci_generic_trb *start_trb;
3617 	bool first_trb;
3618 	int start_cycle;
3619 	u32 field, length_field;
3620 	int running_total, trb_buff_len, td_len, td_remain_len, ret;
3621 	u64 start_addr, addr;
3622 	int i, j;
3623 	bool more_trbs_coming;
3624 	struct xhci_virt_ep *xep;
3625 	int frame_id;
3626 
3627 	xep = &xhci->devs[slot_id]->eps[ep_index];
3628 	ep_ring = xhci->devs[slot_id]->eps[ep_index].ring;
3629 
3630 	num_tds = urb->number_of_packets;
3631 	if (num_tds < 1) {
3632 		xhci_dbg(xhci, "Isoc URB with zero packets?\n");
3633 		return -EINVAL;
3634 	}
3635 	start_addr = (u64) urb->transfer_dma;
3636 	start_trb = &ep_ring->enqueue->generic;
3637 	start_cycle = ep_ring->cycle_state;
3638 
3639 	urb_priv = urb->hcpriv;
3640 	/* Queue the TRBs for each TD, even if they are zero-length */
3641 	for (i = 0; i < num_tds; i++) {
3642 		unsigned int total_pkt_count, max_pkt;
3643 		unsigned int burst_count, last_burst_pkt_count;
3644 		u32 sia_frame_id;
3645 
3646 		first_trb = true;
3647 		running_total = 0;
3648 		addr = start_addr + urb->iso_frame_desc[i].offset;
3649 		td_len = urb->iso_frame_desc[i].length;
3650 		td_remain_len = td_len;
3651 		max_pkt = usb_endpoint_maxp(&urb->ep->desc);
3652 		total_pkt_count = DIV_ROUND_UP(td_len, max_pkt);
3653 
3654 		/* A zero-length transfer still involves at least one packet. */
3655 		if (total_pkt_count == 0)
3656 			total_pkt_count++;
3657 		burst_count = xhci_get_burst_count(xhci, urb, total_pkt_count);
3658 		last_burst_pkt_count = xhci_get_last_burst_packet_count(xhci,
3659 							urb, total_pkt_count);
3660 
3661 		trbs_per_td = count_isoc_trbs_needed(urb, i);
3662 
3663 		ret = prepare_transfer(xhci, xhci->devs[slot_id], ep_index,
3664 				urb->stream_id, trbs_per_td, urb, i, mem_flags);
3665 		if (ret < 0) {
3666 			if (i == 0)
3667 				return ret;
3668 			goto cleanup;
3669 		}
3670 		td = &urb_priv->td[i];
3671 
3672 		/* use SIA as default, if frame id is used overwrite it */
3673 		sia_frame_id = TRB_SIA;
3674 		if (!(urb->transfer_flags & URB_ISO_ASAP) &&
3675 		    HCC_CFC(xhci->hcc_params)) {
3676 			frame_id = xhci_get_isoc_frame_id(xhci, urb, i);
3677 			if (frame_id >= 0)
3678 				sia_frame_id = TRB_FRAME_ID(frame_id);
3679 		}
3680 		/*
3681 		 * Set isoc specific data for the first TRB in a TD.
3682 		 * Prevent HW from getting the TRBs by keeping the cycle state
3683 		 * inverted in the first TDs isoc TRB.
3684 		 */
3685 		field = TRB_TYPE(TRB_ISOC) |
3686 			TRB_TLBPC(last_burst_pkt_count) |
3687 			sia_frame_id |
3688 			(i ? ep_ring->cycle_state : !start_cycle);
3689 
3690 		/* xhci 1.1 with ETE uses TD_Size field for TBC, old is Rsvdz */
3691 		if (!xep->use_extended_tbc)
3692 			field |= TRB_TBC(burst_count);
3693 
3694 		/* fill the rest of the TRB fields, and remaining normal TRBs */
3695 		for (j = 0; j < trbs_per_td; j++) {
3696 			u32 remainder = 0;
3697 
3698 			/* only first TRB is isoc, overwrite otherwise */
3699 			if (!first_trb)
3700 				field = TRB_TYPE(TRB_NORMAL) |
3701 					ep_ring->cycle_state;
3702 
3703 			/* Only set interrupt on short packet for IN EPs */
3704 			if (usb_urb_dir_in(urb))
3705 				field |= TRB_ISP;
3706 
3707 			/* Set the chain bit for all except the last TRB  */
3708 			if (j < trbs_per_td - 1) {
3709 				more_trbs_coming = true;
3710 				field |= TRB_CHAIN;
3711 			} else {
3712 				more_trbs_coming = false;
3713 				td->last_trb = ep_ring->enqueue;
3714 				field |= TRB_IOC;
3715 				/* set BEI, except for the last TD */
3716 				if (xhci->hci_version >= 0x100 &&
3717 				    !(xhci->quirks & XHCI_AVOID_BEI) &&
3718 				    i < num_tds - 1)
3719 					field |= TRB_BEI;
3720 			}
3721 			/* Calculate TRB length */
3722 			trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
3723 			if (trb_buff_len > td_remain_len)
3724 				trb_buff_len = td_remain_len;
3725 
3726 			/* Set the TRB length, TD size, & interrupter fields. */
3727 			remainder = xhci_td_remainder(xhci, running_total,
3728 						   trb_buff_len, td_len,
3729 						   urb, more_trbs_coming);
3730 
3731 			length_field = TRB_LEN(trb_buff_len) |
3732 				TRB_INTR_TARGET(0);
3733 
3734 			/* xhci 1.1 with ETE uses TD Size field for TBC */
3735 			if (first_trb && xep->use_extended_tbc)
3736 				length_field |= TRB_TD_SIZE_TBC(burst_count);
3737 			else
3738 				length_field |= TRB_TD_SIZE(remainder);
3739 			first_trb = false;
3740 
3741 			queue_trb(xhci, ep_ring, more_trbs_coming,
3742 				lower_32_bits(addr),
3743 				upper_32_bits(addr),
3744 				length_field,
3745 				field);
3746 			running_total += trb_buff_len;
3747 
3748 			addr += trb_buff_len;
3749 			td_remain_len -= trb_buff_len;
3750 		}
3751 
3752 		/* Check TD length */
3753 		if (running_total != td_len) {
3754 			xhci_err(xhci, "ISOC TD length unmatch\n");
3755 			ret = -EINVAL;
3756 			goto cleanup;
3757 		}
3758 	}
3759 
3760 	/* store the next frame id */
3761 	if (HCC_CFC(xhci->hcc_params))
3762 		xep->next_frame_id = urb->start_frame + num_tds * urb->interval;
3763 
3764 	if (xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs == 0) {
3765 		if (xhci->quirks & XHCI_AMD_PLL_FIX)
3766 			usb_amd_quirk_pll_disable();
3767 	}
3768 	xhci_to_hcd(xhci)->self.bandwidth_isoc_reqs++;
3769 
3770 	giveback_first_trb(xhci, slot_id, ep_index, urb->stream_id,
3771 			start_cycle, start_trb);
3772 	return 0;
3773 cleanup:
3774 	/* Clean up a partially enqueued isoc transfer. */
3775 
3776 	for (i--; i >= 0; i--)
3777 		list_del_init(&urb_priv->td[i].td_list);
3778 
3779 	/* Use the first TD as a temporary variable to turn the TDs we've queued
3780 	 * into No-ops with a software-owned cycle bit. That way the hardware
3781 	 * won't accidentally start executing bogus TDs when we partially
3782 	 * overwrite them.  td->first_trb and td->start_seg are already set.
3783 	 */
3784 	urb_priv->td[0].last_trb = ep_ring->enqueue;
3785 	/* Every TRB except the first & last will have its cycle bit flipped. */
3786 	td_to_noop(xhci, ep_ring, &urb_priv->td[0], true);
3787 
3788 	/* Reset the ring enqueue back to the first TRB and its cycle bit. */
3789 	ep_ring->enqueue = urb_priv->td[0].first_trb;
3790 	ep_ring->enq_seg = urb_priv->td[0].start_seg;
3791 	ep_ring->cycle_state = start_cycle;
3792 	ep_ring->num_trbs_free = ep_ring->num_trbs_free_temp;
3793 	usb_hcd_unlink_urb_from_ep(bus_to_hcd(urb->dev->bus), urb);
3794 	return ret;
3795 }
3796 
3797 /*
3798  * Check transfer ring to guarantee there is enough room for the urb.
3799  * Update ISO URB start_frame and interval.
3800  * Update interval as xhci_queue_intr_tx does. Use xhci frame_index to
3801  * update urb->start_frame if URB_ISO_ASAP is set in transfer_flags or
3802  * Contiguous Frame ID is not supported by HC.
3803  */
3804 int xhci_queue_isoc_tx_prepare(struct xhci_hcd *xhci, gfp_t mem_flags,
3805 		struct urb *urb, int slot_id, unsigned int ep_index)
3806 {
3807 	struct xhci_virt_device *xdev;
3808 	struct xhci_ring *ep_ring;
3809 	struct xhci_ep_ctx *ep_ctx;
3810 	int start_frame;
3811 	int num_tds, num_trbs, i;
3812 	int ret;
3813 	struct xhci_virt_ep *xep;
3814 	int ist;
3815 
3816 	xdev = xhci->devs[slot_id];
3817 	xep = &xhci->devs[slot_id]->eps[ep_index];
3818 	ep_ring = xdev->eps[ep_index].ring;
3819 	ep_ctx = xhci_get_ep_ctx(xhci, xdev->out_ctx, ep_index);
3820 
3821 	num_trbs = 0;
3822 	num_tds = urb->number_of_packets;
3823 	for (i = 0; i < num_tds; i++)
3824 		num_trbs += count_isoc_trbs_needed(urb, i);
3825 
3826 	/* Check the ring to guarantee there is enough room for the whole urb.
3827 	 * Do not insert any td of the urb to the ring if the check failed.
3828 	 */
3829 	ret = prepare_ring(xhci, ep_ring, GET_EP_CTX_STATE(ep_ctx),
3830 			   num_trbs, mem_flags);
3831 	if (ret)
3832 		return ret;
3833 
3834 	/*
3835 	 * Check interval value. This should be done before we start to
3836 	 * calculate the start frame value.
3837 	 */
3838 	check_interval(xhci, urb, ep_ctx);
3839 
3840 	/* Calculate the start frame and put it in urb->start_frame. */
3841 	if (HCC_CFC(xhci->hcc_params) && !list_empty(&ep_ring->td_list)) {
3842 		if (GET_EP_CTX_STATE(ep_ctx) ==	EP_STATE_RUNNING) {
3843 			urb->start_frame = xep->next_frame_id;
3844 			goto skip_start_over;
3845 		}
3846 	}
3847 
3848 	start_frame = readl(&xhci->run_regs->microframe_index);
3849 	start_frame &= 0x3fff;
3850 	/*
3851 	 * Round up to the next frame and consider the time before trb really
3852 	 * gets scheduled by hardare.
3853 	 */
3854 	ist = HCS_IST(xhci->hcs_params2) & 0x7;
3855 	if (HCS_IST(xhci->hcs_params2) & (1 << 3))
3856 		ist <<= 3;
3857 	start_frame += ist + XHCI_CFC_DELAY;
3858 	start_frame = roundup(start_frame, 8);
3859 
3860 	/*
3861 	 * Round up to the next ESIT (Endpoint Service Interval Time) if ESIT
3862 	 * is greate than 8 microframes.
3863 	 */
3864 	if (urb->dev->speed == USB_SPEED_LOW ||
3865 			urb->dev->speed == USB_SPEED_FULL) {
3866 		start_frame = roundup(start_frame, urb->interval << 3);
3867 		urb->start_frame = start_frame >> 3;
3868 	} else {
3869 		start_frame = roundup(start_frame, urb->interval);
3870 		urb->start_frame = start_frame;
3871 	}
3872 
3873 skip_start_over:
3874 	ep_ring->num_trbs_free_temp = ep_ring->num_trbs_free;
3875 
3876 	return xhci_queue_isoc_tx(xhci, mem_flags, urb, slot_id, ep_index);
3877 }
3878 
3879 /****		Command Ring Operations		****/
3880 
3881 /* Generic function for queueing a command TRB on the command ring.
3882  * Check to make sure there's room on the command ring for one command TRB.
3883  * Also check that there's room reserved for commands that must not fail.
3884  * If this is a command that must not fail, meaning command_must_succeed = TRUE,
3885  * then only check for the number of reserved spots.
3886  * Don't decrement xhci->cmd_ring_reserved_trbs after we've queued the TRB
3887  * because the command event handler may want to resubmit a failed command.
3888  */
3889 static int queue_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
3890 			 u32 field1, u32 field2,
3891 			 u32 field3, u32 field4, bool command_must_succeed)
3892 {
3893 	int reserved_trbs = xhci->cmd_ring_reserved_trbs;
3894 	int ret;
3895 
3896 	if ((xhci->xhc_state & XHCI_STATE_DYING) ||
3897 		(xhci->xhc_state & XHCI_STATE_HALTED)) {
3898 		xhci_dbg(xhci, "xHCI dying or halted, can't queue_command\n");
3899 		return -ESHUTDOWN;
3900 	}
3901 
3902 	if (!command_must_succeed)
3903 		reserved_trbs++;
3904 
3905 	ret = prepare_ring(xhci, xhci->cmd_ring, EP_STATE_RUNNING,
3906 			reserved_trbs, GFP_ATOMIC);
3907 	if (ret < 0) {
3908 		xhci_err(xhci, "ERR: No room for command on command ring\n");
3909 		if (command_must_succeed)
3910 			xhci_err(xhci, "ERR: Reserved TRB counting for "
3911 					"unfailable commands failed.\n");
3912 		return ret;
3913 	}
3914 
3915 	cmd->command_trb = xhci->cmd_ring->enqueue;
3916 
3917 	/* if there are no other commands queued we start the timeout timer */
3918 	if (list_empty(&xhci->cmd_list)) {
3919 		xhci->current_cmd = cmd;
3920 		xhci_mod_cmd_timer(xhci, XHCI_CMD_DEFAULT_TIMEOUT);
3921 	}
3922 
3923 	list_add_tail(&cmd->cmd_list, &xhci->cmd_list);
3924 
3925 	queue_trb(xhci, xhci->cmd_ring, false, field1, field2, field3,
3926 			field4 | xhci->cmd_ring->cycle_state);
3927 	return 0;
3928 }
3929 
3930 /* Queue a slot enable or disable request on the command ring */
3931 int xhci_queue_slot_control(struct xhci_hcd *xhci, struct xhci_command *cmd,
3932 		u32 trb_type, u32 slot_id)
3933 {
3934 	return queue_command(xhci, cmd, 0, 0, 0,
3935 			TRB_TYPE(trb_type) | SLOT_ID_FOR_TRB(slot_id), false);
3936 }
3937 
3938 /* Queue an address device command TRB */
3939 int xhci_queue_address_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
3940 		dma_addr_t in_ctx_ptr, u32 slot_id, enum xhci_setup_dev setup)
3941 {
3942 	return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
3943 			upper_32_bits(in_ctx_ptr), 0,
3944 			TRB_TYPE(TRB_ADDR_DEV) | SLOT_ID_FOR_TRB(slot_id)
3945 			| (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0), false);
3946 }
3947 
3948 int xhci_queue_vendor_command(struct xhci_hcd *xhci, struct xhci_command *cmd,
3949 		u32 field1, u32 field2, u32 field3, u32 field4)
3950 {
3951 	return queue_command(xhci, cmd, field1, field2, field3, field4, false);
3952 }
3953 
3954 /* Queue a reset device command TRB */
3955 int xhci_queue_reset_device(struct xhci_hcd *xhci, struct xhci_command *cmd,
3956 		u32 slot_id)
3957 {
3958 	return queue_command(xhci, cmd, 0, 0, 0,
3959 			TRB_TYPE(TRB_RESET_DEV) | SLOT_ID_FOR_TRB(slot_id),
3960 			false);
3961 }
3962 
3963 /* Queue a configure endpoint command TRB */
3964 int xhci_queue_configure_endpoint(struct xhci_hcd *xhci,
3965 		struct xhci_command *cmd, dma_addr_t in_ctx_ptr,
3966 		u32 slot_id, bool command_must_succeed)
3967 {
3968 	return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
3969 			upper_32_bits(in_ctx_ptr), 0,
3970 			TRB_TYPE(TRB_CONFIG_EP) | SLOT_ID_FOR_TRB(slot_id),
3971 			command_must_succeed);
3972 }
3973 
3974 /* Queue an evaluate context command TRB */
3975 int xhci_queue_evaluate_context(struct xhci_hcd *xhci, struct xhci_command *cmd,
3976 		dma_addr_t in_ctx_ptr, u32 slot_id, bool command_must_succeed)
3977 {
3978 	return queue_command(xhci, cmd, lower_32_bits(in_ctx_ptr),
3979 			upper_32_bits(in_ctx_ptr), 0,
3980 			TRB_TYPE(TRB_EVAL_CONTEXT) | SLOT_ID_FOR_TRB(slot_id),
3981 			command_must_succeed);
3982 }
3983 
3984 /*
3985  * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop
3986  * activity on an endpoint that is about to be suspended.
3987  */
3988 int xhci_queue_stop_endpoint(struct xhci_hcd *xhci, struct xhci_command *cmd,
3989 			     int slot_id, unsigned int ep_index, int suspend)
3990 {
3991 	u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
3992 	u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
3993 	u32 type = TRB_TYPE(TRB_STOP_RING);
3994 	u32 trb_suspend = SUSPEND_PORT_FOR_TRB(suspend);
3995 
3996 	return queue_command(xhci, cmd, 0, 0, 0,
3997 			trb_slot_id | trb_ep_index | type | trb_suspend, false);
3998 }
3999 
4000 /* Set Transfer Ring Dequeue Pointer command */
4001 void xhci_queue_new_dequeue_state(struct xhci_hcd *xhci,
4002 		unsigned int slot_id, unsigned int ep_index,
4003 		struct xhci_dequeue_state *deq_state)
4004 {
4005 	dma_addr_t addr;
4006 	u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
4007 	u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
4008 	u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id);
4009 	u32 trb_sct = 0;
4010 	u32 type = TRB_TYPE(TRB_SET_DEQ);
4011 	struct xhci_virt_ep *ep;
4012 	struct xhci_command *cmd;
4013 	int ret;
4014 
4015 	xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
4016 		"Set TR Deq Ptr cmd, new deq seg = %p (0x%llx dma), new deq ptr = %p (0x%llx dma), new cycle = %u",
4017 		deq_state->new_deq_seg,
4018 		(unsigned long long)deq_state->new_deq_seg->dma,
4019 		deq_state->new_deq_ptr,
4020 		(unsigned long long)xhci_trb_virt_to_dma(
4021 			deq_state->new_deq_seg, deq_state->new_deq_ptr),
4022 		deq_state->new_cycle_state);
4023 
4024 	addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
4025 				    deq_state->new_deq_ptr);
4026 	if (addr == 0) {
4027 		xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4028 		xhci_warn(xhci, "WARN deq seg = %p, deq pt = %p\n",
4029 			  deq_state->new_deq_seg, deq_state->new_deq_ptr);
4030 		return;
4031 	}
4032 	ep = &xhci->devs[slot_id]->eps[ep_index];
4033 	if ((ep->ep_state & SET_DEQ_PENDING)) {
4034 		xhci_warn(xhci, "WARN Cannot submit Set TR Deq Ptr\n");
4035 		xhci_warn(xhci, "A Set TR Deq Ptr command is pending.\n");
4036 		return;
4037 	}
4038 
4039 	/* This function gets called from contexts where it cannot sleep */
4040 	cmd = xhci_alloc_command(xhci, false, false, GFP_ATOMIC);
4041 	if (!cmd)
4042 		return;
4043 
4044 	ep->queued_deq_seg = deq_state->new_deq_seg;
4045 	ep->queued_deq_ptr = deq_state->new_deq_ptr;
4046 	if (deq_state->stream_id)
4047 		trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
4048 	ret = queue_command(xhci, cmd,
4049 		lower_32_bits(addr) | trb_sct | deq_state->new_cycle_state,
4050 		upper_32_bits(addr), trb_stream_id,
4051 		trb_slot_id | trb_ep_index | type, false);
4052 	if (ret < 0) {
4053 		xhci_free_command(xhci, cmd);
4054 		return;
4055 	}
4056 
4057 	/* Stop the TD queueing code from ringing the doorbell until
4058 	 * this command completes.  The HC won't set the dequeue pointer
4059 	 * if the ring is running, and ringing the doorbell starts the
4060 	 * ring running.
4061 	 */
4062 	ep->ep_state |= SET_DEQ_PENDING;
4063 }
4064 
4065 int xhci_queue_reset_ep(struct xhci_hcd *xhci, struct xhci_command *cmd,
4066 			int slot_id, unsigned int ep_index,
4067 			enum xhci_ep_reset_type reset_type)
4068 {
4069 	u32 trb_slot_id = SLOT_ID_FOR_TRB(slot_id);
4070 	u32 trb_ep_index = EP_ID_FOR_TRB(ep_index);
4071 	u32 type = TRB_TYPE(TRB_RESET_EP);
4072 
4073 	if (reset_type == EP_SOFT_RESET)
4074 		type |= TRB_TSP;
4075 
4076 	return queue_command(xhci, cmd, 0, 0, 0,
4077 			trb_slot_id | trb_ep_index | type, false);
4078 }
4079