xref: /openbmc/qemu/hw/usb/hcd-xhci.c (revision 4a09d0bb)
1 /*
2  * USB xHCI controller emulation
3  *
4  * Copyright (c) 2011 Securiforest
5  * Date: 2011-05-11 ;  Author: Hector Martin <hector@marcansoft.com>
6  * Based on usb-ohci.c, emulates Renesas NEC USB 3.0
7  *
8  * This library is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2 of the License, or (at your option) any later version.
12  *
13  * This library is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with this library; if not, see <http://www.gnu.org/licenses/>.
20  */
21 #include "qemu/osdep.h"
22 #include "hw/hw.h"
23 #include "qemu/timer.h"
24 #include "qemu/queue.h"
25 #include "hw/usb.h"
26 #include "hw/pci/pci.h"
27 #include "hw/pci/msi.h"
28 #include "hw/pci/msix.h"
29 #include "trace.h"
30 #include "qapi/error.h"
31 
32 //#define DEBUG_XHCI
33 //#define DEBUG_DATA
34 
35 #ifdef DEBUG_XHCI
36 #define DPRINTF(...) fprintf(stderr, __VA_ARGS__)
37 #else
38 #define DPRINTF(...) do {} while (0)
39 #endif
40 #define FIXME(_msg) do { fprintf(stderr, "FIXME %s:%d %s\n", \
41                                  __func__, __LINE__, _msg); abort(); } while (0)
42 
43 #define MAXPORTS_2 15
44 #define MAXPORTS_3 15
45 
46 #define MAXPORTS (MAXPORTS_2+MAXPORTS_3)
47 #define MAXSLOTS 64
48 #define MAXINTRS 16
49 
50 /* Very pessimistic, let's hope it's enough for all cases */
51 #define EV_QUEUE (((3 * 24) + 16) * MAXSLOTS)
52 /* Do not deliver ER Full events. NEC's driver does some things not bound
53  * to the specs when it gets them */
54 #define ER_FULL_HACK
55 
56 #define TRB_LINK_LIMIT  4
57 
58 #define LEN_CAP         0x40
59 #define LEN_OPER        (0x400 + 0x10 * MAXPORTS)
60 #define LEN_RUNTIME     ((MAXINTRS + 1) * 0x20)
61 #define LEN_DOORBELL    ((MAXSLOTS + 1) * 0x20)
62 
63 #define OFF_OPER        LEN_CAP
64 #define OFF_RUNTIME     0x1000
65 #define OFF_DOORBELL    0x2000
66 #define OFF_MSIX_TABLE  0x3000
67 #define OFF_MSIX_PBA    0x3800
68 /* must be power of 2 */
69 #define LEN_REGS        0x4000
70 
71 #if (OFF_OPER + LEN_OPER) > OFF_RUNTIME
72 #error Increase OFF_RUNTIME
73 #endif
74 #if (OFF_RUNTIME + LEN_RUNTIME) > OFF_DOORBELL
75 #error Increase OFF_DOORBELL
76 #endif
77 #if (OFF_DOORBELL + LEN_DOORBELL) > LEN_REGS
78 # error Increase LEN_REGS
79 #endif
80 
81 /* bit definitions */
82 #define USBCMD_RS       (1<<0)
83 #define USBCMD_HCRST    (1<<1)
84 #define USBCMD_INTE     (1<<2)
85 #define USBCMD_HSEE     (1<<3)
86 #define USBCMD_LHCRST   (1<<7)
87 #define USBCMD_CSS      (1<<8)
88 #define USBCMD_CRS      (1<<9)
89 #define USBCMD_EWE      (1<<10)
90 #define USBCMD_EU3S     (1<<11)
91 
92 #define USBSTS_HCH      (1<<0)
93 #define USBSTS_HSE      (1<<2)
94 #define USBSTS_EINT     (1<<3)
95 #define USBSTS_PCD      (1<<4)
96 #define USBSTS_SSS      (1<<8)
97 #define USBSTS_RSS      (1<<9)
98 #define USBSTS_SRE      (1<<10)
99 #define USBSTS_CNR      (1<<11)
100 #define USBSTS_HCE      (1<<12)
101 
102 
103 #define PORTSC_CCS          (1<<0)
104 #define PORTSC_PED          (1<<1)
105 #define PORTSC_OCA          (1<<3)
106 #define PORTSC_PR           (1<<4)
107 #define PORTSC_PLS_SHIFT        5
108 #define PORTSC_PLS_MASK     0xf
109 #define PORTSC_PP           (1<<9)
110 #define PORTSC_SPEED_SHIFT      10
111 #define PORTSC_SPEED_MASK   0xf
112 #define PORTSC_SPEED_FULL   (1<<10)
113 #define PORTSC_SPEED_LOW    (2<<10)
114 #define PORTSC_SPEED_HIGH   (3<<10)
115 #define PORTSC_SPEED_SUPER  (4<<10)
116 #define PORTSC_PIC_SHIFT        14
117 #define PORTSC_PIC_MASK     0x3
118 #define PORTSC_LWS          (1<<16)
119 #define PORTSC_CSC          (1<<17)
120 #define PORTSC_PEC          (1<<18)
121 #define PORTSC_WRC          (1<<19)
122 #define PORTSC_OCC          (1<<20)
123 #define PORTSC_PRC          (1<<21)
124 #define PORTSC_PLC          (1<<22)
125 #define PORTSC_CEC          (1<<23)
126 #define PORTSC_CAS          (1<<24)
127 #define PORTSC_WCE          (1<<25)
128 #define PORTSC_WDE          (1<<26)
129 #define PORTSC_WOE          (1<<27)
130 #define PORTSC_DR           (1<<30)
131 #define PORTSC_WPR          (1<<31)
132 
133 #define CRCR_RCS        (1<<0)
134 #define CRCR_CS         (1<<1)
135 #define CRCR_CA         (1<<2)
136 #define CRCR_CRR        (1<<3)
137 
138 #define IMAN_IP         (1<<0)
139 #define IMAN_IE         (1<<1)
140 
141 #define ERDP_EHB        (1<<3)
142 
143 #define TRB_SIZE 16
144 typedef struct XHCITRB {
145     uint64_t parameter;
146     uint32_t status;
147     uint32_t control;
148     dma_addr_t addr;
149     bool ccs;
150 } XHCITRB;
151 
152 enum {
153     PLS_U0              =  0,
154     PLS_U1              =  1,
155     PLS_U2              =  2,
156     PLS_U3              =  3,
157     PLS_DISABLED        =  4,
158     PLS_RX_DETECT       =  5,
159     PLS_INACTIVE        =  6,
160     PLS_POLLING         =  7,
161     PLS_RECOVERY        =  8,
162     PLS_HOT_RESET       =  9,
163     PLS_COMPILANCE_MODE = 10,
164     PLS_TEST_MODE       = 11,
165     PLS_RESUME          = 15,
166 };
167 
168 typedef enum TRBType {
169     TRB_RESERVED = 0,
170     TR_NORMAL,
171     TR_SETUP,
172     TR_DATA,
173     TR_STATUS,
174     TR_ISOCH,
175     TR_LINK,
176     TR_EVDATA,
177     TR_NOOP,
178     CR_ENABLE_SLOT,
179     CR_DISABLE_SLOT,
180     CR_ADDRESS_DEVICE,
181     CR_CONFIGURE_ENDPOINT,
182     CR_EVALUATE_CONTEXT,
183     CR_RESET_ENDPOINT,
184     CR_STOP_ENDPOINT,
185     CR_SET_TR_DEQUEUE,
186     CR_RESET_DEVICE,
187     CR_FORCE_EVENT,
188     CR_NEGOTIATE_BW,
189     CR_SET_LATENCY_TOLERANCE,
190     CR_GET_PORT_BANDWIDTH,
191     CR_FORCE_HEADER,
192     CR_NOOP,
193     ER_TRANSFER = 32,
194     ER_COMMAND_COMPLETE,
195     ER_PORT_STATUS_CHANGE,
196     ER_BANDWIDTH_REQUEST,
197     ER_DOORBELL,
198     ER_HOST_CONTROLLER,
199     ER_DEVICE_NOTIFICATION,
200     ER_MFINDEX_WRAP,
201     /* vendor specific bits */
202     CR_VENDOR_VIA_CHALLENGE_RESPONSE = 48,
203     CR_VENDOR_NEC_FIRMWARE_REVISION  = 49,
204     CR_VENDOR_NEC_CHALLENGE_RESPONSE = 50,
205 } TRBType;
206 
207 #define CR_LINK TR_LINK
208 
209 typedef enum TRBCCode {
210     CC_INVALID = 0,
211     CC_SUCCESS,
212     CC_DATA_BUFFER_ERROR,
213     CC_BABBLE_DETECTED,
214     CC_USB_TRANSACTION_ERROR,
215     CC_TRB_ERROR,
216     CC_STALL_ERROR,
217     CC_RESOURCE_ERROR,
218     CC_BANDWIDTH_ERROR,
219     CC_NO_SLOTS_ERROR,
220     CC_INVALID_STREAM_TYPE_ERROR,
221     CC_SLOT_NOT_ENABLED_ERROR,
222     CC_EP_NOT_ENABLED_ERROR,
223     CC_SHORT_PACKET,
224     CC_RING_UNDERRUN,
225     CC_RING_OVERRUN,
226     CC_VF_ER_FULL,
227     CC_PARAMETER_ERROR,
228     CC_BANDWIDTH_OVERRUN,
229     CC_CONTEXT_STATE_ERROR,
230     CC_NO_PING_RESPONSE_ERROR,
231     CC_EVENT_RING_FULL_ERROR,
232     CC_INCOMPATIBLE_DEVICE_ERROR,
233     CC_MISSED_SERVICE_ERROR,
234     CC_COMMAND_RING_STOPPED,
235     CC_COMMAND_ABORTED,
236     CC_STOPPED,
237     CC_STOPPED_LENGTH_INVALID,
238     CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR = 29,
239     CC_ISOCH_BUFFER_OVERRUN = 31,
240     CC_EVENT_LOST_ERROR,
241     CC_UNDEFINED_ERROR,
242     CC_INVALID_STREAM_ID_ERROR,
243     CC_SECONDARY_BANDWIDTH_ERROR,
244     CC_SPLIT_TRANSACTION_ERROR
245 } TRBCCode;
246 
247 #define TRB_C               (1<<0)
248 #define TRB_TYPE_SHIFT          10
249 #define TRB_TYPE_MASK       0x3f
250 #define TRB_TYPE(t)         (((t).control >> TRB_TYPE_SHIFT) & TRB_TYPE_MASK)
251 
252 #define TRB_EV_ED           (1<<2)
253 
254 #define TRB_TR_ENT          (1<<1)
255 #define TRB_TR_ISP          (1<<2)
256 #define TRB_TR_NS           (1<<3)
257 #define TRB_TR_CH           (1<<4)
258 #define TRB_TR_IOC          (1<<5)
259 #define TRB_TR_IDT          (1<<6)
260 #define TRB_TR_TBC_SHIFT        7
261 #define TRB_TR_TBC_MASK     0x3
262 #define TRB_TR_BEI          (1<<9)
263 #define TRB_TR_TLBPC_SHIFT      16
264 #define TRB_TR_TLBPC_MASK   0xf
265 #define TRB_TR_FRAMEID_SHIFT    20
266 #define TRB_TR_FRAMEID_MASK 0x7ff
267 #define TRB_TR_SIA          (1<<31)
268 
269 #define TRB_TR_DIR          (1<<16)
270 
271 #define TRB_CR_SLOTID_SHIFT     24
272 #define TRB_CR_SLOTID_MASK  0xff
273 #define TRB_CR_EPID_SHIFT       16
274 #define TRB_CR_EPID_MASK    0x1f
275 
276 #define TRB_CR_BSR          (1<<9)
277 #define TRB_CR_DC           (1<<9)
278 
279 #define TRB_LK_TC           (1<<1)
280 
281 #define TRB_INTR_SHIFT          22
282 #define TRB_INTR_MASK       0x3ff
283 #define TRB_INTR(t)         (((t).status >> TRB_INTR_SHIFT) & TRB_INTR_MASK)
284 
285 #define EP_TYPE_MASK        0x7
286 #define EP_TYPE_SHIFT           3
287 
288 #define EP_STATE_MASK       0x7
289 #define EP_DISABLED         (0<<0)
290 #define EP_RUNNING          (1<<0)
291 #define EP_HALTED           (2<<0)
292 #define EP_STOPPED          (3<<0)
293 #define EP_ERROR            (4<<0)
294 
295 #define SLOT_STATE_MASK     0x1f
296 #define SLOT_STATE_SHIFT        27
297 #define SLOT_STATE(s)       (((s)>>SLOT_STATE_SHIFT)&SLOT_STATE_MASK)
298 #define SLOT_ENABLED        0
299 #define SLOT_DEFAULT        1
300 #define SLOT_ADDRESSED      2
301 #define SLOT_CONFIGURED     3
302 
303 #define SLOT_CONTEXT_ENTRIES_MASK 0x1f
304 #define SLOT_CONTEXT_ENTRIES_SHIFT 27
305 
306 typedef struct XHCIState XHCIState;
307 typedef struct XHCIStreamContext XHCIStreamContext;
308 typedef struct XHCIEPContext XHCIEPContext;
309 
310 #define get_field(data, field)                  \
311     (((data) >> field##_SHIFT) & field##_MASK)
312 
313 #define set_field(data, newval, field) do {                     \
314         uint32_t val = *data;                                   \
315         val &= ~(field##_MASK << field##_SHIFT);                \
316         val |= ((newval) & field##_MASK) << field##_SHIFT;      \
317         *data = val;                                            \
318     } while (0)
319 
320 typedef enum EPType {
321     ET_INVALID = 0,
322     ET_ISO_OUT,
323     ET_BULK_OUT,
324     ET_INTR_OUT,
325     ET_CONTROL,
326     ET_ISO_IN,
327     ET_BULK_IN,
328     ET_INTR_IN,
329 } EPType;
330 
331 typedef struct XHCIRing {
332     dma_addr_t dequeue;
333     bool ccs;
334 } XHCIRing;
335 
336 typedef struct XHCIPort {
337     XHCIState *xhci;
338     uint32_t portsc;
339     uint32_t portnr;
340     USBPort  *uport;
341     uint32_t speedmask;
342     char name[16];
343     MemoryRegion mem;
344 } XHCIPort;
345 
346 typedef struct XHCITransfer {
347     XHCIEPContext *epctx;
348     USBPacket packet;
349     QEMUSGList sgl;
350     bool running_async;
351     bool running_retry;
352     bool complete;
353     bool int_req;
354     unsigned int iso_pkts;
355     unsigned int streamid;
356     bool in_xfer;
357     bool iso_xfer;
358     bool timed_xfer;
359 
360     unsigned int trb_count;
361     XHCITRB *trbs;
362 
363     TRBCCode status;
364 
365     unsigned int pkts;
366     unsigned int pktsize;
367     unsigned int cur_pkt;
368 
369     uint64_t mfindex_kick;
370 
371     QTAILQ_ENTRY(XHCITransfer) next;
372 } XHCITransfer;
373 
374 struct XHCIStreamContext {
375     dma_addr_t pctx;
376     unsigned int sct;
377     XHCIRing ring;
378 };
379 
380 struct XHCIEPContext {
381     XHCIState *xhci;
382     unsigned int slotid;
383     unsigned int epid;
384 
385     XHCIRing ring;
386     uint32_t xfer_count;
387     QTAILQ_HEAD(, XHCITransfer) transfers;
388     XHCITransfer *retry;
389     EPType type;
390     dma_addr_t pctx;
391     unsigned int max_psize;
392     uint32_t state;
393     uint32_t kick_active;
394 
395     /* streams */
396     unsigned int max_pstreams;
397     bool         lsa;
398     unsigned int nr_pstreams;
399     XHCIStreamContext *pstreams;
400 
401     /* iso xfer scheduling */
402     unsigned int interval;
403     int64_t mfindex_last;
404     QEMUTimer *kick_timer;
405 };
406 
407 typedef struct XHCISlot {
408     bool enabled;
409     bool addressed;
410     dma_addr_t ctx;
411     USBPort *uport;
412     XHCIEPContext * eps[31];
413 } XHCISlot;
414 
415 typedef struct XHCIEvent {
416     TRBType type;
417     TRBCCode ccode;
418     uint64_t ptr;
419     uint32_t length;
420     uint32_t flags;
421     uint8_t slotid;
422     uint8_t epid;
423 } XHCIEvent;
424 
425 typedef struct XHCIInterrupter {
426     uint32_t iman;
427     uint32_t imod;
428     uint32_t erstsz;
429     uint32_t erstba_low;
430     uint32_t erstba_high;
431     uint32_t erdp_low;
432     uint32_t erdp_high;
433 
434     bool msix_used, er_pcs, er_full;
435 
436     dma_addr_t er_start;
437     uint32_t er_size;
438     unsigned int er_ep_idx;
439 
440     XHCIEvent ev_buffer[EV_QUEUE];
441     unsigned int ev_buffer_put;
442     unsigned int ev_buffer_get;
443 
444 } XHCIInterrupter;
445 
446 struct XHCIState {
447     /*< private >*/
448     PCIDevice parent_obj;
449     /*< public >*/
450 
451     USBBus bus;
452     MemoryRegion mem;
453     MemoryRegion mem_cap;
454     MemoryRegion mem_oper;
455     MemoryRegion mem_runtime;
456     MemoryRegion mem_doorbell;
457 
458     /* properties */
459     uint32_t numports_2;
460     uint32_t numports_3;
461     uint32_t numintrs;
462     uint32_t numslots;
463     uint32_t flags;
464     uint32_t max_pstreams_mask;
465     OnOffAuto msi;
466     OnOffAuto msix;
467 
468     /* Operational Registers */
469     uint32_t usbcmd;
470     uint32_t usbsts;
471     uint32_t dnctrl;
472     uint32_t crcr_low;
473     uint32_t crcr_high;
474     uint32_t dcbaap_low;
475     uint32_t dcbaap_high;
476     uint32_t config;
477 
478     USBPort  uports[MAX(MAXPORTS_2, MAXPORTS_3)];
479     XHCIPort ports[MAXPORTS];
480     XHCISlot slots[MAXSLOTS];
481     uint32_t numports;
482 
483     /* Runtime Registers */
484     int64_t mfindex_start;
485     QEMUTimer *mfwrap_timer;
486     XHCIInterrupter intr[MAXINTRS];
487 
488     XHCIRing cmd_ring;
489 };
490 
491 #define TYPE_XHCI "nec-usb-xhci"
492 
493 #define XHCI(obj) \
494     OBJECT_CHECK(XHCIState, (obj), TYPE_XHCI)
495 
496 typedef struct XHCIEvRingSeg {
497     uint32_t addr_low;
498     uint32_t addr_high;
499     uint32_t size;
500     uint32_t rsvd;
501 } XHCIEvRingSeg;
502 
503 enum xhci_flags {
504     XHCI_FLAG_SS_FIRST = 1,
505     XHCI_FLAG_FORCE_PCIE_ENDCAP,
506     XHCI_FLAG_ENABLE_STREAMS,
507 };
508 
509 static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
510                          unsigned int epid, unsigned int streamid);
511 static void xhci_kick_epctx(XHCIEPContext *epctx, unsigned int streamid);
512 static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
513                                 unsigned int epid);
514 static void xhci_xfer_report(XHCITransfer *xfer);
515 static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v);
516 static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v);
517 static USBEndpoint *xhci_epid_to_usbep(XHCIEPContext *epctx);
518 
519 static const char *TRBType_names[] = {
520     [TRB_RESERVED]                     = "TRB_RESERVED",
521     [TR_NORMAL]                        = "TR_NORMAL",
522     [TR_SETUP]                         = "TR_SETUP",
523     [TR_DATA]                          = "TR_DATA",
524     [TR_STATUS]                        = "TR_STATUS",
525     [TR_ISOCH]                         = "TR_ISOCH",
526     [TR_LINK]                          = "TR_LINK",
527     [TR_EVDATA]                        = "TR_EVDATA",
528     [TR_NOOP]                          = "TR_NOOP",
529     [CR_ENABLE_SLOT]                   = "CR_ENABLE_SLOT",
530     [CR_DISABLE_SLOT]                  = "CR_DISABLE_SLOT",
531     [CR_ADDRESS_DEVICE]                = "CR_ADDRESS_DEVICE",
532     [CR_CONFIGURE_ENDPOINT]            = "CR_CONFIGURE_ENDPOINT",
533     [CR_EVALUATE_CONTEXT]              = "CR_EVALUATE_CONTEXT",
534     [CR_RESET_ENDPOINT]                = "CR_RESET_ENDPOINT",
535     [CR_STOP_ENDPOINT]                 = "CR_STOP_ENDPOINT",
536     [CR_SET_TR_DEQUEUE]                = "CR_SET_TR_DEQUEUE",
537     [CR_RESET_DEVICE]                  = "CR_RESET_DEVICE",
538     [CR_FORCE_EVENT]                   = "CR_FORCE_EVENT",
539     [CR_NEGOTIATE_BW]                  = "CR_NEGOTIATE_BW",
540     [CR_SET_LATENCY_TOLERANCE]         = "CR_SET_LATENCY_TOLERANCE",
541     [CR_GET_PORT_BANDWIDTH]            = "CR_GET_PORT_BANDWIDTH",
542     [CR_FORCE_HEADER]                  = "CR_FORCE_HEADER",
543     [CR_NOOP]                          = "CR_NOOP",
544     [ER_TRANSFER]                      = "ER_TRANSFER",
545     [ER_COMMAND_COMPLETE]              = "ER_COMMAND_COMPLETE",
546     [ER_PORT_STATUS_CHANGE]            = "ER_PORT_STATUS_CHANGE",
547     [ER_BANDWIDTH_REQUEST]             = "ER_BANDWIDTH_REQUEST",
548     [ER_DOORBELL]                      = "ER_DOORBELL",
549     [ER_HOST_CONTROLLER]               = "ER_HOST_CONTROLLER",
550     [ER_DEVICE_NOTIFICATION]           = "ER_DEVICE_NOTIFICATION",
551     [ER_MFINDEX_WRAP]                  = "ER_MFINDEX_WRAP",
552     [CR_VENDOR_VIA_CHALLENGE_RESPONSE] = "CR_VENDOR_VIA_CHALLENGE_RESPONSE",
553     [CR_VENDOR_NEC_FIRMWARE_REVISION]  = "CR_VENDOR_NEC_FIRMWARE_REVISION",
554     [CR_VENDOR_NEC_CHALLENGE_RESPONSE] = "CR_VENDOR_NEC_CHALLENGE_RESPONSE",
555 };
556 
557 static const char *TRBCCode_names[] = {
558     [CC_INVALID]                       = "CC_INVALID",
559     [CC_SUCCESS]                       = "CC_SUCCESS",
560     [CC_DATA_BUFFER_ERROR]             = "CC_DATA_BUFFER_ERROR",
561     [CC_BABBLE_DETECTED]               = "CC_BABBLE_DETECTED",
562     [CC_USB_TRANSACTION_ERROR]         = "CC_USB_TRANSACTION_ERROR",
563     [CC_TRB_ERROR]                     = "CC_TRB_ERROR",
564     [CC_STALL_ERROR]                   = "CC_STALL_ERROR",
565     [CC_RESOURCE_ERROR]                = "CC_RESOURCE_ERROR",
566     [CC_BANDWIDTH_ERROR]               = "CC_BANDWIDTH_ERROR",
567     [CC_NO_SLOTS_ERROR]                = "CC_NO_SLOTS_ERROR",
568     [CC_INVALID_STREAM_TYPE_ERROR]     = "CC_INVALID_STREAM_TYPE_ERROR",
569     [CC_SLOT_NOT_ENABLED_ERROR]        = "CC_SLOT_NOT_ENABLED_ERROR",
570     [CC_EP_NOT_ENABLED_ERROR]          = "CC_EP_NOT_ENABLED_ERROR",
571     [CC_SHORT_PACKET]                  = "CC_SHORT_PACKET",
572     [CC_RING_UNDERRUN]                 = "CC_RING_UNDERRUN",
573     [CC_RING_OVERRUN]                  = "CC_RING_OVERRUN",
574     [CC_VF_ER_FULL]                    = "CC_VF_ER_FULL",
575     [CC_PARAMETER_ERROR]               = "CC_PARAMETER_ERROR",
576     [CC_BANDWIDTH_OVERRUN]             = "CC_BANDWIDTH_OVERRUN",
577     [CC_CONTEXT_STATE_ERROR]           = "CC_CONTEXT_STATE_ERROR",
578     [CC_NO_PING_RESPONSE_ERROR]        = "CC_NO_PING_RESPONSE_ERROR",
579     [CC_EVENT_RING_FULL_ERROR]         = "CC_EVENT_RING_FULL_ERROR",
580     [CC_INCOMPATIBLE_DEVICE_ERROR]     = "CC_INCOMPATIBLE_DEVICE_ERROR",
581     [CC_MISSED_SERVICE_ERROR]          = "CC_MISSED_SERVICE_ERROR",
582     [CC_COMMAND_RING_STOPPED]          = "CC_COMMAND_RING_STOPPED",
583     [CC_COMMAND_ABORTED]               = "CC_COMMAND_ABORTED",
584     [CC_STOPPED]                       = "CC_STOPPED",
585     [CC_STOPPED_LENGTH_INVALID]        = "CC_STOPPED_LENGTH_INVALID",
586     [CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR]
587     = "CC_MAX_EXIT_LATENCY_TOO_LARGE_ERROR",
588     [CC_ISOCH_BUFFER_OVERRUN]          = "CC_ISOCH_BUFFER_OVERRUN",
589     [CC_EVENT_LOST_ERROR]              = "CC_EVENT_LOST_ERROR",
590     [CC_UNDEFINED_ERROR]               = "CC_UNDEFINED_ERROR",
591     [CC_INVALID_STREAM_ID_ERROR]       = "CC_INVALID_STREAM_ID_ERROR",
592     [CC_SECONDARY_BANDWIDTH_ERROR]     = "CC_SECONDARY_BANDWIDTH_ERROR",
593     [CC_SPLIT_TRANSACTION_ERROR]       = "CC_SPLIT_TRANSACTION_ERROR",
594 };
595 
596 static const char *ep_state_names[] = {
597     [EP_DISABLED] = "disabled",
598     [EP_RUNNING]  = "running",
599     [EP_HALTED]   = "halted",
600     [EP_STOPPED]  = "stopped",
601     [EP_ERROR]    = "error",
602 };
603 
604 static const char *lookup_name(uint32_t index, const char **list, uint32_t llen)
605 {
606     if (index >= llen || list[index] == NULL) {
607         return "???";
608     }
609     return list[index];
610 }
611 
612 static const char *trb_name(XHCITRB *trb)
613 {
614     return lookup_name(TRB_TYPE(*trb), TRBType_names,
615                        ARRAY_SIZE(TRBType_names));
616 }
617 
618 static const char *event_name(XHCIEvent *event)
619 {
620     return lookup_name(event->ccode, TRBCCode_names,
621                        ARRAY_SIZE(TRBCCode_names));
622 }
623 
624 static const char *ep_state_name(uint32_t state)
625 {
626     return lookup_name(state, ep_state_names,
627                        ARRAY_SIZE(ep_state_names));
628 }
629 
630 static bool xhci_get_flag(XHCIState *xhci, enum xhci_flags bit)
631 {
632     return xhci->flags & (1 << bit);
633 }
634 
635 static uint64_t xhci_mfindex_get(XHCIState *xhci)
636 {
637     int64_t now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
638     return (now - xhci->mfindex_start) / 125000;
639 }
640 
641 static void xhci_mfwrap_update(XHCIState *xhci)
642 {
643     const uint32_t bits = USBCMD_RS | USBCMD_EWE;
644     uint32_t mfindex, left;
645     int64_t now;
646 
647     if ((xhci->usbcmd & bits) == bits) {
648         now = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
649         mfindex = ((now - xhci->mfindex_start) / 125000) & 0x3fff;
650         left = 0x4000 - mfindex;
651         timer_mod(xhci->mfwrap_timer, now + left * 125000);
652     } else {
653         timer_del(xhci->mfwrap_timer);
654     }
655 }
656 
657 static void xhci_mfwrap_timer(void *opaque)
658 {
659     XHCIState *xhci = opaque;
660     XHCIEvent wrap = { ER_MFINDEX_WRAP, CC_SUCCESS };
661 
662     xhci_event(xhci, &wrap, 0);
663     xhci_mfwrap_update(xhci);
664 }
665 
666 static inline dma_addr_t xhci_addr64(uint32_t low, uint32_t high)
667 {
668     if (sizeof(dma_addr_t) == 4) {
669         return low;
670     } else {
671         return low | (((dma_addr_t)high << 16) << 16);
672     }
673 }
674 
675 static inline dma_addr_t xhci_mask64(uint64_t addr)
676 {
677     if (sizeof(dma_addr_t) == 4) {
678         return addr & 0xffffffff;
679     } else {
680         return addr;
681     }
682 }
683 
684 static inline void xhci_dma_read_u32s(XHCIState *xhci, dma_addr_t addr,
685                                       uint32_t *buf, size_t len)
686 {
687     int i;
688 
689     assert((len % sizeof(uint32_t)) == 0);
690 
691     pci_dma_read(PCI_DEVICE(xhci), addr, buf, len);
692 
693     for (i = 0; i < (len / sizeof(uint32_t)); i++) {
694         buf[i] = le32_to_cpu(buf[i]);
695     }
696 }
697 
698 static inline void xhci_dma_write_u32s(XHCIState *xhci, dma_addr_t addr,
699                                        uint32_t *buf, size_t len)
700 {
701     int i;
702     uint32_t tmp[5];
703     uint32_t n = len / sizeof(uint32_t);
704 
705     assert((len % sizeof(uint32_t)) == 0);
706     assert(n <= ARRAY_SIZE(tmp));
707 
708     for (i = 0; i < n; i++) {
709         tmp[i] = cpu_to_le32(buf[i]);
710     }
711     pci_dma_write(PCI_DEVICE(xhci), addr, tmp, len);
712 }
713 
714 static XHCIPort *xhci_lookup_port(XHCIState *xhci, struct USBPort *uport)
715 {
716     int index;
717 
718     if (!uport->dev) {
719         return NULL;
720     }
721     switch (uport->dev->speed) {
722     case USB_SPEED_LOW:
723     case USB_SPEED_FULL:
724     case USB_SPEED_HIGH:
725         if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
726             index = uport->index + xhci->numports_3;
727         } else {
728             index = uport->index;
729         }
730         break;
731     case USB_SPEED_SUPER:
732         if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
733             index = uport->index;
734         } else {
735             index = uport->index + xhci->numports_2;
736         }
737         break;
738     default:
739         return NULL;
740     }
741     return &xhci->ports[index];
742 }
743 
744 static void xhci_intx_update(XHCIState *xhci)
745 {
746     PCIDevice *pci_dev = PCI_DEVICE(xhci);
747     int level = 0;
748 
749     if (msix_enabled(pci_dev) ||
750         msi_enabled(pci_dev)) {
751         return;
752     }
753 
754     if (xhci->intr[0].iman & IMAN_IP &&
755         xhci->intr[0].iman & IMAN_IE &&
756         xhci->usbcmd & USBCMD_INTE) {
757         level = 1;
758     }
759 
760     trace_usb_xhci_irq_intx(level);
761     pci_set_irq(pci_dev, level);
762 }
763 
764 static void xhci_msix_update(XHCIState *xhci, int v)
765 {
766     PCIDevice *pci_dev = PCI_DEVICE(xhci);
767     bool enabled;
768 
769     if (!msix_enabled(pci_dev)) {
770         return;
771     }
772 
773     enabled = xhci->intr[v].iman & IMAN_IE;
774     if (enabled == xhci->intr[v].msix_used) {
775         return;
776     }
777 
778     if (enabled) {
779         trace_usb_xhci_irq_msix_use(v);
780         msix_vector_use(pci_dev, v);
781         xhci->intr[v].msix_used = true;
782     } else {
783         trace_usb_xhci_irq_msix_unuse(v);
784         msix_vector_unuse(pci_dev, v);
785         xhci->intr[v].msix_used = false;
786     }
787 }
788 
789 static void xhci_intr_raise(XHCIState *xhci, int v)
790 {
791     PCIDevice *pci_dev = PCI_DEVICE(xhci);
792     bool pending = (xhci->intr[v].erdp_low & ERDP_EHB);
793 
794     xhci->intr[v].erdp_low |= ERDP_EHB;
795     xhci->intr[v].iman |= IMAN_IP;
796     xhci->usbsts |= USBSTS_EINT;
797 
798     if (pending) {
799         return;
800     }
801     if (!(xhci->intr[v].iman & IMAN_IE)) {
802         return;
803     }
804 
805     if (!(xhci->usbcmd & USBCMD_INTE)) {
806         return;
807     }
808 
809     if (msix_enabled(pci_dev)) {
810         trace_usb_xhci_irq_msix(v);
811         msix_notify(pci_dev, v);
812         return;
813     }
814 
815     if (msi_enabled(pci_dev)) {
816         trace_usb_xhci_irq_msi(v);
817         msi_notify(pci_dev, v);
818         return;
819     }
820 
821     if (v == 0) {
822         trace_usb_xhci_irq_intx(1);
823         pci_irq_assert(pci_dev);
824     }
825 }
826 
827 static inline int xhci_running(XHCIState *xhci)
828 {
829     return !(xhci->usbsts & USBSTS_HCH) && !xhci->intr[0].er_full;
830 }
831 
832 static void xhci_die(XHCIState *xhci)
833 {
834     xhci->usbsts |= USBSTS_HCE;
835     DPRINTF("xhci: asserted controller error\n");
836 }
837 
838 static void xhci_write_event(XHCIState *xhci, XHCIEvent *event, int v)
839 {
840     PCIDevice *pci_dev = PCI_DEVICE(xhci);
841     XHCIInterrupter *intr = &xhci->intr[v];
842     XHCITRB ev_trb;
843     dma_addr_t addr;
844 
845     ev_trb.parameter = cpu_to_le64(event->ptr);
846     ev_trb.status = cpu_to_le32(event->length | (event->ccode << 24));
847     ev_trb.control = (event->slotid << 24) | (event->epid << 16) |
848                      event->flags | (event->type << TRB_TYPE_SHIFT);
849     if (intr->er_pcs) {
850         ev_trb.control |= TRB_C;
851     }
852     ev_trb.control = cpu_to_le32(ev_trb.control);
853 
854     trace_usb_xhci_queue_event(v, intr->er_ep_idx, trb_name(&ev_trb),
855                                event_name(event), ev_trb.parameter,
856                                ev_trb.status, ev_trb.control);
857 
858     addr = intr->er_start + TRB_SIZE*intr->er_ep_idx;
859     pci_dma_write(pci_dev, addr, &ev_trb, TRB_SIZE);
860 
861     intr->er_ep_idx++;
862     if (intr->er_ep_idx >= intr->er_size) {
863         intr->er_ep_idx = 0;
864         intr->er_pcs = !intr->er_pcs;
865     }
866 }
867 
868 static void xhci_events_update(XHCIState *xhci, int v)
869 {
870     XHCIInterrupter *intr = &xhci->intr[v];
871     dma_addr_t erdp;
872     unsigned int dp_idx;
873     bool do_irq = 0;
874 
875     if (xhci->usbsts & USBSTS_HCH) {
876         return;
877     }
878 
879     erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
880     if (erdp < intr->er_start ||
881         erdp >= (intr->er_start + TRB_SIZE*intr->er_size)) {
882         DPRINTF("xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);
883         DPRINTF("xhci: ER[%d] at "DMA_ADDR_FMT" len %d\n",
884                 v, intr->er_start, intr->er_size);
885         xhci_die(xhci);
886         return;
887     }
888     dp_idx = (erdp - intr->er_start) / TRB_SIZE;
889     assert(dp_idx < intr->er_size);
890 
891     /* NEC didn't read section 4.9.4 of the spec (v1.0 p139 top Note) and thus
892      * deadlocks when the ER is full. Hack it by holding off events until
893      * the driver decides to free at least half of the ring */
894     if (intr->er_full) {
895         int er_free = dp_idx - intr->er_ep_idx;
896         if (er_free <= 0) {
897             er_free += intr->er_size;
898         }
899         if (er_free < (intr->er_size/2)) {
900             DPRINTF("xhci_events_update(): event ring still "
901                     "more than half full (hack)\n");
902             return;
903         }
904     }
905 
906     while (intr->ev_buffer_put != intr->ev_buffer_get) {
907         assert(intr->er_full);
908         if (((intr->er_ep_idx+1) % intr->er_size) == dp_idx) {
909             DPRINTF("xhci_events_update(): event ring full again\n");
910 #ifndef ER_FULL_HACK
911             XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};
912             xhci_write_event(xhci, &full, v);
913 #endif
914             do_irq = 1;
915             break;
916         }
917         XHCIEvent *event = &intr->ev_buffer[intr->ev_buffer_get];
918         xhci_write_event(xhci, event, v);
919         intr->ev_buffer_get++;
920         do_irq = 1;
921         if (intr->ev_buffer_get == EV_QUEUE) {
922             intr->ev_buffer_get = 0;
923         }
924     }
925 
926     if (do_irq) {
927         xhci_intr_raise(xhci, v);
928     }
929 
930     if (intr->er_full && intr->ev_buffer_put == intr->ev_buffer_get) {
931         DPRINTF("xhci_events_update(): event ring no longer full\n");
932         intr->er_full = 0;
933     }
934 }
935 
936 static void xhci_event(XHCIState *xhci, XHCIEvent *event, int v)
937 {
938     XHCIInterrupter *intr;
939     dma_addr_t erdp;
940     unsigned int dp_idx;
941 
942     if (v >= xhci->numintrs) {
943         DPRINTF("intr nr out of range (%d >= %d)\n", v, xhci->numintrs);
944         return;
945     }
946     intr = &xhci->intr[v];
947 
948     if (intr->er_full) {
949         DPRINTF("xhci_event(): ER full, queueing\n");
950         if (((intr->ev_buffer_put+1) % EV_QUEUE) == intr->ev_buffer_get) {
951             DPRINTF("xhci: event queue full, dropping event!\n");
952             return;
953         }
954         intr->ev_buffer[intr->ev_buffer_put++] = *event;
955         if (intr->ev_buffer_put == EV_QUEUE) {
956             intr->ev_buffer_put = 0;
957         }
958         return;
959     }
960 
961     erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
962     if (erdp < intr->er_start ||
963         erdp >= (intr->er_start + TRB_SIZE*intr->er_size)) {
964         DPRINTF("xhci: ERDP out of bounds: "DMA_ADDR_FMT"\n", erdp);
965         DPRINTF("xhci: ER[%d] at "DMA_ADDR_FMT" len %d\n",
966                 v, intr->er_start, intr->er_size);
967         xhci_die(xhci);
968         return;
969     }
970 
971     dp_idx = (erdp - intr->er_start) / TRB_SIZE;
972     assert(dp_idx < intr->er_size);
973 
974     if ((intr->er_ep_idx+1) % intr->er_size == dp_idx) {
975         DPRINTF("xhci_event(): ER full, queueing\n");
976 #ifndef ER_FULL_HACK
977         XHCIEvent full = {ER_HOST_CONTROLLER, CC_EVENT_RING_FULL_ERROR};
978         xhci_write_event(xhci, &full);
979 #endif
980         intr->er_full = 1;
981         if (((intr->ev_buffer_put+1) % EV_QUEUE) == intr->ev_buffer_get) {
982             DPRINTF("xhci: event queue full, dropping event!\n");
983             return;
984         }
985         intr->ev_buffer[intr->ev_buffer_put++] = *event;
986         if (intr->ev_buffer_put == EV_QUEUE) {
987             intr->ev_buffer_put = 0;
988         }
989     } else {
990         xhci_write_event(xhci, event, v);
991     }
992 
993     xhci_intr_raise(xhci, v);
994 }
995 
996 static void xhci_ring_init(XHCIState *xhci, XHCIRing *ring,
997                            dma_addr_t base)
998 {
999     ring->dequeue = base;
1000     ring->ccs = 1;
1001 }
1002 
1003 static TRBType xhci_ring_fetch(XHCIState *xhci, XHCIRing *ring, XHCITRB *trb,
1004                                dma_addr_t *addr)
1005 {
1006     PCIDevice *pci_dev = PCI_DEVICE(xhci);
1007     uint32_t link_cnt = 0;
1008 
1009     while (1) {
1010         TRBType type;
1011         pci_dma_read(pci_dev, ring->dequeue, trb, TRB_SIZE);
1012         trb->addr = ring->dequeue;
1013         trb->ccs = ring->ccs;
1014         le64_to_cpus(&trb->parameter);
1015         le32_to_cpus(&trb->status);
1016         le32_to_cpus(&trb->control);
1017 
1018         trace_usb_xhci_fetch_trb(ring->dequeue, trb_name(trb),
1019                                  trb->parameter, trb->status, trb->control);
1020 
1021         if ((trb->control & TRB_C) != ring->ccs) {
1022             return 0;
1023         }
1024 
1025         type = TRB_TYPE(*trb);
1026 
1027         if (type != TR_LINK) {
1028             if (addr) {
1029                 *addr = ring->dequeue;
1030             }
1031             ring->dequeue += TRB_SIZE;
1032             return type;
1033         } else {
1034             if (++link_cnt > TRB_LINK_LIMIT) {
1035                 return 0;
1036             }
1037             ring->dequeue = xhci_mask64(trb->parameter);
1038             if (trb->control & TRB_LK_TC) {
1039                 ring->ccs = !ring->ccs;
1040             }
1041         }
1042     }
1043 }
1044 
1045 static int xhci_ring_chain_length(XHCIState *xhci, const XHCIRing *ring)
1046 {
1047     PCIDevice *pci_dev = PCI_DEVICE(xhci);
1048     XHCITRB trb;
1049     int length = 0;
1050     dma_addr_t dequeue = ring->dequeue;
1051     bool ccs = ring->ccs;
1052     /* hack to bundle together the two/three TDs that make a setup transfer */
1053     bool control_td_set = 0;
1054     uint32_t link_cnt = 0;
1055 
1056     while (1) {
1057         TRBType type;
1058         pci_dma_read(pci_dev, dequeue, &trb, TRB_SIZE);
1059         le64_to_cpus(&trb.parameter);
1060         le32_to_cpus(&trb.status);
1061         le32_to_cpus(&trb.control);
1062 
1063         if ((trb.control & TRB_C) != ccs) {
1064             return -length;
1065         }
1066 
1067         type = TRB_TYPE(trb);
1068 
1069         if (type == TR_LINK) {
1070             if (++link_cnt > TRB_LINK_LIMIT) {
1071                 return -length;
1072             }
1073             dequeue = xhci_mask64(trb.parameter);
1074             if (trb.control & TRB_LK_TC) {
1075                 ccs = !ccs;
1076             }
1077             continue;
1078         }
1079 
1080         length += 1;
1081         dequeue += TRB_SIZE;
1082 
1083         if (type == TR_SETUP) {
1084             control_td_set = 1;
1085         } else if (type == TR_STATUS) {
1086             control_td_set = 0;
1087         }
1088 
1089         if (!control_td_set && !(trb.control & TRB_TR_CH)) {
1090             return length;
1091         }
1092     }
1093 }
1094 
1095 static void xhci_er_reset(XHCIState *xhci, int v)
1096 {
1097     XHCIInterrupter *intr = &xhci->intr[v];
1098     XHCIEvRingSeg seg;
1099 
1100     if (intr->erstsz == 0) {
1101         /* disabled */
1102         intr->er_start = 0;
1103         intr->er_size = 0;
1104         return;
1105     }
1106     /* cache the (sole) event ring segment location */
1107     if (intr->erstsz != 1) {
1108         DPRINTF("xhci: invalid value for ERSTSZ: %d\n", intr->erstsz);
1109         xhci_die(xhci);
1110         return;
1111     }
1112     dma_addr_t erstba = xhci_addr64(intr->erstba_low, intr->erstba_high);
1113     pci_dma_read(PCI_DEVICE(xhci), erstba, &seg, sizeof(seg));
1114     le32_to_cpus(&seg.addr_low);
1115     le32_to_cpus(&seg.addr_high);
1116     le32_to_cpus(&seg.size);
1117     if (seg.size < 16 || seg.size > 4096) {
1118         DPRINTF("xhci: invalid value for segment size: %d\n", seg.size);
1119         xhci_die(xhci);
1120         return;
1121     }
1122     intr->er_start = xhci_addr64(seg.addr_low, seg.addr_high);
1123     intr->er_size = seg.size;
1124 
1125     intr->er_ep_idx = 0;
1126     intr->er_pcs = 1;
1127     intr->er_full = 0;
1128 
1129     DPRINTF("xhci: event ring[%d]:" DMA_ADDR_FMT " [%d]\n",
1130             v, intr->er_start, intr->er_size);
1131 }
1132 
1133 static void xhci_run(XHCIState *xhci)
1134 {
1135     trace_usb_xhci_run();
1136     xhci->usbsts &= ~USBSTS_HCH;
1137     xhci->mfindex_start = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
1138 }
1139 
1140 static void xhci_stop(XHCIState *xhci)
1141 {
1142     trace_usb_xhci_stop();
1143     xhci->usbsts |= USBSTS_HCH;
1144     xhci->crcr_low &= ~CRCR_CRR;
1145 }
1146 
1147 static XHCIStreamContext *xhci_alloc_stream_contexts(unsigned count,
1148                                                      dma_addr_t base)
1149 {
1150     XHCIStreamContext *stctx;
1151     unsigned int i;
1152 
1153     stctx = g_new0(XHCIStreamContext, count);
1154     for (i = 0; i < count; i++) {
1155         stctx[i].pctx = base + i * 16;
1156         stctx[i].sct = -1;
1157     }
1158     return stctx;
1159 }
1160 
1161 static void xhci_reset_streams(XHCIEPContext *epctx)
1162 {
1163     unsigned int i;
1164 
1165     for (i = 0; i < epctx->nr_pstreams; i++) {
1166         epctx->pstreams[i].sct = -1;
1167     }
1168 }
1169 
1170 static void xhci_alloc_streams(XHCIEPContext *epctx, dma_addr_t base)
1171 {
1172     assert(epctx->pstreams == NULL);
1173     epctx->nr_pstreams = 2 << epctx->max_pstreams;
1174     epctx->pstreams = xhci_alloc_stream_contexts(epctx->nr_pstreams, base);
1175 }
1176 
1177 static void xhci_free_streams(XHCIEPContext *epctx)
1178 {
1179     assert(epctx->pstreams != NULL);
1180 
1181     g_free(epctx->pstreams);
1182     epctx->pstreams = NULL;
1183     epctx->nr_pstreams = 0;
1184 }
1185 
1186 static int xhci_epmask_to_eps_with_streams(XHCIState *xhci,
1187                                            unsigned int slotid,
1188                                            uint32_t epmask,
1189                                            XHCIEPContext **epctxs,
1190                                            USBEndpoint **eps)
1191 {
1192     XHCISlot *slot;
1193     XHCIEPContext *epctx;
1194     USBEndpoint *ep;
1195     int i, j;
1196 
1197     assert(slotid >= 1 && slotid <= xhci->numslots);
1198 
1199     slot = &xhci->slots[slotid - 1];
1200 
1201     for (i = 2, j = 0; i <= 31; i++) {
1202         if (!(epmask & (1u << i))) {
1203             continue;
1204         }
1205 
1206         epctx = slot->eps[i - 1];
1207         ep = xhci_epid_to_usbep(epctx);
1208         if (!epctx || !epctx->nr_pstreams || !ep) {
1209             continue;
1210         }
1211 
1212         if (epctxs) {
1213             epctxs[j] = epctx;
1214         }
1215         eps[j++] = ep;
1216     }
1217     return j;
1218 }
1219 
1220 static void xhci_free_device_streams(XHCIState *xhci, unsigned int slotid,
1221                                      uint32_t epmask)
1222 {
1223     USBEndpoint *eps[30];
1224     int nr_eps;
1225 
1226     nr_eps = xhci_epmask_to_eps_with_streams(xhci, slotid, epmask, NULL, eps);
1227     if (nr_eps) {
1228         usb_device_free_streams(eps[0]->dev, eps, nr_eps);
1229     }
1230 }
1231 
1232 static TRBCCode xhci_alloc_device_streams(XHCIState *xhci, unsigned int slotid,
1233                                           uint32_t epmask)
1234 {
1235     XHCIEPContext *epctxs[30];
1236     USBEndpoint *eps[30];
1237     int i, r, nr_eps, req_nr_streams, dev_max_streams;
1238 
1239     nr_eps = xhci_epmask_to_eps_with_streams(xhci, slotid, epmask, epctxs,
1240                                              eps);
1241     if (nr_eps == 0) {
1242         return CC_SUCCESS;
1243     }
1244 
1245     req_nr_streams = epctxs[0]->nr_pstreams;
1246     dev_max_streams = eps[0]->max_streams;
1247 
1248     for (i = 1; i < nr_eps; i++) {
1249         /*
1250          * HdG: I don't expect these to ever trigger, but if they do we need
1251          * to come up with another solution, ie group identical endpoints
1252          * together and make an usb_device_alloc_streams call per group.
1253          */
1254         if (epctxs[i]->nr_pstreams != req_nr_streams) {
1255             FIXME("guest streams config not identical for all eps");
1256             return CC_RESOURCE_ERROR;
1257         }
1258         if (eps[i]->max_streams != dev_max_streams) {
1259             FIXME("device streams config not identical for all eps");
1260             return CC_RESOURCE_ERROR;
1261         }
1262     }
1263 
1264     /*
1265      * max-streams in both the device descriptor and in the controller is a
1266      * power of 2. But stream id 0 is reserved, so if a device can do up to 4
1267      * streams the guest will ask for 5 rounded up to the next power of 2 which
1268      * becomes 8. For emulated devices usb_device_alloc_streams is a nop.
1269      *
1270      * For redirected devices however this is an issue, as there we must ask
1271      * the real xhci controller to alloc streams, and the host driver for the
1272      * real xhci controller will likely disallow allocating more streams then
1273      * the device can handle.
1274      *
1275      * So we limit the requested nr_streams to the maximum number the device
1276      * can handle.
1277      */
1278     if (req_nr_streams > dev_max_streams) {
1279         req_nr_streams = dev_max_streams;
1280     }
1281 
1282     r = usb_device_alloc_streams(eps[0]->dev, eps, nr_eps, req_nr_streams);
1283     if (r != 0) {
1284         DPRINTF("xhci: alloc streams failed\n");
1285         return CC_RESOURCE_ERROR;
1286     }
1287 
1288     return CC_SUCCESS;
1289 }
1290 
1291 static XHCIStreamContext *xhci_find_stream(XHCIEPContext *epctx,
1292                                            unsigned int streamid,
1293                                            uint32_t *cc_error)
1294 {
1295     XHCIStreamContext *sctx;
1296     dma_addr_t base;
1297     uint32_t ctx[2], sct;
1298 
1299     assert(streamid != 0);
1300     if (epctx->lsa) {
1301         if (streamid >= epctx->nr_pstreams) {
1302             *cc_error = CC_INVALID_STREAM_ID_ERROR;
1303             return NULL;
1304         }
1305         sctx = epctx->pstreams + streamid;
1306     } else {
1307         FIXME("secondary streams not implemented yet");
1308     }
1309 
1310     if (sctx->sct == -1) {
1311         xhci_dma_read_u32s(epctx->xhci, sctx->pctx, ctx, sizeof(ctx));
1312         sct = (ctx[0] >> 1) & 0x07;
1313         if (epctx->lsa && sct != 1) {
1314             *cc_error = CC_INVALID_STREAM_TYPE_ERROR;
1315             return NULL;
1316         }
1317         sctx->sct = sct;
1318         base = xhci_addr64(ctx[0] & ~0xf, ctx[1]);
1319         xhci_ring_init(epctx->xhci, &sctx->ring, base);
1320     }
1321     return sctx;
1322 }
1323 
1324 static void xhci_set_ep_state(XHCIState *xhci, XHCIEPContext *epctx,
1325                               XHCIStreamContext *sctx, uint32_t state)
1326 {
1327     XHCIRing *ring = NULL;
1328     uint32_t ctx[5];
1329     uint32_t ctx2[2];
1330 
1331     xhci_dma_read_u32s(xhci, epctx->pctx, ctx, sizeof(ctx));
1332     ctx[0] &= ~EP_STATE_MASK;
1333     ctx[0] |= state;
1334 
1335     /* update ring dequeue ptr */
1336     if (epctx->nr_pstreams) {
1337         if (sctx != NULL) {
1338             ring = &sctx->ring;
1339             xhci_dma_read_u32s(xhci, sctx->pctx, ctx2, sizeof(ctx2));
1340             ctx2[0] &= 0xe;
1341             ctx2[0] |= sctx->ring.dequeue | sctx->ring.ccs;
1342             ctx2[1] = (sctx->ring.dequeue >> 16) >> 16;
1343             xhci_dma_write_u32s(xhci, sctx->pctx, ctx2, sizeof(ctx2));
1344         }
1345     } else {
1346         ring = &epctx->ring;
1347     }
1348     if (ring) {
1349         ctx[2] = ring->dequeue | ring->ccs;
1350         ctx[3] = (ring->dequeue >> 16) >> 16;
1351 
1352         DPRINTF("xhci: set epctx: " DMA_ADDR_FMT " state=%d dequeue=%08x%08x\n",
1353                 epctx->pctx, state, ctx[3], ctx[2]);
1354     }
1355 
1356     xhci_dma_write_u32s(xhci, epctx->pctx, ctx, sizeof(ctx));
1357     if (epctx->state != state) {
1358         trace_usb_xhci_ep_state(epctx->slotid, epctx->epid,
1359                                 ep_state_name(epctx->state),
1360                                 ep_state_name(state));
1361     }
1362     epctx->state = state;
1363 }
1364 
1365 static void xhci_ep_kick_timer(void *opaque)
1366 {
1367     XHCIEPContext *epctx = opaque;
1368     xhci_kick_epctx(epctx, 0);
1369 }
1370 
1371 static XHCIEPContext *xhci_alloc_epctx(XHCIState *xhci,
1372                                        unsigned int slotid,
1373                                        unsigned int epid)
1374 {
1375     XHCIEPContext *epctx;
1376 
1377     epctx = g_new0(XHCIEPContext, 1);
1378     epctx->xhci = xhci;
1379     epctx->slotid = slotid;
1380     epctx->epid = epid;
1381 
1382     QTAILQ_INIT(&epctx->transfers);
1383     epctx->kick_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, xhci_ep_kick_timer, epctx);
1384 
1385     return epctx;
1386 }
1387 
1388 static void xhci_init_epctx(XHCIEPContext *epctx,
1389                             dma_addr_t pctx, uint32_t *ctx)
1390 {
1391     dma_addr_t dequeue;
1392 
1393     dequeue = xhci_addr64(ctx[2] & ~0xf, ctx[3]);
1394 
1395     epctx->type = (ctx[1] >> EP_TYPE_SHIFT) & EP_TYPE_MASK;
1396     epctx->pctx = pctx;
1397     epctx->max_psize = ctx[1]>>16;
1398     epctx->max_psize *= 1+((ctx[1]>>8)&0xff);
1399     epctx->max_pstreams = (ctx[0] >> 10) & epctx->xhci->max_pstreams_mask;
1400     epctx->lsa = (ctx[0] >> 15) & 1;
1401     if (epctx->max_pstreams) {
1402         xhci_alloc_streams(epctx, dequeue);
1403     } else {
1404         xhci_ring_init(epctx->xhci, &epctx->ring, dequeue);
1405         epctx->ring.ccs = ctx[2] & 1;
1406     }
1407 
1408     epctx->interval = 1 << ((ctx[0] >> 16) & 0xff);
1409 }
1410 
1411 static TRBCCode xhci_enable_ep(XHCIState *xhci, unsigned int slotid,
1412                                unsigned int epid, dma_addr_t pctx,
1413                                uint32_t *ctx)
1414 {
1415     XHCISlot *slot;
1416     XHCIEPContext *epctx;
1417 
1418     trace_usb_xhci_ep_enable(slotid, epid);
1419     assert(slotid >= 1 && slotid <= xhci->numslots);
1420     assert(epid >= 1 && epid <= 31);
1421 
1422     slot = &xhci->slots[slotid-1];
1423     if (slot->eps[epid-1]) {
1424         xhci_disable_ep(xhci, slotid, epid);
1425     }
1426 
1427     epctx = xhci_alloc_epctx(xhci, slotid, epid);
1428     slot->eps[epid-1] = epctx;
1429     xhci_init_epctx(epctx, pctx, ctx);
1430 
1431     DPRINTF("xhci: endpoint %d.%d type is %d, max transaction (burst) "
1432             "size is %d\n", epid/2, epid%2, epctx->type, epctx->max_psize);
1433 
1434     epctx->mfindex_last = 0;
1435 
1436     epctx->state = EP_RUNNING;
1437     ctx[0] &= ~EP_STATE_MASK;
1438     ctx[0] |= EP_RUNNING;
1439 
1440     return CC_SUCCESS;
1441 }
1442 
1443 static XHCITransfer *xhci_ep_alloc_xfer(XHCIEPContext *epctx,
1444                                         uint32_t length)
1445 {
1446     uint32_t limit = epctx->nr_pstreams + 16;
1447     XHCITransfer *xfer;
1448 
1449     if (epctx->xfer_count >= limit) {
1450         return NULL;
1451     }
1452 
1453     xfer = g_new0(XHCITransfer, 1);
1454     xfer->epctx = epctx;
1455     xfer->trbs = g_new(XHCITRB, length);
1456     xfer->trb_count = length;
1457     usb_packet_init(&xfer->packet);
1458 
1459     QTAILQ_INSERT_TAIL(&epctx->transfers, xfer, next);
1460     epctx->xfer_count++;
1461 
1462     return xfer;
1463 }
1464 
1465 static void xhci_ep_free_xfer(XHCITransfer *xfer)
1466 {
1467     QTAILQ_REMOVE(&xfer->epctx->transfers, xfer, next);
1468     xfer->epctx->xfer_count--;
1469 
1470     usb_packet_cleanup(&xfer->packet);
1471     g_free(xfer->trbs);
1472     g_free(xfer);
1473 }
1474 
1475 static int xhci_ep_nuke_one_xfer(XHCITransfer *t, TRBCCode report)
1476 {
1477     int killed = 0;
1478 
1479     if (report && (t->running_async || t->running_retry)) {
1480         t->status = report;
1481         xhci_xfer_report(t);
1482     }
1483 
1484     if (t->running_async) {
1485         usb_cancel_packet(&t->packet);
1486         t->running_async = 0;
1487         killed = 1;
1488     }
1489     if (t->running_retry) {
1490         if (t->epctx) {
1491             t->epctx->retry = NULL;
1492             timer_del(t->epctx->kick_timer);
1493         }
1494         t->running_retry = 0;
1495         killed = 1;
1496     }
1497     g_free(t->trbs);
1498 
1499     t->trbs = NULL;
1500     t->trb_count = 0;
1501 
1502     return killed;
1503 }
1504 
1505 static int xhci_ep_nuke_xfers(XHCIState *xhci, unsigned int slotid,
1506                                unsigned int epid, TRBCCode report)
1507 {
1508     XHCISlot *slot;
1509     XHCIEPContext *epctx;
1510     XHCITransfer *xfer;
1511     int killed = 0;
1512     USBEndpoint *ep = NULL;
1513     assert(slotid >= 1 && slotid <= xhci->numslots);
1514     assert(epid >= 1 && epid <= 31);
1515 
1516     DPRINTF("xhci_ep_nuke_xfers(%d, %d)\n", slotid, epid);
1517 
1518     slot = &xhci->slots[slotid-1];
1519 
1520     if (!slot->eps[epid-1]) {
1521         return 0;
1522     }
1523 
1524     epctx = slot->eps[epid-1];
1525 
1526     for (;;) {
1527         xfer = QTAILQ_FIRST(&epctx->transfers);
1528         if (xfer == NULL) {
1529             break;
1530         }
1531         killed += xhci_ep_nuke_one_xfer(xfer, report);
1532         if (killed) {
1533             report = 0; /* Only report once */
1534         }
1535         xhci_ep_free_xfer(xfer);
1536     }
1537 
1538     ep = xhci_epid_to_usbep(epctx);
1539     if (ep) {
1540         usb_device_ep_stopped(ep->dev, ep);
1541     }
1542     return killed;
1543 }
1544 
1545 static TRBCCode xhci_disable_ep(XHCIState *xhci, unsigned int slotid,
1546                                unsigned int epid)
1547 {
1548     XHCISlot *slot;
1549     XHCIEPContext *epctx;
1550 
1551     trace_usb_xhci_ep_disable(slotid, epid);
1552     assert(slotid >= 1 && slotid <= xhci->numslots);
1553     assert(epid >= 1 && epid <= 31);
1554 
1555     slot = &xhci->slots[slotid-1];
1556 
1557     if (!slot->eps[epid-1]) {
1558         DPRINTF("xhci: slot %d ep %d already disabled\n", slotid, epid);
1559         return CC_SUCCESS;
1560     }
1561 
1562     xhci_ep_nuke_xfers(xhci, slotid, epid, 0);
1563 
1564     epctx = slot->eps[epid-1];
1565 
1566     if (epctx->nr_pstreams) {
1567         xhci_free_streams(epctx);
1568     }
1569 
1570     /* only touch guest RAM if we're not resetting the HC */
1571     if (xhci->dcbaap_low || xhci->dcbaap_high) {
1572         xhci_set_ep_state(xhci, epctx, NULL, EP_DISABLED);
1573     }
1574 
1575     timer_free(epctx->kick_timer);
1576     g_free(epctx);
1577     slot->eps[epid-1] = NULL;
1578 
1579     return CC_SUCCESS;
1580 }
1581 
1582 static TRBCCode xhci_stop_ep(XHCIState *xhci, unsigned int slotid,
1583                              unsigned int epid)
1584 {
1585     XHCISlot *slot;
1586     XHCIEPContext *epctx;
1587 
1588     trace_usb_xhci_ep_stop(slotid, epid);
1589     assert(slotid >= 1 && slotid <= xhci->numslots);
1590 
1591     if (epid < 1 || epid > 31) {
1592         DPRINTF("xhci: bad ep %d\n", epid);
1593         return CC_TRB_ERROR;
1594     }
1595 
1596     slot = &xhci->slots[slotid-1];
1597 
1598     if (!slot->eps[epid-1]) {
1599         DPRINTF("xhci: slot %d ep %d not enabled\n", slotid, epid);
1600         return CC_EP_NOT_ENABLED_ERROR;
1601     }
1602 
1603     if (xhci_ep_nuke_xfers(xhci, slotid, epid, CC_STOPPED) > 0) {
1604         DPRINTF("xhci: FIXME: endpoint stopped w/ xfers running, "
1605                 "data might be lost\n");
1606     }
1607 
1608     epctx = slot->eps[epid-1];
1609 
1610     xhci_set_ep_state(xhci, epctx, NULL, EP_STOPPED);
1611 
1612     if (epctx->nr_pstreams) {
1613         xhci_reset_streams(epctx);
1614     }
1615 
1616     return CC_SUCCESS;
1617 }
1618 
1619 static TRBCCode xhci_reset_ep(XHCIState *xhci, unsigned int slotid,
1620                               unsigned int epid)
1621 {
1622     XHCISlot *slot;
1623     XHCIEPContext *epctx;
1624 
1625     trace_usb_xhci_ep_reset(slotid, epid);
1626     assert(slotid >= 1 && slotid <= xhci->numslots);
1627 
1628     if (epid < 1 || epid > 31) {
1629         DPRINTF("xhci: bad ep %d\n", epid);
1630         return CC_TRB_ERROR;
1631     }
1632 
1633     slot = &xhci->slots[slotid-1];
1634 
1635     if (!slot->eps[epid-1]) {
1636         DPRINTF("xhci: slot %d ep %d not enabled\n", slotid, epid);
1637         return CC_EP_NOT_ENABLED_ERROR;
1638     }
1639 
1640     epctx = slot->eps[epid-1];
1641 
1642     if (epctx->state != EP_HALTED) {
1643         DPRINTF("xhci: reset EP while EP %d not halted (%d)\n",
1644                 epid, epctx->state);
1645         return CC_CONTEXT_STATE_ERROR;
1646     }
1647 
1648     if (xhci_ep_nuke_xfers(xhci, slotid, epid, 0) > 0) {
1649         DPRINTF("xhci: FIXME: endpoint reset w/ xfers running, "
1650                 "data might be lost\n");
1651     }
1652 
1653     if (!xhci->slots[slotid-1].uport ||
1654         !xhci->slots[slotid-1].uport->dev ||
1655         !xhci->slots[slotid-1].uport->dev->attached) {
1656         return CC_USB_TRANSACTION_ERROR;
1657     }
1658 
1659     xhci_set_ep_state(xhci, epctx, NULL, EP_STOPPED);
1660 
1661     if (epctx->nr_pstreams) {
1662         xhci_reset_streams(epctx);
1663     }
1664 
1665     return CC_SUCCESS;
1666 }
1667 
1668 static TRBCCode xhci_set_ep_dequeue(XHCIState *xhci, unsigned int slotid,
1669                                     unsigned int epid, unsigned int streamid,
1670                                     uint64_t pdequeue)
1671 {
1672     XHCISlot *slot;
1673     XHCIEPContext *epctx;
1674     XHCIStreamContext *sctx;
1675     dma_addr_t dequeue;
1676 
1677     assert(slotid >= 1 && slotid <= xhci->numslots);
1678 
1679     if (epid < 1 || epid > 31) {
1680         DPRINTF("xhci: bad ep %d\n", epid);
1681         return CC_TRB_ERROR;
1682     }
1683 
1684     trace_usb_xhci_ep_set_dequeue(slotid, epid, streamid, pdequeue);
1685     dequeue = xhci_mask64(pdequeue);
1686 
1687     slot = &xhci->slots[slotid-1];
1688 
1689     if (!slot->eps[epid-1]) {
1690         DPRINTF("xhci: slot %d ep %d not enabled\n", slotid, epid);
1691         return CC_EP_NOT_ENABLED_ERROR;
1692     }
1693 
1694     epctx = slot->eps[epid-1];
1695 
1696     if (epctx->state != EP_STOPPED) {
1697         DPRINTF("xhci: set EP dequeue pointer while EP %d not stopped\n", epid);
1698         return CC_CONTEXT_STATE_ERROR;
1699     }
1700 
1701     if (epctx->nr_pstreams) {
1702         uint32_t err;
1703         sctx = xhci_find_stream(epctx, streamid, &err);
1704         if (sctx == NULL) {
1705             return err;
1706         }
1707         xhci_ring_init(xhci, &sctx->ring, dequeue & ~0xf);
1708         sctx->ring.ccs = dequeue & 1;
1709     } else {
1710         sctx = NULL;
1711         xhci_ring_init(xhci, &epctx->ring, dequeue & ~0xF);
1712         epctx->ring.ccs = dequeue & 1;
1713     }
1714 
1715     xhci_set_ep_state(xhci, epctx, sctx, EP_STOPPED);
1716 
1717     return CC_SUCCESS;
1718 }
1719 
1720 static int xhci_xfer_create_sgl(XHCITransfer *xfer, int in_xfer)
1721 {
1722     XHCIState *xhci = xfer->epctx->xhci;
1723     int i;
1724 
1725     xfer->int_req = false;
1726     pci_dma_sglist_init(&xfer->sgl, PCI_DEVICE(xhci), xfer->trb_count);
1727     for (i = 0; i < xfer->trb_count; i++) {
1728         XHCITRB *trb = &xfer->trbs[i];
1729         dma_addr_t addr;
1730         unsigned int chunk = 0;
1731 
1732         if (trb->control & TRB_TR_IOC) {
1733             xfer->int_req = true;
1734         }
1735 
1736         switch (TRB_TYPE(*trb)) {
1737         case TR_DATA:
1738             if ((!(trb->control & TRB_TR_DIR)) != (!in_xfer)) {
1739                 DPRINTF("xhci: data direction mismatch for TR_DATA\n");
1740                 goto err;
1741             }
1742             /* fallthrough */
1743         case TR_NORMAL:
1744         case TR_ISOCH:
1745             addr = xhci_mask64(trb->parameter);
1746             chunk = trb->status & 0x1ffff;
1747             if (trb->control & TRB_TR_IDT) {
1748                 if (chunk > 8 || in_xfer) {
1749                     DPRINTF("xhci: invalid immediate data TRB\n");
1750                     goto err;
1751                 }
1752                 qemu_sglist_add(&xfer->sgl, trb->addr, chunk);
1753             } else {
1754                 qemu_sglist_add(&xfer->sgl, addr, chunk);
1755             }
1756             break;
1757         }
1758     }
1759 
1760     return 0;
1761 
1762 err:
1763     qemu_sglist_destroy(&xfer->sgl);
1764     xhci_die(xhci);
1765     return -1;
1766 }
1767 
1768 static void xhci_xfer_unmap(XHCITransfer *xfer)
1769 {
1770     usb_packet_unmap(&xfer->packet, &xfer->sgl);
1771     qemu_sglist_destroy(&xfer->sgl);
1772 }
1773 
1774 static void xhci_xfer_report(XHCITransfer *xfer)
1775 {
1776     uint32_t edtla = 0;
1777     unsigned int left;
1778     bool reported = 0;
1779     bool shortpkt = 0;
1780     XHCIEvent event = {ER_TRANSFER, CC_SUCCESS};
1781     XHCIState *xhci = xfer->epctx->xhci;
1782     int i;
1783 
1784     left = xfer->packet.actual_length;
1785 
1786     for (i = 0; i < xfer->trb_count; i++) {
1787         XHCITRB *trb = &xfer->trbs[i];
1788         unsigned int chunk = 0;
1789 
1790         switch (TRB_TYPE(*trb)) {
1791         case TR_SETUP:
1792             chunk = trb->status & 0x1ffff;
1793             if (chunk > 8) {
1794                 chunk = 8;
1795             }
1796             break;
1797         case TR_DATA:
1798         case TR_NORMAL:
1799         case TR_ISOCH:
1800             chunk = trb->status & 0x1ffff;
1801             if (chunk > left) {
1802                 chunk = left;
1803                 if (xfer->status == CC_SUCCESS) {
1804                     shortpkt = 1;
1805                 }
1806             }
1807             left -= chunk;
1808             edtla += chunk;
1809             break;
1810         case TR_STATUS:
1811             reported = 0;
1812             shortpkt = 0;
1813             break;
1814         }
1815 
1816         if (!reported && ((trb->control & TRB_TR_IOC) ||
1817                           (shortpkt && (trb->control & TRB_TR_ISP)) ||
1818                           (xfer->status != CC_SUCCESS && left == 0))) {
1819             event.slotid = xfer->epctx->slotid;
1820             event.epid = xfer->epctx->epid;
1821             event.length = (trb->status & 0x1ffff) - chunk;
1822             event.flags = 0;
1823             event.ptr = trb->addr;
1824             if (xfer->status == CC_SUCCESS) {
1825                 event.ccode = shortpkt ? CC_SHORT_PACKET : CC_SUCCESS;
1826             } else {
1827                 event.ccode = xfer->status;
1828             }
1829             if (TRB_TYPE(*trb) == TR_EVDATA) {
1830                 event.ptr = trb->parameter;
1831                 event.flags |= TRB_EV_ED;
1832                 event.length = edtla & 0xffffff;
1833                 DPRINTF("xhci_xfer_data: EDTLA=%d\n", event.length);
1834                 edtla = 0;
1835             }
1836             xhci_event(xhci, &event, TRB_INTR(*trb));
1837             reported = 1;
1838             if (xfer->status != CC_SUCCESS) {
1839                 return;
1840             }
1841         }
1842 
1843         switch (TRB_TYPE(*trb)) {
1844         case TR_SETUP:
1845             reported = 0;
1846             shortpkt = 0;
1847             break;
1848         }
1849 
1850     }
1851 }
1852 
1853 static void xhci_stall_ep(XHCITransfer *xfer)
1854 {
1855     XHCIEPContext *epctx = xfer->epctx;
1856     XHCIState *xhci = epctx->xhci;
1857     uint32_t err;
1858     XHCIStreamContext *sctx;
1859 
1860     if (epctx->nr_pstreams) {
1861         sctx = xhci_find_stream(epctx, xfer->streamid, &err);
1862         if (sctx == NULL) {
1863             return;
1864         }
1865         sctx->ring.dequeue = xfer->trbs[0].addr;
1866         sctx->ring.ccs = xfer->trbs[0].ccs;
1867         xhci_set_ep_state(xhci, epctx, sctx, EP_HALTED);
1868     } else {
1869         epctx->ring.dequeue = xfer->trbs[0].addr;
1870         epctx->ring.ccs = xfer->trbs[0].ccs;
1871         xhci_set_ep_state(xhci, epctx, NULL, EP_HALTED);
1872     }
1873 }
1874 
1875 static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer,
1876                        XHCIEPContext *epctx);
1877 
1878 static int xhci_setup_packet(XHCITransfer *xfer)
1879 {
1880     USBEndpoint *ep;
1881     int dir;
1882 
1883     dir = xfer->in_xfer ? USB_TOKEN_IN : USB_TOKEN_OUT;
1884 
1885     if (xfer->packet.ep) {
1886         ep = xfer->packet.ep;
1887     } else {
1888         ep = xhci_epid_to_usbep(xfer->epctx);
1889         if (!ep) {
1890             DPRINTF("xhci: slot %d has no device\n",
1891                     xfer->slotid);
1892             return -1;
1893         }
1894     }
1895 
1896     xhci_xfer_create_sgl(xfer, dir == USB_TOKEN_IN); /* Also sets int_req */
1897     usb_packet_setup(&xfer->packet, dir, ep, xfer->streamid,
1898                      xfer->trbs[0].addr, false, xfer->int_req);
1899     usb_packet_map(&xfer->packet, &xfer->sgl);
1900     DPRINTF("xhci: setup packet pid 0x%x addr %d ep %d\n",
1901             xfer->packet.pid, ep->dev->addr, ep->nr);
1902     return 0;
1903 }
1904 
1905 static int xhci_try_complete_packet(XHCITransfer *xfer)
1906 {
1907     if (xfer->packet.status == USB_RET_ASYNC) {
1908         trace_usb_xhci_xfer_async(xfer);
1909         xfer->running_async = 1;
1910         xfer->running_retry = 0;
1911         xfer->complete = 0;
1912         return 0;
1913     } else if (xfer->packet.status == USB_RET_NAK) {
1914         trace_usb_xhci_xfer_nak(xfer);
1915         xfer->running_async = 0;
1916         xfer->running_retry = 1;
1917         xfer->complete = 0;
1918         return 0;
1919     } else {
1920         xfer->running_async = 0;
1921         xfer->running_retry = 0;
1922         xfer->complete = 1;
1923         xhci_xfer_unmap(xfer);
1924     }
1925 
1926     if (xfer->packet.status == USB_RET_SUCCESS) {
1927         trace_usb_xhci_xfer_success(xfer, xfer->packet.actual_length);
1928         xfer->status = CC_SUCCESS;
1929         xhci_xfer_report(xfer);
1930         return 0;
1931     }
1932 
1933     /* error */
1934     trace_usb_xhci_xfer_error(xfer, xfer->packet.status);
1935     switch (xfer->packet.status) {
1936     case USB_RET_NODEV:
1937     case USB_RET_IOERROR:
1938         xfer->status = CC_USB_TRANSACTION_ERROR;
1939         xhci_xfer_report(xfer);
1940         xhci_stall_ep(xfer);
1941         break;
1942     case USB_RET_STALL:
1943         xfer->status = CC_STALL_ERROR;
1944         xhci_xfer_report(xfer);
1945         xhci_stall_ep(xfer);
1946         break;
1947     case USB_RET_BABBLE:
1948         xfer->status = CC_BABBLE_DETECTED;
1949         xhci_xfer_report(xfer);
1950         xhci_stall_ep(xfer);
1951         break;
1952     default:
1953         DPRINTF("%s: FIXME: status = %d\n", __func__,
1954                 xfer->packet.status);
1955         FIXME("unhandled USB_RET_*");
1956     }
1957     return 0;
1958 }
1959 
1960 static int xhci_fire_ctl_transfer(XHCIState *xhci, XHCITransfer *xfer)
1961 {
1962     XHCITRB *trb_setup, *trb_status;
1963     uint8_t bmRequestType;
1964 
1965     trb_setup = &xfer->trbs[0];
1966     trb_status = &xfer->trbs[xfer->trb_count-1];
1967 
1968     trace_usb_xhci_xfer_start(xfer, xfer->epctx->slotid,
1969                               xfer->epctx->epid, xfer->streamid);
1970 
1971     /* at most one Event Data TRB allowed after STATUS */
1972     if (TRB_TYPE(*trb_status) == TR_EVDATA && xfer->trb_count > 2) {
1973         trb_status--;
1974     }
1975 
1976     /* do some sanity checks */
1977     if (TRB_TYPE(*trb_setup) != TR_SETUP) {
1978         DPRINTF("xhci: ep0 first TD not SETUP: %d\n",
1979                 TRB_TYPE(*trb_setup));
1980         return -1;
1981     }
1982     if (TRB_TYPE(*trb_status) != TR_STATUS) {
1983         DPRINTF("xhci: ep0 last TD not STATUS: %d\n",
1984                 TRB_TYPE(*trb_status));
1985         return -1;
1986     }
1987     if (!(trb_setup->control & TRB_TR_IDT)) {
1988         DPRINTF("xhci: Setup TRB doesn't have IDT set\n");
1989         return -1;
1990     }
1991     if ((trb_setup->status & 0x1ffff) != 8) {
1992         DPRINTF("xhci: Setup TRB has bad length (%d)\n",
1993                 (trb_setup->status & 0x1ffff));
1994         return -1;
1995     }
1996 
1997     bmRequestType = trb_setup->parameter;
1998 
1999     xfer->in_xfer = bmRequestType & USB_DIR_IN;
2000     xfer->iso_xfer = false;
2001     xfer->timed_xfer = false;
2002 
2003     if (xhci_setup_packet(xfer) < 0) {
2004         return -1;
2005     }
2006     xfer->packet.parameter = trb_setup->parameter;
2007 
2008     usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
2009     xhci_try_complete_packet(xfer);
2010     return 0;
2011 }
2012 
2013 static void xhci_calc_intr_kick(XHCIState *xhci, XHCITransfer *xfer,
2014                                 XHCIEPContext *epctx, uint64_t mfindex)
2015 {
2016     uint64_t asap = ((mfindex + epctx->interval - 1) &
2017                      ~(epctx->interval-1));
2018     uint64_t kick = epctx->mfindex_last + epctx->interval;
2019 
2020     assert(epctx->interval != 0);
2021     xfer->mfindex_kick = MAX(asap, kick);
2022 }
2023 
2024 static void xhci_calc_iso_kick(XHCIState *xhci, XHCITransfer *xfer,
2025                                XHCIEPContext *epctx, uint64_t mfindex)
2026 {
2027     if (xfer->trbs[0].control & TRB_TR_SIA) {
2028         uint64_t asap = ((mfindex + epctx->interval - 1) &
2029                          ~(epctx->interval-1));
2030         if (asap >= epctx->mfindex_last &&
2031             asap <= epctx->mfindex_last + epctx->interval * 4) {
2032             xfer->mfindex_kick = epctx->mfindex_last + epctx->interval;
2033         } else {
2034             xfer->mfindex_kick = asap;
2035         }
2036     } else {
2037         xfer->mfindex_kick = ((xfer->trbs[0].control >> TRB_TR_FRAMEID_SHIFT)
2038                               & TRB_TR_FRAMEID_MASK) << 3;
2039         xfer->mfindex_kick |= mfindex & ~0x3fff;
2040         if (xfer->mfindex_kick + 0x100 < mfindex) {
2041             xfer->mfindex_kick += 0x4000;
2042         }
2043     }
2044 }
2045 
2046 static void xhci_check_intr_iso_kick(XHCIState *xhci, XHCITransfer *xfer,
2047                                      XHCIEPContext *epctx, uint64_t mfindex)
2048 {
2049     if (xfer->mfindex_kick > mfindex) {
2050         timer_mod(epctx->kick_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL) +
2051                        (xfer->mfindex_kick - mfindex) * 125000);
2052         xfer->running_retry = 1;
2053     } else {
2054         epctx->mfindex_last = xfer->mfindex_kick;
2055         timer_del(epctx->kick_timer);
2056         xfer->running_retry = 0;
2057     }
2058 }
2059 
2060 
2061 static int xhci_submit(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx)
2062 {
2063     uint64_t mfindex;
2064 
2065     DPRINTF("xhci_submit(slotid=%d,epid=%d)\n", xfer->slotid, xfer->epid);
2066 
2067     xfer->in_xfer = epctx->type>>2;
2068 
2069     switch(epctx->type) {
2070     case ET_INTR_OUT:
2071     case ET_INTR_IN:
2072         xfer->pkts = 0;
2073         xfer->iso_xfer = false;
2074         xfer->timed_xfer = true;
2075         mfindex = xhci_mfindex_get(xhci);
2076         xhci_calc_intr_kick(xhci, xfer, epctx, mfindex);
2077         xhci_check_intr_iso_kick(xhci, xfer, epctx, mfindex);
2078         if (xfer->running_retry) {
2079             return -1;
2080         }
2081         break;
2082     case ET_BULK_OUT:
2083     case ET_BULK_IN:
2084         xfer->pkts = 0;
2085         xfer->iso_xfer = false;
2086         xfer->timed_xfer = false;
2087         break;
2088     case ET_ISO_OUT:
2089     case ET_ISO_IN:
2090         xfer->pkts = 1;
2091         xfer->iso_xfer = true;
2092         xfer->timed_xfer = true;
2093         mfindex = xhci_mfindex_get(xhci);
2094         xhci_calc_iso_kick(xhci, xfer, epctx, mfindex);
2095         xhci_check_intr_iso_kick(xhci, xfer, epctx, mfindex);
2096         if (xfer->running_retry) {
2097             return -1;
2098         }
2099         break;
2100     default:
2101         trace_usb_xhci_unimplemented("endpoint type", epctx->type);
2102         return -1;
2103     }
2104 
2105     if (xhci_setup_packet(xfer) < 0) {
2106         return -1;
2107     }
2108     usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
2109     xhci_try_complete_packet(xfer);
2110     return 0;
2111 }
2112 
2113 static int xhci_fire_transfer(XHCIState *xhci, XHCITransfer *xfer, XHCIEPContext *epctx)
2114 {
2115     trace_usb_xhci_xfer_start(xfer, xfer->epctx->slotid,
2116                               xfer->epctx->epid, xfer->streamid);
2117     return xhci_submit(xhci, xfer, epctx);
2118 }
2119 
2120 static void xhci_kick_ep(XHCIState *xhci, unsigned int slotid,
2121                          unsigned int epid, unsigned int streamid)
2122 {
2123     XHCIEPContext *epctx;
2124 
2125     assert(slotid >= 1 && slotid <= xhci->numslots);
2126     assert(epid >= 1 && epid <= 31);
2127 
2128     if (!xhci->slots[slotid-1].enabled) {
2129         DPRINTF("xhci: xhci_kick_ep for disabled slot %d\n", slotid);
2130         return;
2131     }
2132     epctx = xhci->slots[slotid-1].eps[epid-1];
2133     if (!epctx) {
2134         DPRINTF("xhci: xhci_kick_ep for disabled endpoint %d,%d\n",
2135                 epid, slotid);
2136         return;
2137     }
2138 
2139     if (epctx->kick_active) {
2140         return;
2141     }
2142     xhci_kick_epctx(epctx, streamid);
2143 }
2144 
2145 static void xhci_kick_epctx(XHCIEPContext *epctx, unsigned int streamid)
2146 {
2147     XHCIState *xhci = epctx->xhci;
2148     XHCIStreamContext *stctx;
2149     XHCITransfer *xfer;
2150     XHCIRing *ring;
2151     USBEndpoint *ep = NULL;
2152     uint64_t mfindex;
2153     int length;
2154     int i;
2155 
2156     trace_usb_xhci_ep_kick(epctx->slotid, epctx->epid, streamid);
2157     assert(!epctx->kick_active);
2158 
2159     /* If the device has been detached, but the guest has not noticed this
2160        yet the 2 above checks will succeed, but we must NOT continue */
2161     if (!xhci->slots[epctx->slotid - 1].uport ||
2162         !xhci->slots[epctx->slotid - 1].uport->dev ||
2163         !xhci->slots[epctx->slotid - 1].uport->dev->attached) {
2164         return;
2165     }
2166 
2167     if (epctx->retry) {
2168         XHCITransfer *xfer = epctx->retry;
2169 
2170         trace_usb_xhci_xfer_retry(xfer);
2171         assert(xfer->running_retry);
2172         if (xfer->timed_xfer) {
2173             /* time to kick the transfer? */
2174             mfindex = xhci_mfindex_get(xhci);
2175             xhci_check_intr_iso_kick(xhci, xfer, epctx, mfindex);
2176             if (xfer->running_retry) {
2177                 return;
2178             }
2179             xfer->timed_xfer = 0;
2180             xfer->running_retry = 1;
2181         }
2182         if (xfer->iso_xfer) {
2183             /* retry iso transfer */
2184             if (xhci_setup_packet(xfer) < 0) {
2185                 return;
2186             }
2187             usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
2188             assert(xfer->packet.status != USB_RET_NAK);
2189             xhci_try_complete_packet(xfer);
2190         } else {
2191             /* retry nak'ed transfer */
2192             if (xhci_setup_packet(xfer) < 0) {
2193                 return;
2194             }
2195             usb_handle_packet(xfer->packet.ep->dev, &xfer->packet);
2196             if (xfer->packet.status == USB_RET_NAK) {
2197                 return;
2198             }
2199             xhci_try_complete_packet(xfer);
2200         }
2201         assert(!xfer->running_retry);
2202         if (xfer->complete) {
2203             xhci_ep_free_xfer(epctx->retry);
2204         }
2205         epctx->retry = NULL;
2206     }
2207 
2208     if (epctx->state == EP_HALTED) {
2209         DPRINTF("xhci: ep halted, not running schedule\n");
2210         return;
2211     }
2212 
2213 
2214     if (epctx->nr_pstreams) {
2215         uint32_t err;
2216         stctx = xhci_find_stream(epctx, streamid, &err);
2217         if (stctx == NULL) {
2218             return;
2219         }
2220         ring = &stctx->ring;
2221         xhci_set_ep_state(xhci, epctx, stctx, EP_RUNNING);
2222     } else {
2223         ring = &epctx->ring;
2224         streamid = 0;
2225         xhci_set_ep_state(xhci, epctx, NULL, EP_RUNNING);
2226     }
2227     assert(ring->dequeue != 0);
2228 
2229     epctx->kick_active++;
2230     while (1) {
2231         length = xhci_ring_chain_length(xhci, ring);
2232         if (length <= 0) {
2233             break;
2234         }
2235         xfer = xhci_ep_alloc_xfer(epctx, length);
2236         if (xfer == NULL) {
2237             break;
2238         }
2239 
2240         for (i = 0; i < length; i++) {
2241             TRBType type;
2242             type = xhci_ring_fetch(xhci, ring, &xfer->trbs[i], NULL);
2243             assert(type);
2244         }
2245         xfer->streamid = streamid;
2246 
2247         if (epctx->epid == 1) {
2248             xhci_fire_ctl_transfer(xhci, xfer);
2249         } else {
2250             xhci_fire_transfer(xhci, xfer, epctx);
2251         }
2252         if (xfer->complete) {
2253             xhci_ep_free_xfer(xfer);
2254             xfer = NULL;
2255         }
2256 
2257         if (epctx->state == EP_HALTED) {
2258             break;
2259         }
2260         if (xfer != NULL && xfer->running_retry) {
2261             DPRINTF("xhci: xfer nacked, stopping schedule\n");
2262             epctx->retry = xfer;
2263             break;
2264         }
2265     }
2266     epctx->kick_active--;
2267 
2268     ep = xhci_epid_to_usbep(epctx);
2269     if (ep) {
2270         usb_device_flush_ep_queue(ep->dev, ep);
2271     }
2272 }
2273 
2274 static TRBCCode xhci_enable_slot(XHCIState *xhci, unsigned int slotid)
2275 {
2276     trace_usb_xhci_slot_enable(slotid);
2277     assert(slotid >= 1 && slotid <= xhci->numslots);
2278     xhci->slots[slotid-1].enabled = 1;
2279     xhci->slots[slotid-1].uport = NULL;
2280     memset(xhci->slots[slotid-1].eps, 0, sizeof(XHCIEPContext*)*31);
2281 
2282     return CC_SUCCESS;
2283 }
2284 
2285 static TRBCCode xhci_disable_slot(XHCIState *xhci, unsigned int slotid)
2286 {
2287     int i;
2288 
2289     trace_usb_xhci_slot_disable(slotid);
2290     assert(slotid >= 1 && slotid <= xhci->numslots);
2291 
2292     for (i = 1; i <= 31; i++) {
2293         if (xhci->slots[slotid-1].eps[i-1]) {
2294             xhci_disable_ep(xhci, slotid, i);
2295         }
2296     }
2297 
2298     xhci->slots[slotid-1].enabled = 0;
2299     xhci->slots[slotid-1].addressed = 0;
2300     xhci->slots[slotid-1].uport = NULL;
2301     return CC_SUCCESS;
2302 }
2303 
2304 static USBPort *xhci_lookup_uport(XHCIState *xhci, uint32_t *slot_ctx)
2305 {
2306     USBPort *uport;
2307     char path[32];
2308     int i, pos, port;
2309 
2310     port = (slot_ctx[1]>>16) & 0xFF;
2311     if (port < 1 || port > xhci->numports) {
2312         return NULL;
2313     }
2314     port = xhci->ports[port-1].uport->index+1;
2315     pos = snprintf(path, sizeof(path), "%d", port);
2316     for (i = 0; i < 5; i++) {
2317         port = (slot_ctx[0] >> 4*i) & 0x0f;
2318         if (!port) {
2319             break;
2320         }
2321         pos += snprintf(path + pos, sizeof(path) - pos, ".%d", port);
2322     }
2323 
2324     QTAILQ_FOREACH(uport, &xhci->bus.used, next) {
2325         if (strcmp(uport->path, path) == 0) {
2326             return uport;
2327         }
2328     }
2329     return NULL;
2330 }
2331 
2332 static TRBCCode xhci_address_slot(XHCIState *xhci, unsigned int slotid,
2333                                   uint64_t pictx, bool bsr)
2334 {
2335     XHCISlot *slot;
2336     USBPort *uport;
2337     USBDevice *dev;
2338     dma_addr_t ictx, octx, dcbaap;
2339     uint64_t poctx;
2340     uint32_t ictl_ctx[2];
2341     uint32_t slot_ctx[4];
2342     uint32_t ep0_ctx[5];
2343     int i;
2344     TRBCCode res;
2345 
2346     assert(slotid >= 1 && slotid <= xhci->numslots);
2347 
2348     dcbaap = xhci_addr64(xhci->dcbaap_low, xhci->dcbaap_high);
2349     poctx = ldq_le_pci_dma(PCI_DEVICE(xhci), dcbaap + 8 * slotid);
2350     ictx = xhci_mask64(pictx);
2351     octx = xhci_mask64(poctx);
2352 
2353     DPRINTF("xhci: input context at "DMA_ADDR_FMT"\n", ictx);
2354     DPRINTF("xhci: output context at "DMA_ADDR_FMT"\n", octx);
2355 
2356     xhci_dma_read_u32s(xhci, ictx, ictl_ctx, sizeof(ictl_ctx));
2357 
2358     if (ictl_ctx[0] != 0x0 || ictl_ctx[1] != 0x3) {
2359         DPRINTF("xhci: invalid input context control %08x %08x\n",
2360                 ictl_ctx[0], ictl_ctx[1]);
2361         return CC_TRB_ERROR;
2362     }
2363 
2364     xhci_dma_read_u32s(xhci, ictx+32, slot_ctx, sizeof(slot_ctx));
2365     xhci_dma_read_u32s(xhci, ictx+64, ep0_ctx, sizeof(ep0_ctx));
2366 
2367     DPRINTF("xhci: input slot context: %08x %08x %08x %08x\n",
2368             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
2369 
2370     DPRINTF("xhci: input ep0 context: %08x %08x %08x %08x %08x\n",
2371             ep0_ctx[0], ep0_ctx[1], ep0_ctx[2], ep0_ctx[3], ep0_ctx[4]);
2372 
2373     uport = xhci_lookup_uport(xhci, slot_ctx);
2374     if (uport == NULL) {
2375         DPRINTF("xhci: port not found\n");
2376         return CC_TRB_ERROR;
2377     }
2378     trace_usb_xhci_slot_address(slotid, uport->path);
2379 
2380     dev = uport->dev;
2381     if (!dev || !dev->attached) {
2382         DPRINTF("xhci: port %s not connected\n", uport->path);
2383         return CC_USB_TRANSACTION_ERROR;
2384     }
2385 
2386     for (i = 0; i < xhci->numslots; i++) {
2387         if (i == slotid-1) {
2388             continue;
2389         }
2390         if (xhci->slots[i].uport == uport) {
2391             DPRINTF("xhci: port %s already assigned to slot %d\n",
2392                     uport->path, i+1);
2393             return CC_TRB_ERROR;
2394         }
2395     }
2396 
2397     slot = &xhci->slots[slotid-1];
2398     slot->uport = uport;
2399     slot->ctx = octx;
2400 
2401     /* Make sure device is in USB_STATE_DEFAULT state */
2402     usb_device_reset(dev);
2403     if (bsr) {
2404         slot_ctx[3] = SLOT_DEFAULT << SLOT_STATE_SHIFT;
2405     } else {
2406         USBPacket p;
2407         uint8_t buf[1];
2408 
2409         slot_ctx[3] = (SLOT_ADDRESSED << SLOT_STATE_SHIFT) | slotid;
2410         memset(&p, 0, sizeof(p));
2411         usb_packet_addbuf(&p, buf, sizeof(buf));
2412         usb_packet_setup(&p, USB_TOKEN_OUT,
2413                          usb_ep_get(dev, USB_TOKEN_OUT, 0), 0,
2414                          0, false, false);
2415         usb_device_handle_control(dev, &p,
2416                                   DeviceOutRequest | USB_REQ_SET_ADDRESS,
2417                                   slotid, 0, 0, NULL);
2418         assert(p.status != USB_RET_ASYNC);
2419     }
2420 
2421     res = xhci_enable_ep(xhci, slotid, 1, octx+32, ep0_ctx);
2422 
2423     DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
2424             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
2425     DPRINTF("xhci: output ep0 context: %08x %08x %08x %08x %08x\n",
2426             ep0_ctx[0], ep0_ctx[1], ep0_ctx[2], ep0_ctx[3], ep0_ctx[4]);
2427 
2428     xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
2429     xhci_dma_write_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
2430 
2431     xhci->slots[slotid-1].addressed = 1;
2432     return res;
2433 }
2434 
2435 
2436 static TRBCCode xhci_configure_slot(XHCIState *xhci, unsigned int slotid,
2437                                   uint64_t pictx, bool dc)
2438 {
2439     dma_addr_t ictx, octx;
2440     uint32_t ictl_ctx[2];
2441     uint32_t slot_ctx[4];
2442     uint32_t islot_ctx[4];
2443     uint32_t ep_ctx[5];
2444     int i;
2445     TRBCCode res;
2446 
2447     trace_usb_xhci_slot_configure(slotid);
2448     assert(slotid >= 1 && slotid <= xhci->numslots);
2449 
2450     ictx = xhci_mask64(pictx);
2451     octx = xhci->slots[slotid-1].ctx;
2452 
2453     DPRINTF("xhci: input context at "DMA_ADDR_FMT"\n", ictx);
2454     DPRINTF("xhci: output context at "DMA_ADDR_FMT"\n", octx);
2455 
2456     if (dc) {
2457         for (i = 2; i <= 31; i++) {
2458             if (xhci->slots[slotid-1].eps[i-1]) {
2459                 xhci_disable_ep(xhci, slotid, i);
2460             }
2461         }
2462 
2463         xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
2464         slot_ctx[3] &= ~(SLOT_STATE_MASK << SLOT_STATE_SHIFT);
2465         slot_ctx[3] |= SLOT_ADDRESSED << SLOT_STATE_SHIFT;
2466         DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
2467                 slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
2468         xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
2469 
2470         return CC_SUCCESS;
2471     }
2472 
2473     xhci_dma_read_u32s(xhci, ictx, ictl_ctx, sizeof(ictl_ctx));
2474 
2475     if ((ictl_ctx[0] & 0x3) != 0x0 || (ictl_ctx[1] & 0x3) != 0x1) {
2476         DPRINTF("xhci: invalid input context control %08x %08x\n",
2477                 ictl_ctx[0], ictl_ctx[1]);
2478         return CC_TRB_ERROR;
2479     }
2480 
2481     xhci_dma_read_u32s(xhci, ictx+32, islot_ctx, sizeof(islot_ctx));
2482     xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
2483 
2484     if (SLOT_STATE(slot_ctx[3]) < SLOT_ADDRESSED) {
2485         DPRINTF("xhci: invalid slot state %08x\n", slot_ctx[3]);
2486         return CC_CONTEXT_STATE_ERROR;
2487     }
2488 
2489     xhci_free_device_streams(xhci, slotid, ictl_ctx[0] | ictl_ctx[1]);
2490 
2491     for (i = 2; i <= 31; i++) {
2492         if (ictl_ctx[0] & (1<<i)) {
2493             xhci_disable_ep(xhci, slotid, i);
2494         }
2495         if (ictl_ctx[1] & (1<<i)) {
2496             xhci_dma_read_u32s(xhci, ictx+32+(32*i), ep_ctx, sizeof(ep_ctx));
2497             DPRINTF("xhci: input ep%d.%d context: %08x %08x %08x %08x %08x\n",
2498                     i/2, i%2, ep_ctx[0], ep_ctx[1], ep_ctx[2],
2499                     ep_ctx[3], ep_ctx[4]);
2500             xhci_disable_ep(xhci, slotid, i);
2501             res = xhci_enable_ep(xhci, slotid, i, octx+(32*i), ep_ctx);
2502             if (res != CC_SUCCESS) {
2503                 return res;
2504             }
2505             DPRINTF("xhci: output ep%d.%d context: %08x %08x %08x %08x %08x\n",
2506                     i/2, i%2, ep_ctx[0], ep_ctx[1], ep_ctx[2],
2507                     ep_ctx[3], ep_ctx[4]);
2508             xhci_dma_write_u32s(xhci, octx+(32*i), ep_ctx, sizeof(ep_ctx));
2509         }
2510     }
2511 
2512     res = xhci_alloc_device_streams(xhci, slotid, ictl_ctx[1]);
2513     if (res != CC_SUCCESS) {
2514         for (i = 2; i <= 31; i++) {
2515             if (ictl_ctx[1] & (1u << i)) {
2516                 xhci_disable_ep(xhci, slotid, i);
2517             }
2518         }
2519         return res;
2520     }
2521 
2522     slot_ctx[3] &= ~(SLOT_STATE_MASK << SLOT_STATE_SHIFT);
2523     slot_ctx[3] |= SLOT_CONFIGURED << SLOT_STATE_SHIFT;
2524     slot_ctx[0] &= ~(SLOT_CONTEXT_ENTRIES_MASK << SLOT_CONTEXT_ENTRIES_SHIFT);
2525     slot_ctx[0] |= islot_ctx[0] & (SLOT_CONTEXT_ENTRIES_MASK <<
2526                                    SLOT_CONTEXT_ENTRIES_SHIFT);
2527     DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
2528             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
2529 
2530     xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
2531 
2532     return CC_SUCCESS;
2533 }
2534 
2535 
2536 static TRBCCode xhci_evaluate_slot(XHCIState *xhci, unsigned int slotid,
2537                                    uint64_t pictx)
2538 {
2539     dma_addr_t ictx, octx;
2540     uint32_t ictl_ctx[2];
2541     uint32_t iep0_ctx[5];
2542     uint32_t ep0_ctx[5];
2543     uint32_t islot_ctx[4];
2544     uint32_t slot_ctx[4];
2545 
2546     trace_usb_xhci_slot_evaluate(slotid);
2547     assert(slotid >= 1 && slotid <= xhci->numslots);
2548 
2549     ictx = xhci_mask64(pictx);
2550     octx = xhci->slots[slotid-1].ctx;
2551 
2552     DPRINTF("xhci: input context at "DMA_ADDR_FMT"\n", ictx);
2553     DPRINTF("xhci: output context at "DMA_ADDR_FMT"\n", octx);
2554 
2555     xhci_dma_read_u32s(xhci, ictx, ictl_ctx, sizeof(ictl_ctx));
2556 
2557     if (ictl_ctx[0] != 0x0 || ictl_ctx[1] & ~0x3) {
2558         DPRINTF("xhci: invalid input context control %08x %08x\n",
2559                 ictl_ctx[0], ictl_ctx[1]);
2560         return CC_TRB_ERROR;
2561     }
2562 
2563     if (ictl_ctx[1] & 0x1) {
2564         xhci_dma_read_u32s(xhci, ictx+32, islot_ctx, sizeof(islot_ctx));
2565 
2566         DPRINTF("xhci: input slot context: %08x %08x %08x %08x\n",
2567                 islot_ctx[0], islot_ctx[1], islot_ctx[2], islot_ctx[3]);
2568 
2569         xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
2570 
2571         slot_ctx[1] &= ~0xFFFF; /* max exit latency */
2572         slot_ctx[1] |= islot_ctx[1] & 0xFFFF;
2573         slot_ctx[2] &= ~0xFF00000; /* interrupter target */
2574         slot_ctx[2] |= islot_ctx[2] & 0xFF000000;
2575 
2576         DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
2577                 slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
2578 
2579         xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
2580     }
2581 
2582     if (ictl_ctx[1] & 0x2) {
2583         xhci_dma_read_u32s(xhci, ictx+64, iep0_ctx, sizeof(iep0_ctx));
2584 
2585         DPRINTF("xhci: input ep0 context: %08x %08x %08x %08x %08x\n",
2586                 iep0_ctx[0], iep0_ctx[1], iep0_ctx[2],
2587                 iep0_ctx[3], iep0_ctx[4]);
2588 
2589         xhci_dma_read_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
2590 
2591         ep0_ctx[1] &= ~0xFFFF0000; /* max packet size*/
2592         ep0_ctx[1] |= iep0_ctx[1] & 0xFFFF0000;
2593 
2594         DPRINTF("xhci: output ep0 context: %08x %08x %08x %08x %08x\n",
2595                 ep0_ctx[0], ep0_ctx[1], ep0_ctx[2], ep0_ctx[3], ep0_ctx[4]);
2596 
2597         xhci_dma_write_u32s(xhci, octx+32, ep0_ctx, sizeof(ep0_ctx));
2598     }
2599 
2600     return CC_SUCCESS;
2601 }
2602 
2603 static TRBCCode xhci_reset_slot(XHCIState *xhci, unsigned int slotid)
2604 {
2605     uint32_t slot_ctx[4];
2606     dma_addr_t octx;
2607     int i;
2608 
2609     trace_usb_xhci_slot_reset(slotid);
2610     assert(slotid >= 1 && slotid <= xhci->numslots);
2611 
2612     octx = xhci->slots[slotid-1].ctx;
2613 
2614     DPRINTF("xhci: output context at "DMA_ADDR_FMT"\n", octx);
2615 
2616     for (i = 2; i <= 31; i++) {
2617         if (xhci->slots[slotid-1].eps[i-1]) {
2618             xhci_disable_ep(xhci, slotid, i);
2619         }
2620     }
2621 
2622     xhci_dma_read_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
2623     slot_ctx[3] &= ~(SLOT_STATE_MASK << SLOT_STATE_SHIFT);
2624     slot_ctx[3] |= SLOT_DEFAULT << SLOT_STATE_SHIFT;
2625     DPRINTF("xhci: output slot context: %08x %08x %08x %08x\n",
2626             slot_ctx[0], slot_ctx[1], slot_ctx[2], slot_ctx[3]);
2627     xhci_dma_write_u32s(xhci, octx, slot_ctx, sizeof(slot_ctx));
2628 
2629     return CC_SUCCESS;
2630 }
2631 
2632 static unsigned int xhci_get_slot(XHCIState *xhci, XHCIEvent *event, XHCITRB *trb)
2633 {
2634     unsigned int slotid;
2635     slotid = (trb->control >> TRB_CR_SLOTID_SHIFT) & TRB_CR_SLOTID_MASK;
2636     if (slotid < 1 || slotid > xhci->numslots) {
2637         DPRINTF("xhci: bad slot id %d\n", slotid);
2638         event->ccode = CC_TRB_ERROR;
2639         return 0;
2640     } else if (!xhci->slots[slotid-1].enabled) {
2641         DPRINTF("xhci: slot id %d not enabled\n", slotid);
2642         event->ccode = CC_SLOT_NOT_ENABLED_ERROR;
2643         return 0;
2644     }
2645     return slotid;
2646 }
2647 
2648 /* cleanup slot state on usb device detach */
2649 static void xhci_detach_slot(XHCIState *xhci, USBPort *uport)
2650 {
2651     int slot, ep;
2652 
2653     for (slot = 0; slot < xhci->numslots; slot++) {
2654         if (xhci->slots[slot].uport == uport) {
2655             break;
2656         }
2657     }
2658     if (slot == xhci->numslots) {
2659         return;
2660     }
2661 
2662     for (ep = 0; ep < 31; ep++) {
2663         if (xhci->slots[slot].eps[ep]) {
2664             xhci_ep_nuke_xfers(xhci, slot + 1, ep + 1, 0);
2665         }
2666     }
2667     xhci->slots[slot].uport = NULL;
2668 }
2669 
2670 static TRBCCode xhci_get_port_bandwidth(XHCIState *xhci, uint64_t pctx)
2671 {
2672     dma_addr_t ctx;
2673     uint8_t bw_ctx[xhci->numports+1];
2674 
2675     DPRINTF("xhci_get_port_bandwidth()\n");
2676 
2677     ctx = xhci_mask64(pctx);
2678 
2679     DPRINTF("xhci: bandwidth context at "DMA_ADDR_FMT"\n", ctx);
2680 
2681     /* TODO: actually implement real values here */
2682     bw_ctx[0] = 0;
2683     memset(&bw_ctx[1], 80, xhci->numports); /* 80% */
2684     pci_dma_write(PCI_DEVICE(xhci), ctx, bw_ctx, sizeof(bw_ctx));
2685 
2686     return CC_SUCCESS;
2687 }
2688 
2689 static uint32_t rotl(uint32_t v, unsigned count)
2690 {
2691     count &= 31;
2692     return (v << count) | (v >> (32 - count));
2693 }
2694 
2695 
2696 static uint32_t xhci_nec_challenge(uint32_t hi, uint32_t lo)
2697 {
2698     uint32_t val;
2699     val = rotl(lo - 0x49434878, 32 - ((hi>>8) & 0x1F));
2700     val += rotl(lo + 0x49434878, hi & 0x1F);
2701     val -= rotl(hi ^ 0x49434878, (lo >> 16) & 0x1F);
2702     return ~val;
2703 }
2704 
2705 static void xhci_via_challenge(XHCIState *xhci, uint64_t addr)
2706 {
2707     PCIDevice *pci_dev = PCI_DEVICE(xhci);
2708     uint32_t buf[8];
2709     uint32_t obuf[8];
2710     dma_addr_t paddr = xhci_mask64(addr);
2711 
2712     pci_dma_read(pci_dev, paddr, &buf, 32);
2713 
2714     memcpy(obuf, buf, sizeof(obuf));
2715 
2716     if ((buf[0] & 0xff) == 2) {
2717         obuf[0] = 0x49932000 + 0x54dc200 * buf[2] + 0x7429b578 * buf[3];
2718         obuf[0] |=  (buf[2] * buf[3]) & 0xff;
2719         obuf[1] = 0x0132bb37 + 0xe89 * buf[2] + 0xf09 * buf[3];
2720         obuf[2] = 0x0066c2e9 + 0x2091 * buf[2] + 0x19bd * buf[3];
2721         obuf[3] = 0xd5281342 + 0x2cc9691 * buf[2] + 0x2367662 * buf[3];
2722         obuf[4] = 0x0123c75c + 0x1595 * buf[2] + 0x19ec * buf[3];
2723         obuf[5] = 0x00f695de + 0x26fd * buf[2] + 0x3e9 * buf[3];
2724         obuf[6] = obuf[2] ^ obuf[3] ^ 0x29472956;
2725         obuf[7] = obuf[2] ^ obuf[3] ^ 0x65866593;
2726     }
2727 
2728     pci_dma_write(pci_dev, paddr, &obuf, 32);
2729 }
2730 
2731 static void xhci_process_commands(XHCIState *xhci)
2732 {
2733     XHCITRB trb;
2734     TRBType type;
2735     XHCIEvent event = {ER_COMMAND_COMPLETE, CC_SUCCESS};
2736     dma_addr_t addr;
2737     unsigned int i, slotid = 0;
2738 
2739     DPRINTF("xhci_process_commands()\n");
2740     if (!xhci_running(xhci)) {
2741         DPRINTF("xhci_process_commands() called while xHC stopped or paused\n");
2742         return;
2743     }
2744 
2745     xhci->crcr_low |= CRCR_CRR;
2746 
2747     while ((type = xhci_ring_fetch(xhci, &xhci->cmd_ring, &trb, &addr))) {
2748         event.ptr = addr;
2749         switch (type) {
2750         case CR_ENABLE_SLOT:
2751             for (i = 0; i < xhci->numslots; i++) {
2752                 if (!xhci->slots[i].enabled) {
2753                     break;
2754                 }
2755             }
2756             if (i >= xhci->numslots) {
2757                 DPRINTF("xhci: no device slots available\n");
2758                 event.ccode = CC_NO_SLOTS_ERROR;
2759             } else {
2760                 slotid = i+1;
2761                 event.ccode = xhci_enable_slot(xhci, slotid);
2762             }
2763             break;
2764         case CR_DISABLE_SLOT:
2765             slotid = xhci_get_slot(xhci, &event, &trb);
2766             if (slotid) {
2767                 event.ccode = xhci_disable_slot(xhci, slotid);
2768             }
2769             break;
2770         case CR_ADDRESS_DEVICE:
2771             slotid = xhci_get_slot(xhci, &event, &trb);
2772             if (slotid) {
2773                 event.ccode = xhci_address_slot(xhci, slotid, trb.parameter,
2774                                                 trb.control & TRB_CR_BSR);
2775             }
2776             break;
2777         case CR_CONFIGURE_ENDPOINT:
2778             slotid = xhci_get_slot(xhci, &event, &trb);
2779             if (slotid) {
2780                 event.ccode = xhci_configure_slot(xhci, slotid, trb.parameter,
2781                                                   trb.control & TRB_CR_DC);
2782             }
2783             break;
2784         case CR_EVALUATE_CONTEXT:
2785             slotid = xhci_get_slot(xhci, &event, &trb);
2786             if (slotid) {
2787                 event.ccode = xhci_evaluate_slot(xhci, slotid, trb.parameter);
2788             }
2789             break;
2790         case CR_STOP_ENDPOINT:
2791             slotid = xhci_get_slot(xhci, &event, &trb);
2792             if (slotid) {
2793                 unsigned int epid = (trb.control >> TRB_CR_EPID_SHIFT)
2794                     & TRB_CR_EPID_MASK;
2795                 event.ccode = xhci_stop_ep(xhci, slotid, epid);
2796             }
2797             break;
2798         case CR_RESET_ENDPOINT:
2799             slotid = xhci_get_slot(xhci, &event, &trb);
2800             if (slotid) {
2801                 unsigned int epid = (trb.control >> TRB_CR_EPID_SHIFT)
2802                     & TRB_CR_EPID_MASK;
2803                 event.ccode = xhci_reset_ep(xhci, slotid, epid);
2804             }
2805             break;
2806         case CR_SET_TR_DEQUEUE:
2807             slotid = xhci_get_slot(xhci, &event, &trb);
2808             if (slotid) {
2809                 unsigned int epid = (trb.control >> TRB_CR_EPID_SHIFT)
2810                     & TRB_CR_EPID_MASK;
2811                 unsigned int streamid = (trb.status >> 16) & 0xffff;
2812                 event.ccode = xhci_set_ep_dequeue(xhci, slotid,
2813                                                   epid, streamid,
2814                                                   trb.parameter);
2815             }
2816             break;
2817         case CR_RESET_DEVICE:
2818             slotid = xhci_get_slot(xhci, &event, &trb);
2819             if (slotid) {
2820                 event.ccode = xhci_reset_slot(xhci, slotid);
2821             }
2822             break;
2823         case CR_GET_PORT_BANDWIDTH:
2824             event.ccode = xhci_get_port_bandwidth(xhci, trb.parameter);
2825             break;
2826         case CR_VENDOR_VIA_CHALLENGE_RESPONSE:
2827             xhci_via_challenge(xhci, trb.parameter);
2828             break;
2829         case CR_VENDOR_NEC_FIRMWARE_REVISION:
2830             event.type = 48; /* NEC reply */
2831             event.length = 0x3025;
2832             break;
2833         case CR_VENDOR_NEC_CHALLENGE_RESPONSE:
2834         {
2835             uint32_t chi = trb.parameter >> 32;
2836             uint32_t clo = trb.parameter;
2837             uint32_t val = xhci_nec_challenge(chi, clo);
2838             event.length = val & 0xFFFF;
2839             event.epid = val >> 16;
2840             slotid = val >> 24;
2841             event.type = 48; /* NEC reply */
2842         }
2843         break;
2844         default:
2845             trace_usb_xhci_unimplemented("command", type);
2846             event.ccode = CC_TRB_ERROR;
2847             break;
2848         }
2849         event.slotid = slotid;
2850         xhci_event(xhci, &event, 0);
2851     }
2852 }
2853 
2854 static bool xhci_port_have_device(XHCIPort *port)
2855 {
2856     if (!port->uport->dev || !port->uport->dev->attached) {
2857         return false; /* no device present */
2858     }
2859     if (!((1 << port->uport->dev->speed) & port->speedmask)) {
2860         return false; /* speed mismatch */
2861     }
2862     return true;
2863 }
2864 
2865 static void xhci_port_notify(XHCIPort *port, uint32_t bits)
2866 {
2867     XHCIEvent ev = { ER_PORT_STATUS_CHANGE, CC_SUCCESS,
2868                      port->portnr << 24 };
2869 
2870     if ((port->portsc & bits) == bits) {
2871         return;
2872     }
2873     trace_usb_xhci_port_notify(port->portnr, bits);
2874     port->portsc |= bits;
2875     if (!xhci_running(port->xhci)) {
2876         return;
2877     }
2878     xhci_event(port->xhci, &ev, 0);
2879 }
2880 
2881 static void xhci_port_update(XHCIPort *port, int is_detach)
2882 {
2883     uint32_t pls = PLS_RX_DETECT;
2884 
2885     port->portsc = PORTSC_PP;
2886     if (!is_detach && xhci_port_have_device(port)) {
2887         port->portsc |= PORTSC_CCS;
2888         switch (port->uport->dev->speed) {
2889         case USB_SPEED_LOW:
2890             port->portsc |= PORTSC_SPEED_LOW;
2891             pls = PLS_POLLING;
2892             break;
2893         case USB_SPEED_FULL:
2894             port->portsc |= PORTSC_SPEED_FULL;
2895             pls = PLS_POLLING;
2896             break;
2897         case USB_SPEED_HIGH:
2898             port->portsc |= PORTSC_SPEED_HIGH;
2899             pls = PLS_POLLING;
2900             break;
2901         case USB_SPEED_SUPER:
2902             port->portsc |= PORTSC_SPEED_SUPER;
2903             port->portsc |= PORTSC_PED;
2904             pls = PLS_U0;
2905             break;
2906         }
2907     }
2908     set_field(&port->portsc, pls, PORTSC_PLS);
2909     trace_usb_xhci_port_link(port->portnr, pls);
2910     xhci_port_notify(port, PORTSC_CSC);
2911 }
2912 
2913 static void xhci_port_reset(XHCIPort *port, bool warm_reset)
2914 {
2915     trace_usb_xhci_port_reset(port->portnr, warm_reset);
2916 
2917     if (!xhci_port_have_device(port)) {
2918         return;
2919     }
2920 
2921     usb_device_reset(port->uport->dev);
2922 
2923     switch (port->uport->dev->speed) {
2924     case USB_SPEED_SUPER:
2925         if (warm_reset) {
2926             port->portsc |= PORTSC_WRC;
2927         }
2928         /* fall through */
2929     case USB_SPEED_LOW:
2930     case USB_SPEED_FULL:
2931     case USB_SPEED_HIGH:
2932         set_field(&port->portsc, PLS_U0, PORTSC_PLS);
2933         trace_usb_xhci_port_link(port->portnr, PLS_U0);
2934         port->portsc |= PORTSC_PED;
2935         break;
2936     }
2937 
2938     port->portsc &= ~PORTSC_PR;
2939     xhci_port_notify(port, PORTSC_PRC);
2940 }
2941 
2942 static void xhci_reset(DeviceState *dev)
2943 {
2944     XHCIState *xhci = XHCI(dev);
2945     int i;
2946 
2947     trace_usb_xhci_reset();
2948     if (!(xhci->usbsts & USBSTS_HCH)) {
2949         DPRINTF("xhci: reset while running!\n");
2950     }
2951 
2952     xhci->usbcmd = 0;
2953     xhci->usbsts = USBSTS_HCH;
2954     xhci->dnctrl = 0;
2955     xhci->crcr_low = 0;
2956     xhci->crcr_high = 0;
2957     xhci->dcbaap_low = 0;
2958     xhci->dcbaap_high = 0;
2959     xhci->config = 0;
2960 
2961     for (i = 0; i < xhci->numslots; i++) {
2962         xhci_disable_slot(xhci, i+1);
2963     }
2964 
2965     for (i = 0; i < xhci->numports; i++) {
2966         xhci_port_update(xhci->ports + i, 0);
2967     }
2968 
2969     for (i = 0; i < xhci->numintrs; i++) {
2970         xhci->intr[i].iman = 0;
2971         xhci->intr[i].imod = 0;
2972         xhci->intr[i].erstsz = 0;
2973         xhci->intr[i].erstba_low = 0;
2974         xhci->intr[i].erstba_high = 0;
2975         xhci->intr[i].erdp_low = 0;
2976         xhci->intr[i].erdp_high = 0;
2977         xhci->intr[i].msix_used = 0;
2978 
2979         xhci->intr[i].er_ep_idx = 0;
2980         xhci->intr[i].er_pcs = 1;
2981         xhci->intr[i].er_full = 0;
2982         xhci->intr[i].ev_buffer_put = 0;
2983         xhci->intr[i].ev_buffer_get = 0;
2984     }
2985 
2986     xhci->mfindex_start = qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL);
2987     xhci_mfwrap_update(xhci);
2988 }
2989 
2990 static uint64_t xhci_cap_read(void *ptr, hwaddr reg, unsigned size)
2991 {
2992     XHCIState *xhci = ptr;
2993     uint32_t ret;
2994 
2995     switch (reg) {
2996     case 0x00: /* HCIVERSION, CAPLENGTH */
2997         ret = 0x01000000 | LEN_CAP;
2998         break;
2999     case 0x04: /* HCSPARAMS 1 */
3000         ret = ((xhci->numports_2+xhci->numports_3)<<24)
3001             | (xhci->numintrs<<8) | xhci->numslots;
3002         break;
3003     case 0x08: /* HCSPARAMS 2 */
3004         ret = 0x0000000f;
3005         break;
3006     case 0x0c: /* HCSPARAMS 3 */
3007         ret = 0x00000000;
3008         break;
3009     case 0x10: /* HCCPARAMS */
3010         if (sizeof(dma_addr_t) == 4) {
3011             ret = 0x00080000 | (xhci->max_pstreams_mask << 12);
3012         } else {
3013             ret = 0x00080001 | (xhci->max_pstreams_mask << 12);
3014         }
3015         break;
3016     case 0x14: /* DBOFF */
3017         ret = OFF_DOORBELL;
3018         break;
3019     case 0x18: /* RTSOFF */
3020         ret = OFF_RUNTIME;
3021         break;
3022 
3023     /* extended capabilities */
3024     case 0x20: /* Supported Protocol:00 */
3025         ret = 0x02000402; /* USB 2.0 */
3026         break;
3027     case 0x24: /* Supported Protocol:04 */
3028         ret = 0x20425355; /* "USB " */
3029         break;
3030     case 0x28: /* Supported Protocol:08 */
3031         if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
3032             ret = (xhci->numports_2<<8) | (xhci->numports_3+1);
3033         } else {
3034             ret = (xhci->numports_2<<8) | 1;
3035         }
3036         break;
3037     case 0x2c: /* Supported Protocol:0c */
3038         ret = 0x00000000; /* reserved */
3039         break;
3040     case 0x30: /* Supported Protocol:00 */
3041         ret = 0x03000002; /* USB 3.0 */
3042         break;
3043     case 0x34: /* Supported Protocol:04 */
3044         ret = 0x20425355; /* "USB " */
3045         break;
3046     case 0x38: /* Supported Protocol:08 */
3047         if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
3048             ret = (xhci->numports_3<<8) | 1;
3049         } else {
3050             ret = (xhci->numports_3<<8) | (xhci->numports_2+1);
3051         }
3052         break;
3053     case 0x3c: /* Supported Protocol:0c */
3054         ret = 0x00000000; /* reserved */
3055         break;
3056     default:
3057         trace_usb_xhci_unimplemented("cap read", reg);
3058         ret = 0;
3059     }
3060 
3061     trace_usb_xhci_cap_read(reg, ret);
3062     return ret;
3063 }
3064 
3065 static uint64_t xhci_port_read(void *ptr, hwaddr reg, unsigned size)
3066 {
3067     XHCIPort *port = ptr;
3068     uint32_t ret;
3069 
3070     switch (reg) {
3071     case 0x00: /* PORTSC */
3072         ret = port->portsc;
3073         break;
3074     case 0x04: /* PORTPMSC */
3075     case 0x08: /* PORTLI */
3076         ret = 0;
3077         break;
3078     case 0x0c: /* reserved */
3079     default:
3080         trace_usb_xhci_unimplemented("port read", reg);
3081         ret = 0;
3082     }
3083 
3084     trace_usb_xhci_port_read(port->portnr, reg, ret);
3085     return ret;
3086 }
3087 
3088 static void xhci_port_write(void *ptr, hwaddr reg,
3089                             uint64_t val, unsigned size)
3090 {
3091     XHCIPort *port = ptr;
3092     uint32_t portsc, notify;
3093 
3094     trace_usb_xhci_port_write(port->portnr, reg, val);
3095 
3096     switch (reg) {
3097     case 0x00: /* PORTSC */
3098         /* write-1-to-start bits */
3099         if (val & PORTSC_WPR) {
3100             xhci_port_reset(port, true);
3101             break;
3102         }
3103         if (val & PORTSC_PR) {
3104             xhci_port_reset(port, false);
3105             break;
3106         }
3107 
3108         portsc = port->portsc;
3109         notify = 0;
3110         /* write-1-to-clear bits*/
3111         portsc &= ~(val & (PORTSC_CSC|PORTSC_PEC|PORTSC_WRC|PORTSC_OCC|
3112                            PORTSC_PRC|PORTSC_PLC|PORTSC_CEC));
3113         if (val & PORTSC_LWS) {
3114             /* overwrite PLS only when LWS=1 */
3115             uint32_t old_pls = get_field(port->portsc, PORTSC_PLS);
3116             uint32_t new_pls = get_field(val, PORTSC_PLS);
3117             switch (new_pls) {
3118             case PLS_U0:
3119                 if (old_pls != PLS_U0) {
3120                     set_field(&portsc, new_pls, PORTSC_PLS);
3121                     trace_usb_xhci_port_link(port->portnr, new_pls);
3122                     notify = PORTSC_PLC;
3123                 }
3124                 break;
3125             case PLS_U3:
3126                 if (old_pls < PLS_U3) {
3127                     set_field(&portsc, new_pls, PORTSC_PLS);
3128                     trace_usb_xhci_port_link(port->portnr, new_pls);
3129                 }
3130                 break;
3131             case PLS_RESUME:
3132                 /* windows does this for some reason, don't spam stderr */
3133                 break;
3134             default:
3135                 DPRINTF("%s: ignore pls write (old %d, new %d)\n",
3136                         __func__, old_pls, new_pls);
3137                 break;
3138             }
3139         }
3140         /* read/write bits */
3141         portsc &= ~(PORTSC_PP|PORTSC_WCE|PORTSC_WDE|PORTSC_WOE);
3142         portsc |= (val & (PORTSC_PP|PORTSC_WCE|PORTSC_WDE|PORTSC_WOE));
3143         port->portsc = portsc;
3144         if (notify) {
3145             xhci_port_notify(port, notify);
3146         }
3147         break;
3148     case 0x04: /* PORTPMSC */
3149     case 0x08: /* PORTLI */
3150     default:
3151         trace_usb_xhci_unimplemented("port write", reg);
3152     }
3153 }
3154 
3155 static uint64_t xhci_oper_read(void *ptr, hwaddr reg, unsigned size)
3156 {
3157     XHCIState *xhci = ptr;
3158     uint32_t ret;
3159 
3160     switch (reg) {
3161     case 0x00: /* USBCMD */
3162         ret = xhci->usbcmd;
3163         break;
3164     case 0x04: /* USBSTS */
3165         ret = xhci->usbsts;
3166         break;
3167     case 0x08: /* PAGESIZE */
3168         ret = 1; /* 4KiB */
3169         break;
3170     case 0x14: /* DNCTRL */
3171         ret = xhci->dnctrl;
3172         break;
3173     case 0x18: /* CRCR low */
3174         ret = xhci->crcr_low & ~0xe;
3175         break;
3176     case 0x1c: /* CRCR high */
3177         ret = xhci->crcr_high;
3178         break;
3179     case 0x30: /* DCBAAP low */
3180         ret = xhci->dcbaap_low;
3181         break;
3182     case 0x34: /* DCBAAP high */
3183         ret = xhci->dcbaap_high;
3184         break;
3185     case 0x38: /* CONFIG */
3186         ret = xhci->config;
3187         break;
3188     default:
3189         trace_usb_xhci_unimplemented("oper read", reg);
3190         ret = 0;
3191     }
3192 
3193     trace_usb_xhci_oper_read(reg, ret);
3194     return ret;
3195 }
3196 
3197 static void xhci_oper_write(void *ptr, hwaddr reg,
3198                             uint64_t val, unsigned size)
3199 {
3200     XHCIState *xhci = ptr;
3201     DeviceState *d = DEVICE(ptr);
3202 
3203     trace_usb_xhci_oper_write(reg, val);
3204 
3205     switch (reg) {
3206     case 0x00: /* USBCMD */
3207         if ((val & USBCMD_RS) && !(xhci->usbcmd & USBCMD_RS)) {
3208             xhci_run(xhci);
3209         } else if (!(val & USBCMD_RS) && (xhci->usbcmd & USBCMD_RS)) {
3210             xhci_stop(xhci);
3211         }
3212         if (val & USBCMD_CSS) {
3213             /* save state */
3214             xhci->usbsts &= ~USBSTS_SRE;
3215         }
3216         if (val & USBCMD_CRS) {
3217             /* restore state */
3218             xhci->usbsts |= USBSTS_SRE;
3219         }
3220         xhci->usbcmd = val & 0xc0f;
3221         xhci_mfwrap_update(xhci);
3222         if (val & USBCMD_HCRST) {
3223             xhci_reset(d);
3224         }
3225         xhci_intx_update(xhci);
3226         break;
3227 
3228     case 0x04: /* USBSTS */
3229         /* these bits are write-1-to-clear */
3230         xhci->usbsts &= ~(val & (USBSTS_HSE|USBSTS_EINT|USBSTS_PCD|USBSTS_SRE));
3231         xhci_intx_update(xhci);
3232         break;
3233 
3234     case 0x14: /* DNCTRL */
3235         xhci->dnctrl = val & 0xffff;
3236         break;
3237     case 0x18: /* CRCR low */
3238         xhci->crcr_low = (val & 0xffffffcf) | (xhci->crcr_low & CRCR_CRR);
3239         break;
3240     case 0x1c: /* CRCR high */
3241         xhci->crcr_high = val;
3242         if (xhci->crcr_low & (CRCR_CA|CRCR_CS) && (xhci->crcr_low & CRCR_CRR)) {
3243             XHCIEvent event = {ER_COMMAND_COMPLETE, CC_COMMAND_RING_STOPPED};
3244             xhci->crcr_low &= ~CRCR_CRR;
3245             xhci_event(xhci, &event, 0);
3246             DPRINTF("xhci: command ring stopped (CRCR=%08x)\n", xhci->crcr_low);
3247         } else {
3248             dma_addr_t base = xhci_addr64(xhci->crcr_low & ~0x3f, val);
3249             xhci_ring_init(xhci, &xhci->cmd_ring, base);
3250         }
3251         xhci->crcr_low &= ~(CRCR_CA | CRCR_CS);
3252         break;
3253     case 0x30: /* DCBAAP low */
3254         xhci->dcbaap_low = val & 0xffffffc0;
3255         break;
3256     case 0x34: /* DCBAAP high */
3257         xhci->dcbaap_high = val;
3258         break;
3259     case 0x38: /* CONFIG */
3260         xhci->config = val & 0xff;
3261         break;
3262     default:
3263         trace_usb_xhci_unimplemented("oper write", reg);
3264     }
3265 }
3266 
3267 static uint64_t xhci_runtime_read(void *ptr, hwaddr reg,
3268                                   unsigned size)
3269 {
3270     XHCIState *xhci = ptr;
3271     uint32_t ret = 0;
3272 
3273     if (reg < 0x20) {
3274         switch (reg) {
3275         case 0x00: /* MFINDEX */
3276             ret = xhci_mfindex_get(xhci) & 0x3fff;
3277             break;
3278         default:
3279             trace_usb_xhci_unimplemented("runtime read", reg);
3280             break;
3281         }
3282     } else {
3283         int v = (reg - 0x20) / 0x20;
3284         XHCIInterrupter *intr = &xhci->intr[v];
3285         switch (reg & 0x1f) {
3286         case 0x00: /* IMAN */
3287             ret = intr->iman;
3288             break;
3289         case 0x04: /* IMOD */
3290             ret = intr->imod;
3291             break;
3292         case 0x08: /* ERSTSZ */
3293             ret = intr->erstsz;
3294             break;
3295         case 0x10: /* ERSTBA low */
3296             ret = intr->erstba_low;
3297             break;
3298         case 0x14: /* ERSTBA high */
3299             ret = intr->erstba_high;
3300             break;
3301         case 0x18: /* ERDP low */
3302             ret = intr->erdp_low;
3303             break;
3304         case 0x1c: /* ERDP high */
3305             ret = intr->erdp_high;
3306             break;
3307         }
3308     }
3309 
3310     trace_usb_xhci_runtime_read(reg, ret);
3311     return ret;
3312 }
3313 
3314 static void xhci_runtime_write(void *ptr, hwaddr reg,
3315                                uint64_t val, unsigned size)
3316 {
3317     XHCIState *xhci = ptr;
3318     int v = (reg - 0x20) / 0x20;
3319     XHCIInterrupter *intr = &xhci->intr[v];
3320     trace_usb_xhci_runtime_write(reg, val);
3321 
3322     if (reg < 0x20) {
3323         trace_usb_xhci_unimplemented("runtime write", reg);
3324         return;
3325     }
3326 
3327     switch (reg & 0x1f) {
3328     case 0x00: /* IMAN */
3329         if (val & IMAN_IP) {
3330             intr->iman &= ~IMAN_IP;
3331         }
3332         intr->iman &= ~IMAN_IE;
3333         intr->iman |= val & IMAN_IE;
3334         if (v == 0) {
3335             xhci_intx_update(xhci);
3336         }
3337         xhci_msix_update(xhci, v);
3338         break;
3339     case 0x04: /* IMOD */
3340         intr->imod = val;
3341         break;
3342     case 0x08: /* ERSTSZ */
3343         intr->erstsz = val & 0xffff;
3344         break;
3345     case 0x10: /* ERSTBA low */
3346         /* XXX NEC driver bug: it doesn't align this to 64 bytes
3347         intr->erstba_low = val & 0xffffffc0; */
3348         intr->erstba_low = val & 0xfffffff0;
3349         break;
3350     case 0x14: /* ERSTBA high */
3351         intr->erstba_high = val;
3352         xhci_er_reset(xhci, v);
3353         break;
3354     case 0x18: /* ERDP low */
3355         if (val & ERDP_EHB) {
3356             intr->erdp_low &= ~ERDP_EHB;
3357         }
3358         intr->erdp_low = (val & ~ERDP_EHB) | (intr->erdp_low & ERDP_EHB);
3359         if (val & ERDP_EHB) {
3360             dma_addr_t erdp = xhci_addr64(intr->erdp_low, intr->erdp_high);
3361             unsigned int dp_idx = (erdp - intr->er_start) / TRB_SIZE;
3362             if (erdp >= intr->er_start &&
3363                 erdp < (intr->er_start + TRB_SIZE * intr->er_size) &&
3364                 dp_idx != intr->er_ep_idx) {
3365                 xhci_intr_raise(xhci, v);
3366             }
3367         }
3368         break;
3369     case 0x1c: /* ERDP high */
3370         intr->erdp_high = val;
3371         xhci_events_update(xhci, v);
3372         break;
3373     default:
3374         trace_usb_xhci_unimplemented("oper write", reg);
3375     }
3376 }
3377 
3378 static uint64_t xhci_doorbell_read(void *ptr, hwaddr reg,
3379                                    unsigned size)
3380 {
3381     /* doorbells always read as 0 */
3382     trace_usb_xhci_doorbell_read(reg, 0);
3383     return 0;
3384 }
3385 
3386 static void xhci_doorbell_write(void *ptr, hwaddr reg,
3387                                 uint64_t val, unsigned size)
3388 {
3389     XHCIState *xhci = ptr;
3390     unsigned int epid, streamid;
3391 
3392     trace_usb_xhci_doorbell_write(reg, val);
3393 
3394     if (!xhci_running(xhci)) {
3395         DPRINTF("xhci: wrote doorbell while xHC stopped or paused\n");
3396         return;
3397     }
3398 
3399     reg >>= 2;
3400 
3401     if (reg == 0) {
3402         if (val == 0) {
3403             xhci_process_commands(xhci);
3404         } else {
3405             DPRINTF("xhci: bad doorbell 0 write: 0x%x\n",
3406                     (uint32_t)val);
3407         }
3408     } else {
3409         epid = val & 0xff;
3410         streamid = (val >> 16) & 0xffff;
3411         if (reg > xhci->numslots) {
3412             DPRINTF("xhci: bad doorbell %d\n", (int)reg);
3413         } else if (epid > 31) {
3414             DPRINTF("xhci: bad doorbell %d write: 0x%x\n",
3415                     (int)reg, (uint32_t)val);
3416         } else {
3417             xhci_kick_ep(xhci, reg, epid, streamid);
3418         }
3419     }
3420 }
3421 
3422 static void xhci_cap_write(void *opaque, hwaddr addr, uint64_t val,
3423                            unsigned width)
3424 {
3425     /* nothing */
3426 }
3427 
3428 static const MemoryRegionOps xhci_cap_ops = {
3429     .read = xhci_cap_read,
3430     .write = xhci_cap_write,
3431     .valid.min_access_size = 1,
3432     .valid.max_access_size = 4,
3433     .impl.min_access_size = 4,
3434     .impl.max_access_size = 4,
3435     .endianness = DEVICE_LITTLE_ENDIAN,
3436 };
3437 
3438 static const MemoryRegionOps xhci_oper_ops = {
3439     .read = xhci_oper_read,
3440     .write = xhci_oper_write,
3441     .valid.min_access_size = 4,
3442     .valid.max_access_size = 4,
3443     .endianness = DEVICE_LITTLE_ENDIAN,
3444 };
3445 
3446 static const MemoryRegionOps xhci_port_ops = {
3447     .read = xhci_port_read,
3448     .write = xhci_port_write,
3449     .valid.min_access_size = 4,
3450     .valid.max_access_size = 4,
3451     .endianness = DEVICE_LITTLE_ENDIAN,
3452 };
3453 
3454 static const MemoryRegionOps xhci_runtime_ops = {
3455     .read = xhci_runtime_read,
3456     .write = xhci_runtime_write,
3457     .valid.min_access_size = 4,
3458     .valid.max_access_size = 4,
3459     .endianness = DEVICE_LITTLE_ENDIAN,
3460 };
3461 
3462 static const MemoryRegionOps xhci_doorbell_ops = {
3463     .read = xhci_doorbell_read,
3464     .write = xhci_doorbell_write,
3465     .valid.min_access_size = 4,
3466     .valid.max_access_size = 4,
3467     .endianness = DEVICE_LITTLE_ENDIAN,
3468 };
3469 
3470 static void xhci_attach(USBPort *usbport)
3471 {
3472     XHCIState *xhci = usbport->opaque;
3473     XHCIPort *port = xhci_lookup_port(xhci, usbport);
3474 
3475     xhci_port_update(port, 0);
3476 }
3477 
3478 static void xhci_detach(USBPort *usbport)
3479 {
3480     XHCIState *xhci = usbport->opaque;
3481     XHCIPort *port = xhci_lookup_port(xhci, usbport);
3482 
3483     xhci_detach_slot(xhci, usbport);
3484     xhci_port_update(port, 1);
3485 }
3486 
3487 static void xhci_wakeup(USBPort *usbport)
3488 {
3489     XHCIState *xhci = usbport->opaque;
3490     XHCIPort *port = xhci_lookup_port(xhci, usbport);
3491 
3492     if (get_field(port->portsc, PORTSC_PLS) != PLS_U3) {
3493         return;
3494     }
3495     set_field(&port->portsc, PLS_RESUME, PORTSC_PLS);
3496     xhci_port_notify(port, PORTSC_PLC);
3497 }
3498 
3499 static void xhci_complete(USBPort *port, USBPacket *packet)
3500 {
3501     XHCITransfer *xfer = container_of(packet, XHCITransfer, packet);
3502 
3503     if (packet->status == USB_RET_REMOVE_FROM_QUEUE) {
3504         xhci_ep_nuke_one_xfer(xfer, 0);
3505         return;
3506     }
3507     xhci_try_complete_packet(xfer);
3508     xhci_kick_epctx(xfer->epctx, xfer->streamid);
3509     if (xfer->complete) {
3510         xhci_ep_free_xfer(xfer);
3511     }
3512 }
3513 
3514 static void xhci_child_detach(USBPort *uport, USBDevice *child)
3515 {
3516     USBBus *bus = usb_bus_from_device(child);
3517     XHCIState *xhci = container_of(bus, XHCIState, bus);
3518 
3519     xhci_detach_slot(xhci, child->port);
3520 }
3521 
3522 static USBPortOps xhci_uport_ops = {
3523     .attach   = xhci_attach,
3524     .detach   = xhci_detach,
3525     .wakeup   = xhci_wakeup,
3526     .complete = xhci_complete,
3527     .child_detach = xhci_child_detach,
3528 };
3529 
3530 static int xhci_find_epid(USBEndpoint *ep)
3531 {
3532     if (ep->nr == 0) {
3533         return 1;
3534     }
3535     if (ep->pid == USB_TOKEN_IN) {
3536         return ep->nr * 2 + 1;
3537     } else {
3538         return ep->nr * 2;
3539     }
3540 }
3541 
3542 static USBEndpoint *xhci_epid_to_usbep(XHCIEPContext *epctx)
3543 {
3544     USBPort *uport;
3545     uint32_t token;
3546 
3547     if (!epctx) {
3548         return NULL;
3549     }
3550     uport = epctx->xhci->slots[epctx->slotid - 1].uport;
3551     token = (epctx->epid & 1) ? USB_TOKEN_IN : USB_TOKEN_OUT;
3552     if (!uport) {
3553         return NULL;
3554     }
3555     return usb_ep_get(uport->dev, token, epctx->epid >> 1);
3556 }
3557 
3558 static void xhci_wakeup_endpoint(USBBus *bus, USBEndpoint *ep,
3559                                  unsigned int stream)
3560 {
3561     XHCIState *xhci = container_of(bus, XHCIState, bus);
3562     int slotid;
3563 
3564     DPRINTF("%s\n", __func__);
3565     slotid = ep->dev->addr;
3566     if (slotid == 0 || !xhci->slots[slotid-1].enabled) {
3567         DPRINTF("%s: oops, no slot for dev %d\n", __func__, ep->dev->addr);
3568         return;
3569     }
3570     xhci_kick_ep(xhci, slotid, xhci_find_epid(ep), stream);
3571 }
3572 
3573 static USBBusOps xhci_bus_ops = {
3574     .wakeup_endpoint = xhci_wakeup_endpoint,
3575 };
3576 
3577 static void usb_xhci_init(XHCIState *xhci)
3578 {
3579     DeviceState *dev = DEVICE(xhci);
3580     XHCIPort *port;
3581     int i, usbports, speedmask;
3582 
3583     xhci->usbsts = USBSTS_HCH;
3584 
3585     if (xhci->numports_2 > MAXPORTS_2) {
3586         xhci->numports_2 = MAXPORTS_2;
3587     }
3588     if (xhci->numports_3 > MAXPORTS_3) {
3589         xhci->numports_3 = MAXPORTS_3;
3590     }
3591     usbports = MAX(xhci->numports_2, xhci->numports_3);
3592     xhci->numports = xhci->numports_2 + xhci->numports_3;
3593 
3594     usb_bus_new(&xhci->bus, sizeof(xhci->bus), &xhci_bus_ops, dev);
3595 
3596     for (i = 0; i < usbports; i++) {
3597         speedmask = 0;
3598         if (i < xhci->numports_2) {
3599             if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
3600                 port = &xhci->ports[i + xhci->numports_3];
3601                 port->portnr = i + 1 + xhci->numports_3;
3602             } else {
3603                 port = &xhci->ports[i];
3604                 port->portnr = i + 1;
3605             }
3606             port->uport = &xhci->uports[i];
3607             port->speedmask =
3608                 USB_SPEED_MASK_LOW  |
3609                 USB_SPEED_MASK_FULL |
3610                 USB_SPEED_MASK_HIGH;
3611             snprintf(port->name, sizeof(port->name), "usb2 port #%d", i+1);
3612             speedmask |= port->speedmask;
3613         }
3614         if (i < xhci->numports_3) {
3615             if (xhci_get_flag(xhci, XHCI_FLAG_SS_FIRST)) {
3616                 port = &xhci->ports[i];
3617                 port->portnr = i + 1;
3618             } else {
3619                 port = &xhci->ports[i + xhci->numports_2];
3620                 port->portnr = i + 1 + xhci->numports_2;
3621             }
3622             port->uport = &xhci->uports[i];
3623             port->speedmask = USB_SPEED_MASK_SUPER;
3624             snprintf(port->name, sizeof(port->name), "usb3 port #%d", i+1);
3625             speedmask |= port->speedmask;
3626         }
3627         usb_register_port(&xhci->bus, &xhci->uports[i], xhci, i,
3628                           &xhci_uport_ops, speedmask);
3629     }
3630 }
3631 
3632 static void usb_xhci_realize(struct PCIDevice *dev, Error **errp)
3633 {
3634     int i, ret;
3635     Error *err = NULL;
3636 
3637     XHCIState *xhci = XHCI(dev);
3638 
3639     dev->config[PCI_CLASS_PROG] = 0x30;    /* xHCI */
3640     dev->config[PCI_INTERRUPT_PIN] = 0x01; /* interrupt pin 1 */
3641     dev->config[PCI_CACHE_LINE_SIZE] = 0x10;
3642     dev->config[0x60] = 0x30; /* release number */
3643 
3644     if (xhci->numintrs > MAXINTRS) {
3645         xhci->numintrs = MAXINTRS;
3646     }
3647     while (xhci->numintrs & (xhci->numintrs - 1)) {   /* ! power of 2 */
3648         xhci->numintrs++;
3649     }
3650     if (xhci->numintrs < 1) {
3651         xhci->numintrs = 1;
3652     }
3653     if (xhci->numslots > MAXSLOTS) {
3654         xhci->numslots = MAXSLOTS;
3655     }
3656     if (xhci->numslots < 1) {
3657         xhci->numslots = 1;
3658     }
3659     if (xhci_get_flag(xhci, XHCI_FLAG_ENABLE_STREAMS)) {
3660         xhci->max_pstreams_mask = 7; /* == 256 primary streams */
3661     } else {
3662         xhci->max_pstreams_mask = 0;
3663     }
3664 
3665     if (xhci->msi != ON_OFF_AUTO_OFF) {
3666         ret = msi_init(dev, 0x70, xhci->numintrs, true, false, &err);
3667         /* Any error other than -ENOTSUP(board's MSI support is broken)
3668          * is a programming error */
3669         assert(!ret || ret == -ENOTSUP);
3670         if (ret && xhci->msi == ON_OFF_AUTO_ON) {
3671             /* Can't satisfy user's explicit msi=on request, fail */
3672             error_append_hint(&err, "You have to use msi=auto (default) or "
3673                     "msi=off with this machine type.\n");
3674             error_propagate(errp, err);
3675             return;
3676         }
3677         assert(!err || xhci->msi == ON_OFF_AUTO_AUTO);
3678         /* With msi=auto, we fall back to MSI off silently */
3679         error_free(err);
3680     }
3681 
3682     usb_xhci_init(xhci);
3683     xhci->mfwrap_timer = timer_new_ns(QEMU_CLOCK_VIRTUAL, xhci_mfwrap_timer, xhci);
3684 
3685     memory_region_init(&xhci->mem, OBJECT(xhci), "xhci", LEN_REGS);
3686     memory_region_init_io(&xhci->mem_cap, OBJECT(xhci), &xhci_cap_ops, xhci,
3687                           "capabilities", LEN_CAP);
3688     memory_region_init_io(&xhci->mem_oper, OBJECT(xhci), &xhci_oper_ops, xhci,
3689                           "operational", 0x400);
3690     memory_region_init_io(&xhci->mem_runtime, OBJECT(xhci), &xhci_runtime_ops, xhci,
3691                           "runtime", LEN_RUNTIME);
3692     memory_region_init_io(&xhci->mem_doorbell, OBJECT(xhci), &xhci_doorbell_ops, xhci,
3693                           "doorbell", LEN_DOORBELL);
3694 
3695     memory_region_add_subregion(&xhci->mem, 0,            &xhci->mem_cap);
3696     memory_region_add_subregion(&xhci->mem, OFF_OPER,     &xhci->mem_oper);
3697     memory_region_add_subregion(&xhci->mem, OFF_RUNTIME,  &xhci->mem_runtime);
3698     memory_region_add_subregion(&xhci->mem, OFF_DOORBELL, &xhci->mem_doorbell);
3699 
3700     for (i = 0; i < xhci->numports; i++) {
3701         XHCIPort *port = &xhci->ports[i];
3702         uint32_t offset = OFF_OPER + 0x400 + 0x10 * i;
3703         port->xhci = xhci;
3704         memory_region_init_io(&port->mem, OBJECT(xhci), &xhci_port_ops, port,
3705                               port->name, 0x10);
3706         memory_region_add_subregion(&xhci->mem, offset, &port->mem);
3707     }
3708 
3709     pci_register_bar(dev, 0,
3710                      PCI_BASE_ADDRESS_SPACE_MEMORY|PCI_BASE_ADDRESS_MEM_TYPE_64,
3711                      &xhci->mem);
3712 
3713     if (pci_bus_is_express(dev->bus) ||
3714         xhci_get_flag(xhci, XHCI_FLAG_FORCE_PCIE_ENDCAP)) {
3715         ret = pcie_endpoint_cap_init(dev, 0xa0);
3716         assert(ret >= 0);
3717     }
3718 
3719     if (xhci->msix != ON_OFF_AUTO_OFF) {
3720         /* TODO check for errors, and should fail when msix=on */
3721         msix_init(dev, xhci->numintrs,
3722                   &xhci->mem, 0, OFF_MSIX_TABLE,
3723                   &xhci->mem, 0, OFF_MSIX_PBA,
3724                   0x90, NULL);
3725     }
3726 }
3727 
3728 static void usb_xhci_exit(PCIDevice *dev)
3729 {
3730     int i;
3731     XHCIState *xhci = XHCI(dev);
3732 
3733     trace_usb_xhci_exit();
3734 
3735     for (i = 0; i < xhci->numslots; i++) {
3736         xhci_disable_slot(xhci, i + 1);
3737     }
3738 
3739     if (xhci->mfwrap_timer) {
3740         timer_del(xhci->mfwrap_timer);
3741         timer_free(xhci->mfwrap_timer);
3742         xhci->mfwrap_timer = NULL;
3743     }
3744 
3745     memory_region_del_subregion(&xhci->mem, &xhci->mem_cap);
3746     memory_region_del_subregion(&xhci->mem, &xhci->mem_oper);
3747     memory_region_del_subregion(&xhci->mem, &xhci->mem_runtime);
3748     memory_region_del_subregion(&xhci->mem, &xhci->mem_doorbell);
3749 
3750     for (i = 0; i < xhci->numports; i++) {
3751         XHCIPort *port = &xhci->ports[i];
3752         memory_region_del_subregion(&xhci->mem, &port->mem);
3753     }
3754 
3755     /* destroy msix memory region */
3756     if (dev->msix_table && dev->msix_pba
3757         && dev->msix_entry_used) {
3758         msix_uninit(dev, &xhci->mem, &xhci->mem);
3759     }
3760 
3761     usb_bus_release(&xhci->bus);
3762 }
3763 
3764 static int usb_xhci_post_load(void *opaque, int version_id)
3765 {
3766     XHCIState *xhci = opaque;
3767     PCIDevice *pci_dev = PCI_DEVICE(xhci);
3768     XHCISlot *slot;
3769     XHCIEPContext *epctx;
3770     dma_addr_t dcbaap, pctx;
3771     uint32_t slot_ctx[4];
3772     uint32_t ep_ctx[5];
3773     int slotid, epid, state, intr;
3774 
3775     dcbaap = xhci_addr64(xhci->dcbaap_low, xhci->dcbaap_high);
3776 
3777     for (slotid = 1; slotid <= xhci->numslots; slotid++) {
3778         slot = &xhci->slots[slotid-1];
3779         if (!slot->addressed) {
3780             continue;
3781         }
3782         slot->ctx =
3783             xhci_mask64(ldq_le_pci_dma(pci_dev, dcbaap + 8 * slotid));
3784         xhci_dma_read_u32s(xhci, slot->ctx, slot_ctx, sizeof(slot_ctx));
3785         slot->uport = xhci_lookup_uport(xhci, slot_ctx);
3786         if (!slot->uport) {
3787             /* should not happen, but may trigger on guest bugs */
3788             slot->enabled = 0;
3789             slot->addressed = 0;
3790             continue;
3791         }
3792         assert(slot->uport && slot->uport->dev);
3793 
3794         for (epid = 1; epid <= 31; epid++) {
3795             pctx = slot->ctx + 32 * epid;
3796             xhci_dma_read_u32s(xhci, pctx, ep_ctx, sizeof(ep_ctx));
3797             state = ep_ctx[0] & EP_STATE_MASK;
3798             if (state == EP_DISABLED) {
3799                 continue;
3800             }
3801             epctx = xhci_alloc_epctx(xhci, slotid, epid);
3802             slot->eps[epid-1] = epctx;
3803             xhci_init_epctx(epctx, pctx, ep_ctx);
3804             epctx->state = state;
3805             if (state == EP_RUNNING) {
3806                 /* kick endpoint after vmload is finished */
3807                 timer_mod(epctx->kick_timer, qemu_clock_get_ns(QEMU_CLOCK_VIRTUAL));
3808             }
3809         }
3810     }
3811 
3812     for (intr = 0; intr < xhci->numintrs; intr++) {
3813         if (xhci->intr[intr].msix_used) {
3814             msix_vector_use(pci_dev, intr);
3815         } else {
3816             msix_vector_unuse(pci_dev, intr);
3817         }
3818     }
3819 
3820     return 0;
3821 }
3822 
3823 static const VMStateDescription vmstate_xhci_ring = {
3824     .name = "xhci-ring",
3825     .version_id = 1,
3826     .fields = (VMStateField[]) {
3827         VMSTATE_UINT64(dequeue, XHCIRing),
3828         VMSTATE_BOOL(ccs, XHCIRing),
3829         VMSTATE_END_OF_LIST()
3830     }
3831 };
3832 
3833 static const VMStateDescription vmstate_xhci_port = {
3834     .name = "xhci-port",
3835     .version_id = 1,
3836     .fields = (VMStateField[]) {
3837         VMSTATE_UINT32(portsc, XHCIPort),
3838         VMSTATE_END_OF_LIST()
3839     }
3840 };
3841 
3842 static const VMStateDescription vmstate_xhci_slot = {
3843     .name = "xhci-slot",
3844     .version_id = 1,
3845     .fields = (VMStateField[]) {
3846         VMSTATE_BOOL(enabled,   XHCISlot),
3847         VMSTATE_BOOL(addressed, XHCISlot),
3848         VMSTATE_END_OF_LIST()
3849     }
3850 };
3851 
3852 static const VMStateDescription vmstate_xhci_event = {
3853     .name = "xhci-event",
3854     .version_id = 1,
3855     .fields = (VMStateField[]) {
3856         VMSTATE_UINT32(type,   XHCIEvent),
3857         VMSTATE_UINT32(ccode,  XHCIEvent),
3858         VMSTATE_UINT64(ptr,    XHCIEvent),
3859         VMSTATE_UINT32(length, XHCIEvent),
3860         VMSTATE_UINT32(flags,  XHCIEvent),
3861         VMSTATE_UINT8(slotid,  XHCIEvent),
3862         VMSTATE_UINT8(epid,    XHCIEvent),
3863         VMSTATE_END_OF_LIST()
3864     }
3865 };
3866 
3867 static bool xhci_er_full(void *opaque, int version_id)
3868 {
3869     struct XHCIInterrupter *intr = opaque;
3870     return intr->er_full;
3871 }
3872 
3873 static const VMStateDescription vmstate_xhci_intr = {
3874     .name = "xhci-intr",
3875     .version_id = 1,
3876     .fields = (VMStateField[]) {
3877         /* registers */
3878         VMSTATE_UINT32(iman,          XHCIInterrupter),
3879         VMSTATE_UINT32(imod,          XHCIInterrupter),
3880         VMSTATE_UINT32(erstsz,        XHCIInterrupter),
3881         VMSTATE_UINT32(erstba_low,    XHCIInterrupter),
3882         VMSTATE_UINT32(erstba_high,   XHCIInterrupter),
3883         VMSTATE_UINT32(erdp_low,      XHCIInterrupter),
3884         VMSTATE_UINT32(erdp_high,     XHCIInterrupter),
3885 
3886         /* state */
3887         VMSTATE_BOOL(msix_used,       XHCIInterrupter),
3888         VMSTATE_BOOL(er_pcs,          XHCIInterrupter),
3889         VMSTATE_UINT64(er_start,      XHCIInterrupter),
3890         VMSTATE_UINT32(er_size,       XHCIInterrupter),
3891         VMSTATE_UINT32(er_ep_idx,     XHCIInterrupter),
3892 
3893         /* event queue (used if ring is full) */
3894         VMSTATE_BOOL(er_full,         XHCIInterrupter),
3895         VMSTATE_UINT32_TEST(ev_buffer_put, XHCIInterrupter, xhci_er_full),
3896         VMSTATE_UINT32_TEST(ev_buffer_get, XHCIInterrupter, xhci_er_full),
3897         VMSTATE_STRUCT_ARRAY_TEST(ev_buffer, XHCIInterrupter, EV_QUEUE,
3898                                   xhci_er_full, 1,
3899                                   vmstate_xhci_event, XHCIEvent),
3900 
3901         VMSTATE_END_OF_LIST()
3902     }
3903 };
3904 
3905 static const VMStateDescription vmstate_xhci = {
3906     .name = "xhci",
3907     .version_id = 1,
3908     .post_load = usb_xhci_post_load,
3909     .fields = (VMStateField[]) {
3910         VMSTATE_PCI_DEVICE(parent_obj, XHCIState),
3911         VMSTATE_MSIX(parent_obj, XHCIState),
3912 
3913         VMSTATE_STRUCT_VARRAY_UINT32(ports, XHCIState, numports, 1,
3914                                      vmstate_xhci_port, XHCIPort),
3915         VMSTATE_STRUCT_VARRAY_UINT32(slots, XHCIState, numslots, 1,
3916                                      vmstate_xhci_slot, XHCISlot),
3917         VMSTATE_STRUCT_VARRAY_UINT32(intr, XHCIState, numintrs, 1,
3918                                      vmstate_xhci_intr, XHCIInterrupter),
3919 
3920         /* Operational Registers */
3921         VMSTATE_UINT32(usbcmd,        XHCIState),
3922         VMSTATE_UINT32(usbsts,        XHCIState),
3923         VMSTATE_UINT32(dnctrl,        XHCIState),
3924         VMSTATE_UINT32(crcr_low,      XHCIState),
3925         VMSTATE_UINT32(crcr_high,     XHCIState),
3926         VMSTATE_UINT32(dcbaap_low,    XHCIState),
3927         VMSTATE_UINT32(dcbaap_high,   XHCIState),
3928         VMSTATE_UINT32(config,        XHCIState),
3929 
3930         /* Runtime Registers & state */
3931         VMSTATE_INT64(mfindex_start,  XHCIState),
3932         VMSTATE_TIMER_PTR(mfwrap_timer,   XHCIState),
3933         VMSTATE_STRUCT(cmd_ring, XHCIState, 1, vmstate_xhci_ring, XHCIRing),
3934 
3935         VMSTATE_END_OF_LIST()
3936     }
3937 };
3938 
3939 static Property xhci_properties[] = {
3940     DEFINE_PROP_ON_OFF_AUTO("msi", XHCIState, msi, ON_OFF_AUTO_AUTO),
3941     DEFINE_PROP_ON_OFF_AUTO("msix", XHCIState, msix, ON_OFF_AUTO_AUTO),
3942     DEFINE_PROP_BIT("superspeed-ports-first",
3943                     XHCIState, flags, XHCI_FLAG_SS_FIRST, true),
3944     DEFINE_PROP_BIT("force-pcie-endcap", XHCIState, flags,
3945                     XHCI_FLAG_FORCE_PCIE_ENDCAP, false),
3946     DEFINE_PROP_BIT("streams", XHCIState, flags,
3947                     XHCI_FLAG_ENABLE_STREAMS, true),
3948     DEFINE_PROP_UINT32("intrs", XHCIState, numintrs, MAXINTRS),
3949     DEFINE_PROP_UINT32("slots", XHCIState, numslots, MAXSLOTS),
3950     DEFINE_PROP_UINT32("p2",    XHCIState, numports_2, 4),
3951     DEFINE_PROP_UINT32("p3",    XHCIState, numports_3, 4),
3952     DEFINE_PROP_END_OF_LIST(),
3953 };
3954 
3955 static void xhci_class_init(ObjectClass *klass, void *data)
3956 {
3957     PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
3958     DeviceClass *dc = DEVICE_CLASS(klass);
3959 
3960     dc->vmsd    = &vmstate_xhci;
3961     dc->props   = xhci_properties;
3962     dc->reset   = xhci_reset;
3963     set_bit(DEVICE_CATEGORY_USB, dc->categories);
3964     k->realize      = usb_xhci_realize;
3965     k->exit         = usb_xhci_exit;
3966     k->vendor_id    = PCI_VENDOR_ID_NEC;
3967     k->device_id    = PCI_DEVICE_ID_NEC_UPD720200;
3968     k->class_id     = PCI_CLASS_SERIAL_USB;
3969     k->revision     = 0x03;
3970     k->is_express   = 1;
3971 }
3972 
3973 static const TypeInfo xhci_info = {
3974     .name          = TYPE_XHCI,
3975     .parent        = TYPE_PCI_DEVICE,
3976     .instance_size = sizeof(XHCIState),
3977     .class_init    = xhci_class_init,
3978 };
3979 
3980 static void xhci_register_types(void)
3981 {
3982     type_register_static(&xhci_info);
3983 }
3984 
3985 type_init(xhci_register_types)
3986