xref: /openbmc/linux/drivers/usb/host/octeon-hcd.c (revision dff03381)

// SPDX-License-Identifier: GPL-2.0
/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 2008 Cavium Networks
 *
 * Some parts of the code were originally released under BSD license:
 *
 * Copyright (c) 2003-2010 Cavium Networks (support@cavium.com). All rights
 * reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *   * Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *
 *   * Redistributions in binary form must reproduce the above
 *     copyright notice, this list of conditions and the following
 *     disclaimer in the documentation and/or other materials provided
 *     with the distribution.
 *
 *   * Neither the name of Cavium Networks nor the names of
 *     its contributors may be used to endorse or promote products
 *     derived from this software without specific prior written
 *     permission.
 *
 * This Software, including technical data, may be subject to U.S. export
 * control laws, including the U.S. Export Administration Act and its associated
 * regulations, and may be subject to export or import regulations in other
 * countries.
 *
 * TO THE MAXIMUM EXTENT PERMITTED BY LAW, THE SOFTWARE IS PROVIDED "AS IS"
 * AND WITH ALL FAULTS AND CAVIUM NETWORKS MAKES NO PROMISES, REPRESENTATIONS OR
 * WARRANTIES, EITHER EXPRESS, IMPLIED, STATUTORY, OR OTHERWISE, WITH RESPECT TO
 * THE SOFTWARE, INCLUDING ITS CONDITION, ITS CONFORMITY TO ANY REPRESENTATION
 * OR DESCRIPTION, OR THE EXISTENCE OF ANY LATENT OR PATENT DEFECTS, AND CAVIUM
 * SPECIFICALLY DISCLAIMS ALL IMPLIED (IF ANY) WARRANTIES OF TITLE,
 * MERCHANTABILITY, NONINFRINGEMENT, FITNESS FOR A PARTICULAR PURPOSE, LACK OF
 * VIRUSES, ACCURACY OR COMPLETENESS, QUIET ENJOYMENT, QUIET POSSESSION OR
 * CORRESPONDENCE TO DESCRIPTION. THE ENTIRE RISK ARISING OUT OF USE OR
 * PERFORMANCE OF THE SOFTWARE LIES WITH YOU.
 */

#include <linux/usb.h>
#include <linux/slab.h>
#include <linux/module.h>
#include <linux/usb/hcd.h>
#include <linux/prefetch.h>
#include <linux/irqdomain.h>
#include <linux/dma-mapping.h>
#include <linux/platform_device.h>
#include <linux/of.h>

#include <asm/octeon/octeon.h>

#include "octeon-hcd.h"

/**
 * enum cvmx_usb_speed - the possible USB device speeds
 *
 * @CVMX_USB_SPEED_HIGH: Device is operation at 480Mbps
 * @CVMX_USB_SPEED_FULL: Device is operation at 12Mbps
 * @CVMX_USB_SPEED_LOW:  Device is operation at 1.5Mbps
 */
enum cvmx_usb_speed {
	CVMX_USB_SPEED_HIGH = 0,
	CVMX_USB_SPEED_FULL = 1,
	CVMX_USB_SPEED_LOW = 2,
};

/**
 * enum cvmx_usb_transfer - the possible USB transfer types
 *
 * @CVMX_USB_TRANSFER_CONTROL:	   USB transfer type control for hub and status
 *				   transfers
 * @CVMX_USB_TRANSFER_ISOCHRONOUS: USB transfer type isochronous for low
 *				   priority periodic transfers
 * @CVMX_USB_TRANSFER_BULK:	   USB transfer type bulk for large low priority
 *				   transfers
 * @CVMX_USB_TRANSFER_INTERRUPT:   USB transfer type interrupt for high priority
 *				   periodic transfers
 */
enum cvmx_usb_transfer {
	CVMX_USB_TRANSFER_CONTROL = 0,
	CVMX_USB_TRANSFER_ISOCHRONOUS = 1,
	CVMX_USB_TRANSFER_BULK = 2,
	CVMX_USB_TRANSFER_INTERRUPT = 3,
};

/**
 * enum cvmx_usb_direction - the transfer directions
 *
 * @CVMX_USB_DIRECTION_OUT: Data is transferring from Octeon to the device/host
 * @CVMX_USB_DIRECTION_IN:  Data is transferring from the device/host to Octeon
 */
enum cvmx_usb_direction {
	CVMX_USB_DIRECTION_OUT,
	CVMX_USB_DIRECTION_IN,
};

/**
 * enum cvmx_usb_status - possible callback function status codes
 *
 * @CVMX_USB_STATUS_OK:		  The transaction / operation finished without
 *				  any errors
 * @CVMX_USB_STATUS_SHORT:	  FIXME: This is currently not implemented
 * @CVMX_USB_STATUS_CANCEL:	  The transaction was canceled while in flight
 *				  by a user call to cvmx_usb_cancel
 * @CVMX_USB_STATUS_ERROR:	  The transaction aborted with an unexpected
 *				  error status
 * @CVMX_USB_STATUS_STALL:	  The transaction received a USB STALL response
 *				  from the device
 * @CVMX_USB_STATUS_XACTERR:	  The transaction failed with an error from the
 *				  device even after a number of retries
 * @CVMX_USB_STATUS_DATATGLERR:	  The transaction failed with a data toggle
 *				  error even after a number of retries
 * @CVMX_USB_STATUS_BABBLEERR:	  The transaction failed with a babble error
 * @CVMX_USB_STATUS_FRAMEERR:	  The transaction failed with a frame error
 *				  even after a number of retries
 */
enum cvmx_usb_status {
	CVMX_USB_STATUS_OK,
	CVMX_USB_STATUS_SHORT,
	CVMX_USB_STATUS_CANCEL,
	CVMX_USB_STATUS_ERROR,
	CVMX_USB_STATUS_STALL,
	CVMX_USB_STATUS_XACTERR,
	CVMX_USB_STATUS_DATATGLERR,
	CVMX_USB_STATUS_BABBLEERR,
	CVMX_USB_STATUS_FRAMEERR,
};

/**
 * struct cvmx_usb_port_status - the USB port status information
 *
 * @port_enabled:	1 = Usb port is enabled, 0 = disabled
 * @port_over_current:	1 = Over current detected, 0 = Over current not
 *			detected. Octeon doesn't support over current detection.
 * @port_powered:	1 = Port power is being supplied to the device, 0 =
 *			power is off. Octeon doesn't support turning port power
 *			off.
 * @port_speed:		Current port speed.
 * @connected:		1 = A device is connected to the port, 0 = No device is
 *			connected.
 * @connect_change:	1 = Device connected state changed since the last set
 *			status call.
 */
struct cvmx_usb_port_status {
	u32 reserved			: 25;
	u32 port_enabled		: 1;
	u32 port_over_current		: 1;
	u32 port_powered		: 1;
	enum cvmx_usb_speed port_speed	: 2;
	u32 connected			: 1;
	u32 connect_change		: 1;
};

/**
 * struct cvmx_usb_iso_packet - descriptor for Isochronous packets
 *
 * @offset:	This is the offset in bytes into the main buffer where this data
 *		is stored.
 * @length:	This is the length in bytes of the data.
 * @status:	This is the status of this individual packet transfer.
 */
struct cvmx_usb_iso_packet {
	int offset;
	int length;
	enum cvmx_usb_status status;
};

/**
 * enum cvmx_usb_initialize_flags - flags used by the initialization function
 *
 * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI:    The USB port uses a 12MHz crystal
 *					      as clock source at USB_XO and
 *					      USB_XI.
 * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND:   The USB port uses 12/24/48MHz 2.5V
 *					      board clock source at USB_XO.
 *					      USB_XI should be tied to GND.
 * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK: Mask for clock speed field
 * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:    Speed of reference clock or
 *					      crystal
 * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:    Speed of reference clock
 * @CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:    Speed of reference clock
 * @CVMX_USB_INITIALIZE_FLAGS_NO_DMA:	      Disable DMA and used polled IO for
 *					      data transfer use for the USB
 */
enum cvmx_usb_initialize_flags {
	CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI		= 1 << 0,
	CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND		= 1 << 1,
	CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK	= 3 << 3,
	CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ		= 1 << 3,
	CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ		= 2 << 3,
	CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ		= 3 << 3,
	/* Bits 3-4 used to encode the clock frequency */
	CVMX_USB_INITIALIZE_FLAGS_NO_DMA		= 1 << 5,
};

/**
 * enum cvmx_usb_pipe_flags - internal flags for a pipe.
 *
 * @CVMX_USB_PIPE_FLAGS_SCHEDULED: Used internally to determine if a pipe is
 *				   actively using hardware.
 * @CVMX_USB_PIPE_FLAGS_NEED_PING: Used internally to determine if a high speed
 *				   pipe is in the ping state.
 */
enum cvmx_usb_pipe_flags {
	CVMX_USB_PIPE_FLAGS_SCHEDULED	= 1 << 17,
	CVMX_USB_PIPE_FLAGS_NEED_PING	= 1 << 18,
};

/* Maximum number of times to retry failed transactions */
#define MAX_RETRIES		3

/* Maximum number of hardware channels supported by the USB block */
#define MAX_CHANNELS		8

/*
 * The low level hardware can transfer a maximum of this number of bytes in each
 * transfer. The field is 19 bits wide
 */
#define MAX_TRANSFER_BYTES	((1 << 19) - 1)

/*
 * The low level hardware can transfer a maximum of this number of packets in
 * each transfer. The field is 10 bits wide
 */
#define MAX_TRANSFER_PACKETS	((1 << 10) - 1)

/**
 * Logical transactions may take numerous low level
 * transactions, especially when splits are concerned. This
 * enum represents all of the possible stages a transaction can
 * be in. Note that split completes are always even. This is so
 * the NAK handler can backup to the previous low level
 * transaction with a simple clearing of bit 0.
 */
enum cvmx_usb_stage {
	CVMX_USB_STAGE_NON_CONTROL,
	CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE,
	CVMX_USB_STAGE_SETUP,
	CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE,
	CVMX_USB_STAGE_DATA,
	CVMX_USB_STAGE_DATA_SPLIT_COMPLETE,
	CVMX_USB_STAGE_STATUS,
	CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE,
};

/**
 * struct cvmx_usb_transaction - describes each pending USB transaction
 *				 regardless of type. These are linked together
 *				 to form a list of pending requests for a pipe.
 *
 * @node:		List node for transactions in the pipe.
 * @type:		Type of transaction, duplicated of the pipe.
 * @flags:		State flags for this transaction.
 * @buffer:		User's physical buffer address to read/write.
 * @buffer_length:	Size of the user's buffer in bytes.
 * @control_header:	For control transactions, physical address of the 8
 *			byte standard header.
 * @iso_start_frame:	For ISO transactions, the starting frame number.
 * @iso_number_packets:	For ISO transactions, the number of packets in the
 *			request.
 * @iso_packets:	For ISO transactions, the sub packets in the request.
 * @actual_bytes:	Actual bytes transfer for this transaction.
 * @stage:		For control transactions, the current stage.
 * @urb:		URB.
 */
struct cvmx_usb_transaction {
	struct list_head node;
	enum cvmx_usb_transfer type;
	u64 buffer;
	int buffer_length;
	u64 control_header;
	int iso_start_frame;
	int iso_number_packets;
	struct cvmx_usb_iso_packet *iso_packets;
	int xfersize;
	int pktcnt;
	int retries;
	int actual_bytes;
	enum cvmx_usb_stage stage;
	struct urb *urb;
};

/**
 * struct cvmx_usb_pipe - a pipe represents a virtual connection between Octeon
 *			  and some USB device. It contains a list of pending
 *			  request to the device.
 *
 * @node:		List node for pipe list
 * @next:		Pipe after this one in the list
 * @transactions:	List of pending transactions
 * @interval:		For periodic pipes, the interval between packets in
 *			frames
 * @next_tx_frame:	The next frame this pipe is allowed to transmit on
 * @flags:		State flags for this pipe
 * @device_speed:	Speed of device connected to this pipe
 * @transfer_type:	Type of transaction supported by this pipe
 * @transfer_dir:	IN or OUT. Ignored for Control
 * @multi_count:	Max packet in a row for the device
 * @max_packet:		The device's maximum packet size in bytes
 * @device_addr:	USB device address at other end of pipe
 * @endpoint_num:	USB endpoint number at other end of pipe
 * @hub_device_addr:	Hub address this device is connected to
 * @hub_port:		Hub port this device is connected to
 * @pid_toggle:		This toggles between 0/1 on every packet send to track
 *			the data pid needed
 * @channel:		Hardware DMA channel for this pipe
 * @split_sc_frame:	The low order bits of the frame number the split
 *			complete should be sent on
 */
struct cvmx_usb_pipe {
	struct list_head node;
	struct list_head transactions;
	u64 interval;
	u64 next_tx_frame;
	enum cvmx_usb_pipe_flags flags;
	enum cvmx_usb_speed device_speed;
	enum cvmx_usb_transfer transfer_type;
	enum cvmx_usb_direction transfer_dir;
	int multi_count;
	u16 max_packet;
	u8 device_addr;
	u8 endpoint_num;
	u8 hub_device_addr;
	u8 hub_port;
	u8 pid_toggle;
	u8 channel;
	s8 split_sc_frame;
};

struct cvmx_usb_tx_fifo {
	struct {
		int channel;
		int size;
		u64 address;
	} entry[MAX_CHANNELS + 1];
	int head;
	int tail;
};

/**
 * struct octeon_hcd - the state of the USB block
 *
 * lock:		   Serialization lock.
 * init_flags:		   Flags passed to initialize.
 * index:		   Which USB block this is for.
 * idle_hardware_channels: Bit set for every idle hardware channel.
 * usbcx_hprt:		   Stored port status so we don't need to read a CSR to
 *			   determine splits.
 * pipe_for_channel:	   Map channels to pipes.
 * pipe:		   Storage for pipes.
 * indent:		   Used by debug output to indent functions.
 * port_status:		   Last port status used for change notification.
 * idle_pipes:		   List of open pipes that have no transactions.
 * active_pipes:	   Active pipes indexed by transfer type.
 * frame_number:	   Increments every SOF interrupt for time keeping.
 * active_split:	   Points to the current active split, or NULL.
 */
struct octeon_hcd {
	spinlock_t lock; /* serialization lock */
	int init_flags;
	int index;
	int idle_hardware_channels;
	union cvmx_usbcx_hprt usbcx_hprt;
	struct cvmx_usb_pipe *pipe_for_channel[MAX_CHANNELS];
	int indent;
	struct cvmx_usb_port_status port_status;
	struct list_head idle_pipes;
	struct list_head active_pipes[4];
	u64 frame_number;
	struct cvmx_usb_transaction *active_split;
	struct cvmx_usb_tx_fifo periodic;
	struct cvmx_usb_tx_fifo nonperiodic;
};

/*
 * This macro logically sets a single field in a CSR. It does the sequence
 * read, modify, and write
 */
#define USB_SET_FIELD32(address, _union, field, value)		\
	do {							\
		union _union c;					\
								\
		c.u32 = cvmx_usb_read_csr32(usb, address);	\
		c.s.field = value;				\
		cvmx_usb_write_csr32(usb, address, c.u32);	\
	} while (0)

/* Returns the IO address to push/pop stuff data from the FIFOs */
#define USB_FIFO_ADDRESS(channel, usb_index) \
	(CVMX_USBCX_GOTGCTL(usb_index) + ((channel) + 1) * 0x1000)

/**
 * struct octeon_temp_buffer - a bounce buffer for USB transfers
 * @orig_buffer: the original buffer passed by the USB stack
 * @data:	 the newly allocated temporary buffer (excluding meta-data)
 *
 * Both the DMA engine and FIFO mode will always transfer full 32-bit words. If
 * the buffer is too short, we need to allocate a temporary one, and this struct
 * represents it.
 */
struct octeon_temp_buffer {
	void *orig_buffer;
	u8 data[];
};

static inline struct usb_hcd *octeon_to_hcd(struct octeon_hcd *p)
{
	return container_of((void *)p, struct usb_hcd, hcd_priv);
}

/**
 * octeon_alloc_temp_buffer - allocate a temporary buffer for USB transfer
 *                            (if needed)
 * @urb:	URB.
 * @mem_flags:	Memory allocation flags.
 *
 * This function allocates a temporary bounce buffer whenever it's needed
 * due to HW limitations.
 */
static int octeon_alloc_temp_buffer(struct urb *urb, gfp_t mem_flags)
{
	struct octeon_temp_buffer *temp;

	if (urb->num_sgs || urb->sg ||
	    (urb->transfer_flags & URB_NO_TRANSFER_DMA_MAP) ||
	    !(urb->transfer_buffer_length % sizeof(u32)))
		return 0;

	temp = kmalloc(ALIGN(urb->transfer_buffer_length, sizeof(u32)) +
		       sizeof(*temp), mem_flags);
	if (!temp)
		return -ENOMEM;

	temp->orig_buffer = urb->transfer_buffer;
	if (usb_urb_dir_out(urb))
		memcpy(temp->data, urb->transfer_buffer,
		       urb->transfer_buffer_length);
	urb->transfer_buffer = temp->data;
	urb->transfer_flags |= URB_ALIGNED_TEMP_BUFFER;

	return 0;
}

/**
 * octeon_free_temp_buffer - free a temporary buffer used by USB transfers.
 * @urb: URB.
 *
 * Frees a buffer allocated by octeon_alloc_temp_buffer().
 */
static void octeon_free_temp_buffer(struct urb *urb)
{
	struct octeon_temp_buffer *temp;
	size_t length;

	if (!(urb->transfer_flags & URB_ALIGNED_TEMP_BUFFER))
		return;

	temp = container_of(urb->transfer_buffer, struct octeon_temp_buffer,
			    data);
	if (usb_urb_dir_in(urb)) {
		if (usb_pipeisoc(urb->pipe))
			length = urb->transfer_buffer_length;
		else
			length = urb->actual_length;

		memcpy(temp->orig_buffer, urb->transfer_buffer, length);
	}
	urb->transfer_buffer = temp->orig_buffer;
	urb->transfer_flags &= ~URB_ALIGNED_TEMP_BUFFER;
	kfree(temp);
}

/**
 * octeon_map_urb_for_dma - Octeon-specific map_urb_for_dma().
 * @hcd:	USB HCD structure.
 * @urb:	URB.
 * @mem_flags:	Memory allocation flags.
 */
static int octeon_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
				  gfp_t mem_flags)
{
	int ret;

	ret = octeon_alloc_temp_buffer(urb, mem_flags);
	if (ret)
		return ret;

	ret = usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
	if (ret)
		octeon_free_temp_buffer(urb);

	return ret;
}

/**
 * octeon_unmap_urb_for_dma - Octeon-specific unmap_urb_for_dma()
 * @hcd:	USB HCD structure.
 * @urb:	URB.
 */
static void octeon_unmap_urb_for_dma(struct usb_hcd *hcd, struct urb *urb)
{
	usb_hcd_unmap_urb_for_dma(hcd, urb);
	octeon_free_temp_buffer(urb);
}

/**
 * Read a USB 32bit CSR. It performs the necessary address swizzle
 * for 32bit CSRs and logs the value in a readable format if
 * debugging is on.
 *
 * @usb:     USB block this access is for
 * @address: 64bit address to read
 *
 * Returns: Result of the read
 */
static inline u32 cvmx_usb_read_csr32(struct octeon_hcd *usb, u64 address)
{
	return cvmx_read64_uint32(address ^ 4);
}

/**
 * Write a USB 32bit CSR. It performs the necessary address
 * swizzle for 32bit CSRs and logs the value in a readable format
 * if debugging is on.
 *
 * @usb:     USB block this access is for
 * @address: 64bit address to write
 * @value:   Value to write
 */
static inline void cvmx_usb_write_csr32(struct octeon_hcd *usb,
					u64 address, u32 value)
{
	cvmx_write64_uint32(address ^ 4, value);
	cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
}

/**
 * Return non zero if this pipe connects to a non HIGH speed
 * device through a high speed hub.
 *
 * @usb:    USB block this access is for
 * @pipe:   Pipe to check
 *
 * Returns: Non zero if we need to do split transactions
 */
static inline int cvmx_usb_pipe_needs_split(struct octeon_hcd *usb,
					    struct cvmx_usb_pipe *pipe)
{
	return pipe->device_speed != CVMX_USB_SPEED_HIGH &&
	       usb->usbcx_hprt.s.prtspd == CVMX_USB_SPEED_HIGH;
}

/**
 * Trivial utility function to return the correct PID for a pipe
 *
 * @pipe:   pipe to check
 *
 * Returns: PID for pipe
 */
static inline int cvmx_usb_get_data_pid(struct cvmx_usb_pipe *pipe)
{
	if (pipe->pid_toggle)
		return 2; /* Data1 */
	return 0; /* Data0 */
}

/* Loops through register until txfflsh or rxfflsh become zero.*/
static int cvmx_wait_tx_rx(struct octeon_hcd *usb, int fflsh_type)
{
	int result;
	u64 address = CVMX_USBCX_GRSTCTL(usb->index);
	u64 done = cvmx_get_cycle() + 100 *
		   (u64)octeon_get_clock_rate / 1000000;
	union cvmx_usbcx_grstctl c;

	while (1) {
		c.u32 = cvmx_usb_read_csr32(usb, address);
		if (fflsh_type == 0 && c.s.txfflsh == 0) {
			result = 0;
			break;
		} else if (fflsh_type == 1 && c.s.rxfflsh == 0) {
			result = 0;
			break;
		} else if (cvmx_get_cycle() > done) {
			result = -1;
			break;
		}

		__delay(100);
	}
	return result;
}

static void cvmx_fifo_setup(struct octeon_hcd *usb)
{
	union cvmx_usbcx_ghwcfg3 usbcx_ghwcfg3;
	union cvmx_usbcx_gnptxfsiz npsiz;
	union cvmx_usbcx_hptxfsiz psiz;

	usbcx_ghwcfg3.u32 = cvmx_usb_read_csr32(usb,
						CVMX_USBCX_GHWCFG3(usb->index));

	/*
	 * Program the USBC_GRXFSIZ register to select the size of the receive
	 * FIFO (25%).
	 */
	USB_SET_FIELD32(CVMX_USBCX_GRXFSIZ(usb->index), cvmx_usbcx_grxfsiz,
			rxfdep, usbcx_ghwcfg3.s.dfifodepth / 4);

	/*
	 * Program the USBC_GNPTXFSIZ register to select the size and the start
	 * address of the non-periodic transmit FIFO for nonperiodic
	 * transactions (50%).
	 */
	npsiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index));
	npsiz.s.nptxfdep = usbcx_ghwcfg3.s.dfifodepth / 2;
	npsiz.s.nptxfstaddr = usbcx_ghwcfg3.s.dfifodepth / 4;
	cvmx_usb_write_csr32(usb, CVMX_USBCX_GNPTXFSIZ(usb->index), npsiz.u32);

	/*
	 * Program the USBC_HPTXFSIZ register to select the size and start
	 * address of the periodic transmit FIFO for periodic transactions
	 * (25%).
	 */
	psiz.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index));
	psiz.s.ptxfsize = usbcx_ghwcfg3.s.dfifodepth / 4;
	psiz.s.ptxfstaddr = 3 * usbcx_ghwcfg3.s.dfifodepth / 4;
	cvmx_usb_write_csr32(usb, CVMX_USBCX_HPTXFSIZ(usb->index), psiz.u32);

	/* Flush all FIFOs */
	USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
			cvmx_usbcx_grstctl, txfnum, 0x10);
	USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
			cvmx_usbcx_grstctl, txfflsh, 1);
	cvmx_wait_tx_rx(usb, 0);
	USB_SET_FIELD32(CVMX_USBCX_GRSTCTL(usb->index),
			cvmx_usbcx_grstctl, rxfflsh, 1);
	cvmx_wait_tx_rx(usb, 1);
}

/**
 * Shutdown a USB port after a call to cvmx_usb_initialize().
 * The port should be disabled with all pipes closed when this
 * function is called.
 *
 * @usb: USB device state populated by cvmx_usb_initialize().
 *
 * Returns: 0 or a negative error code.
 */
static int cvmx_usb_shutdown(struct octeon_hcd *usb)
{
	union cvmx_usbnx_clk_ctl usbn_clk_ctl;

	/* Make sure all pipes are closed */
	if (!list_empty(&usb->idle_pipes) ||
	    !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_ISOCHRONOUS]) ||
	    !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_INTERRUPT]) ||
	    !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_CONTROL]) ||
	    !list_empty(&usb->active_pipes[CVMX_USB_TRANSFER_BULK]))
		return -EBUSY;

	/* Disable the clocks and put them in power on reset */
	usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
	usbn_clk_ctl.s.enable = 1;
	usbn_clk_ctl.s.por = 1;
	usbn_clk_ctl.s.hclk_rst = 1;
	usbn_clk_ctl.s.prst = 0;
	usbn_clk_ctl.s.hrst = 0;
	cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
	return 0;
}

/**
 * Initialize a USB port for use. This must be called before any
 * other access to the Octeon USB port is made. The port starts
 * off in the disabled state.
 *
 * @dev:	 Pointer to struct device for logging purposes.
 * @usb:	 Pointer to struct octeon_hcd.
 *
 * Returns: 0 or a negative error code.
 */
static int cvmx_usb_initialize(struct device *dev,
			       struct octeon_hcd *usb)
{
	int channel;
	int divisor;
	int retries = 0;
	union cvmx_usbcx_hcfg usbcx_hcfg;
	union cvmx_usbnx_clk_ctl usbn_clk_ctl;
	union cvmx_usbcx_gintsts usbc_gintsts;
	union cvmx_usbcx_gahbcfg usbcx_gahbcfg;
	union cvmx_usbcx_gintmsk usbcx_gintmsk;
	union cvmx_usbcx_gusbcfg usbcx_gusbcfg;
	union cvmx_usbnx_usbp_ctl_status usbn_usbp_ctl_status;

retry:
	/*
	 * Power On Reset and PHY Initialization
	 *
	 * 1. Wait for DCOK to assert (nothing to do)
	 *
	 * 2a. Write USBN0/1_CLK_CTL[POR] = 1 and
	 *     USBN0/1_CLK_CTL[HRST,PRST,HCLK_RST] = 0
	 */
	usbn_clk_ctl.u64 = cvmx_read64_uint64(CVMX_USBNX_CLK_CTL(usb->index));
	usbn_clk_ctl.s.por = 1;
	usbn_clk_ctl.s.hrst = 0;
	usbn_clk_ctl.s.prst = 0;
	usbn_clk_ctl.s.hclk_rst = 0;
	usbn_clk_ctl.s.enable = 0;
	/*
	 * 2b. Select the USB reference clock/crystal parameters by writing
	 *     appropriate values to USBN0/1_CLK_CTL[P_C_SEL, P_RTYPE, P_COM_ON]
	 */
	if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND) {
		/*
		 * The USB port uses 12/24/48MHz 2.5V board clock
		 * source at USB_XO. USB_XI should be tied to GND.
		 * Most Octeon evaluation boards require this setting
		 */
		if (OCTEON_IS_MODEL(OCTEON_CN3XXX) ||
		    OCTEON_IS_MODEL(OCTEON_CN56XX) ||
		    OCTEON_IS_MODEL(OCTEON_CN50XX))
			/* From CN56XX,CN50XX,CN31XX,CN30XX manuals */
			usbn_clk_ctl.s.p_rtype = 2; /* p_rclk=1 & p_xenbn=0 */
		else
			/* From CN52XX manual */
			usbn_clk_ctl.s.p_rtype = 1;

		switch (usb->init_flags &
			CVMX_USB_INITIALIZE_FLAGS_CLOCK_MHZ_MASK) {
		case CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ:
			usbn_clk_ctl.s.p_c_sel = 0;
			break;
		case CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ:
			usbn_clk_ctl.s.p_c_sel = 1;
			break;
		case CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ:
			usbn_clk_ctl.s.p_c_sel = 2;
			break;
		}
	} else {
		/*
		 * The USB port uses a 12MHz crystal as clock source
		 * at USB_XO and USB_XI
		 */
		if (OCTEON_IS_MODEL(OCTEON_CN3XXX))
			/* From CN31XX,CN30XX manual */
			usbn_clk_ctl.s.p_rtype = 3; /* p_rclk=1 & p_xenbn=1 */
		else
			/* From CN56XX,CN52XX,CN50XX manuals. */
			usbn_clk_ctl.s.p_rtype = 0;

		usbn_clk_ctl.s.p_c_sel = 0;
	}
	/*
	 * 2c. Select the HCLK via writing USBN0/1_CLK_CTL[DIVIDE, DIVIDE2] and
	 *     setting USBN0/1_CLK_CTL[ENABLE] = 1. Divide the core clock down
	 *     such that USB is as close as possible to 125Mhz
	 */
	divisor = DIV_ROUND_UP(octeon_get_clock_rate(), 125000000);
	/* Lower than 4 doesn't seem to work properly */
	if (divisor < 4)
		divisor = 4;
	usbn_clk_ctl.s.divide = divisor;
	usbn_clk_ctl.s.divide2 = 0;
	cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);

	/* 2d. Write USBN0/1_CLK_CTL[HCLK_RST] = 1 */
	usbn_clk_ctl.s.hclk_rst = 1;
	cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
	/* 2e.  Wait 64 core-clock cycles for HCLK to stabilize */
	__delay(64);
	/*
	 * 3. Program the power-on reset field in the USBN clock-control
	 *    register:
	 *    USBN_CLK_CTL[POR] = 0
	 */
	usbn_clk_ctl.s.por = 0;
	cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
	/* 4. Wait 1 ms for PHY clock to start */
	mdelay(1);
	/*
	 * 5. Program the Reset input from automatic test equipment field in the
	 *    USBP control and status register:
	 *    USBN_USBP_CTL_STATUS[ATE_RESET] = 1
	 */
	usbn_usbp_ctl_status.u64 =
		cvmx_read64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index));
	usbn_usbp_ctl_status.s.ate_reset = 1;
	cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
			    usbn_usbp_ctl_status.u64);
	/* 6. Wait 10 cycles */
	__delay(10);
	/*
	 * 7. Clear ATE_RESET field in the USBN clock-control register:
	 *    USBN_USBP_CTL_STATUS[ATE_RESET] = 0
	 */
	usbn_usbp_ctl_status.s.ate_reset = 0;
	cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
			    usbn_usbp_ctl_status.u64);
	/*
	 * 8. Program the PHY reset field in the USBN clock-control register:
	 *    USBN_CLK_CTL[PRST] = 1
	 */
	usbn_clk_ctl.s.prst = 1;
	cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
	/*
	 * 9. Program the USBP control and status register to select host or
	 *    device mode. USBN_USBP_CTL_STATUS[HST_MODE] = 0 for host, = 1 for
	 *    device
	 */
	usbn_usbp_ctl_status.s.hst_mode = 0;
	cvmx_write64_uint64(CVMX_USBNX_USBP_CTL_STATUS(usb->index),
			    usbn_usbp_ctl_status.u64);
	/* 10. Wait 1 us */
	udelay(1);
	/*
	 * 11. Program the hreset_n field in the USBN clock-control register:
	 *     USBN_CLK_CTL[HRST] = 1
	 */
	usbn_clk_ctl.s.hrst = 1;
	cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
	/* 12. Proceed to USB core initialization */
	usbn_clk_ctl.s.enable = 1;
	cvmx_write64_uint64(CVMX_USBNX_CLK_CTL(usb->index), usbn_clk_ctl.u64);
	udelay(1);

	/*
	 * USB Core Initialization
	 *
	 * 1. Read USBC_GHWCFG1, USBC_GHWCFG2, USBC_GHWCFG3, USBC_GHWCFG4 to
	 *    determine USB core configuration parameters.
	 *
	 *    Nothing needed
	 *
	 * 2. Program the following fields in the global AHB configuration
	 *    register (USBC_GAHBCFG)
	 *    DMA mode, USBC_GAHBCFG[DMAEn]: 1 = DMA mode, 0 = slave mode
	 *    Burst length, USBC_GAHBCFG[HBSTLEN] = 0
	 *    Nonperiodic TxFIFO empty level (slave mode only),
	 *    USBC_GAHBCFG[NPTXFEMPLVL]
	 *    Periodic TxFIFO empty level (slave mode only),
	 *    USBC_GAHBCFG[PTXFEMPLVL]
	 *    Global interrupt mask, USBC_GAHBCFG[GLBLINTRMSK] = 1
	 */
	usbcx_gahbcfg.u32 = 0;
	usbcx_gahbcfg.s.dmaen = !(usb->init_flags &
				  CVMX_USB_INITIALIZE_FLAGS_NO_DMA);
	usbcx_gahbcfg.s.hbstlen = 0;
	usbcx_gahbcfg.s.nptxfemplvl = 1;
	usbcx_gahbcfg.s.ptxfemplvl = 1;
	usbcx_gahbcfg.s.glblintrmsk = 1;
	cvmx_usb_write_csr32(usb, CVMX_USBCX_GAHBCFG(usb->index),
			     usbcx_gahbcfg.u32);

	/*
	 * 3. Program the following fields in USBC_GUSBCFG register.
	 *    HS/FS timeout calibration, USBC_GUSBCFG[TOUTCAL] = 0
	 *    ULPI DDR select, USBC_GUSBCFG[DDRSEL] = 0
	 *    USB turnaround time, USBC_GUSBCFG[USBTRDTIM] = 0x5
	 *    PHY low-power clock select, USBC_GUSBCFG[PHYLPWRCLKSEL] = 0
	 */
	usbcx_gusbcfg.u32 = cvmx_usb_read_csr32(usb,
						CVMX_USBCX_GUSBCFG(usb->index));
	usbcx_gusbcfg.s.toutcal = 0;
	usbcx_gusbcfg.s.ddrsel = 0;
	usbcx_gusbcfg.s.usbtrdtim = 0x5;
	usbcx_gusbcfg.s.phylpwrclksel = 0;
	cvmx_usb_write_csr32(usb, CVMX_USBCX_GUSBCFG(usb->index),
			     usbcx_gusbcfg.u32);

	/*
	 * 4. The software must unmask the following bits in the USBC_GINTMSK
	 *    register.
	 *    OTG interrupt mask, USBC_GINTMSK[OTGINTMSK] = 1
	 *    Mode mismatch interrupt mask, USBC_GINTMSK[MODEMISMSK] = 1
	 */
	usbcx_gintmsk.u32 = cvmx_usb_read_csr32(usb,
						CVMX_USBCX_GINTMSK(usb->index));
	usbcx_gintmsk.s.otgintmsk = 1;
	usbcx_gintmsk.s.modemismsk = 1;
	usbcx_gintmsk.s.hchintmsk = 1;
	usbcx_gintmsk.s.sofmsk = 0;
	/* We need RX FIFO interrupts if we don't have DMA */
	if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
		usbcx_gintmsk.s.rxflvlmsk = 1;
	cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTMSK(usb->index),
			     usbcx_gintmsk.u32);

	/*
	 * Disable all channel interrupts. We'll enable them per channel later.
	 */
	for (channel = 0; channel < 8; channel++)
		cvmx_usb_write_csr32(usb,
				     CVMX_USBCX_HCINTMSKX(channel, usb->index),
				     0);

	/*
	 * Host Port Initialization
	 *
	 * 1. Program the host-port interrupt-mask field to unmask,
	 *    USBC_GINTMSK[PRTINT] = 1
	 */
	USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
			cvmx_usbcx_gintmsk, prtintmsk, 1);
	USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
			cvmx_usbcx_gintmsk, disconnintmsk, 1);

	/*
	 * 2. Program the USBC_HCFG register to select full-speed host
	 *    or high-speed host.
	 */
	usbcx_hcfg.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HCFG(usb->index));
	usbcx_hcfg.s.fslssupp = 0;
	usbcx_hcfg.s.fslspclksel = 0;
	cvmx_usb_write_csr32(usb, CVMX_USBCX_HCFG(usb->index), usbcx_hcfg.u32);

	cvmx_fifo_setup(usb);

	/*
	 * If the controller is getting port events right after the reset, it
	 * means the initialization failed. Try resetting the controller again
	 * in such case. This is seen to happen after cold boot on DSR-1000N.
	 */
	usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
					       CVMX_USBCX_GINTSTS(usb->index));
	cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
			     usbc_gintsts.u32);
	dev_dbg(dev, "gintsts after reset: 0x%x\n", (int)usbc_gintsts.u32);
	if (!usbc_gintsts.s.disconnint && !usbc_gintsts.s.prtint)
		return 0;
	if (retries++ >= 5)
		return -EAGAIN;
	dev_info(dev, "controller reset failed (gintsts=0x%x) - retrying\n",
		 (int)usbc_gintsts.u32);
	msleep(50);
	cvmx_usb_shutdown(usb);
	msleep(50);
	goto retry;
}

/**
 * Reset a USB port. After this call succeeds, the USB port is
 * online and servicing requests.
 *
 * @usb: USB device state populated by cvmx_usb_initialize().
 */
static void cvmx_usb_reset_port(struct octeon_hcd *usb)
{
	usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
						  CVMX_USBCX_HPRT(usb->index));

	/* Program the port reset bit to start the reset process */
	USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
			prtrst, 1);

	/*
	 * Wait at least 50ms (high speed), or 10ms (full speed) for the reset
	 * process to complete.
	 */
	mdelay(50);

	/* Program the port reset bit to 0, USBC_HPRT[PRTRST] = 0 */
	USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
			prtrst, 0);

	/*
	 * Read the port speed field to get the enumerated speed,
	 * USBC_HPRT[PRTSPD].
	 */
	usb->usbcx_hprt.u32 = cvmx_usb_read_csr32(usb,
						  CVMX_USBCX_HPRT(usb->index));
}

/**
 * Disable a USB port. After this call the USB port will not
 * generate data transfers and will not generate events.
 * Transactions in process will fail and call their
 * associated callbacks.
 *
 * @usb: USB device state populated by cvmx_usb_initialize().
 *
 * Returns: 0 or a negative error code.
 */
static int cvmx_usb_disable(struct octeon_hcd *usb)
{
	/* Disable the port */
	USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index), cvmx_usbcx_hprt,
			prtena, 1);
	return 0;
}

/**
 * Get the current state of the USB port. Use this call to
 * determine if the usb port has anything connected, is enabled,
 * or has some sort of error condition. The return value of this
 * call has "changed" bits to signal of the value of some fields
 * have changed between calls.
 *
 * @usb: USB device state populated by cvmx_usb_initialize().
 *
 * Returns: Port status information
 */
static struct cvmx_usb_port_status cvmx_usb_get_status(struct octeon_hcd *usb)
{
	union cvmx_usbcx_hprt usbc_hprt;
	struct cvmx_usb_port_status result;

	memset(&result, 0, sizeof(result));

	usbc_hprt.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
	result.port_enabled = usbc_hprt.s.prtena;
	result.port_over_current = usbc_hprt.s.prtovrcurract;
	result.port_powered = usbc_hprt.s.prtpwr;
	result.port_speed = usbc_hprt.s.prtspd;
	result.connected = usbc_hprt.s.prtconnsts;
	result.connect_change =
		result.connected != usb->port_status.connected;

	return result;
}

/**
 * Open a virtual pipe between the host and a USB device. A pipe
 * must be opened before data can be transferred between a device
 * and Octeon.
 *
 * @usb:	     USB device state populated by cvmx_usb_initialize().
 * @device_addr:
 *		     USB device address to open the pipe to
 *		     (0-127).
 * @endpoint_num:
 *		     USB endpoint number to open the pipe to
 *		     (0-15).
 * @device_speed:
 *		     The speed of the device the pipe is going
 *		     to. This must match the device's speed,
 *		     which may be different than the port speed.
 * @max_packet:	     The maximum packet length the device can
 *		     transmit/receive (low speed=0-8, full
 *		     speed=0-1023, high speed=0-1024). This value
 *		     comes from the standard endpoint descriptor
 *		     field wMaxPacketSize bits <10:0>.
 * @transfer_type:
 *		     The type of transfer this pipe is for.
 * @transfer_dir:
 *		     The direction the pipe is in. This is not
 *		     used for control pipes.
 * @interval:	     For ISOCHRONOUS and INTERRUPT transfers,
 *		     this is how often the transfer is scheduled
 *		     for. All other transfers should specify
 *		     zero. The units are in frames (8000/sec at
 *		     high speed, 1000/sec for full speed).
 * @multi_count:
 *		     For high speed devices, this is the maximum
 *		     allowed number of packet per microframe.
 *		     Specify zero for non high speed devices. This
 *		     value comes from the standard endpoint descriptor
 *		     field wMaxPacketSize bits <12:11>.
 * @hub_device_addr:
 *		     Hub device address this device is connected
 *		     to. Devices connected directly to Octeon
 *		     use zero. This is only used when the device
 *		     is full/low speed behind a high speed hub.
 *		     The address will be of the high speed hub,
 *		     not and full speed hubs after it.
 * @hub_port:	     Which port on the hub the device is
 *		     connected. Use zero for devices connected
 *		     directly to Octeon. Like hub_device_addr,
 *		     this is only used for full/low speed
 *		     devices behind a high speed hub.
 *
 * Returns: A non-NULL value is a pipe. NULL means an error.
 */
static struct cvmx_usb_pipe *cvmx_usb_open_pipe(struct octeon_hcd *usb,
						int device_addr,
						int endpoint_num,
						enum cvmx_usb_speed
							device_speed,
						int max_packet,
						enum cvmx_usb_transfer
							transfer_type,
						enum cvmx_usb_direction
							transfer_dir,
						int interval, int multi_count,
						int hub_device_addr,
						int hub_port)
{
	struct cvmx_usb_pipe *pipe;

	pipe = kzalloc(sizeof(*pipe), GFP_ATOMIC);
	if (!pipe)
		return NULL;
	if ((device_speed == CVMX_USB_SPEED_HIGH) &&
	    (transfer_dir == CVMX_USB_DIRECTION_OUT) &&
	    (transfer_type == CVMX_USB_TRANSFER_BULK))
		pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
	pipe->device_addr = device_addr;
	pipe->endpoint_num = endpoint_num;
	pipe->device_speed = device_speed;
	pipe->max_packet = max_packet;
	pipe->transfer_type = transfer_type;
	pipe->transfer_dir = transfer_dir;
	INIT_LIST_HEAD(&pipe->transactions);

	/*
	 * All pipes use interval to rate limit NAK processing. Force an
	 * interval if one wasn't supplied
	 */
	if (!interval)
		interval = 1;
	if (cvmx_usb_pipe_needs_split(usb, pipe)) {
		pipe->interval = interval * 8;
		/* Force start splits to be schedule on uFrame 0 */
		pipe->next_tx_frame = ((usb->frame_number + 7) & ~7) +
					pipe->interval;
	} else {
		pipe->interval = interval;
		pipe->next_tx_frame = usb->frame_number + pipe->interval;
	}
	pipe->multi_count = multi_count;
	pipe->hub_device_addr = hub_device_addr;
	pipe->hub_port = hub_port;
	pipe->pid_toggle = 0;
	pipe->split_sc_frame = -1;
	list_add_tail(&pipe->node, &usb->idle_pipes);

	/*
	 * We don't need to tell the hardware about this pipe yet since
	 * it doesn't have any submitted requests
	 */

	return pipe;
}

/**
 * Poll the RX FIFOs and remove data as needed. This function is only used
 * in non DMA mode. It is very important that this function be called quickly
 * enough to prevent FIFO overflow.
 *
 * @usb:	USB device state populated by cvmx_usb_initialize().
 */
static void cvmx_usb_poll_rx_fifo(struct octeon_hcd *usb)
{
	union cvmx_usbcx_grxstsph rx_status;
	int channel;
	int bytes;
	u64 address;
	u32 *ptr;

	rx_status.u32 = cvmx_usb_read_csr32(usb,
					    CVMX_USBCX_GRXSTSPH(usb->index));
	/* Only read data if IN data is there */
	if (rx_status.s.pktsts != 2)
		return;
	/* Check if no data is available */
	if (!rx_status.s.bcnt)
		return;

	channel = rx_status.s.chnum;
	bytes = rx_status.s.bcnt;
	if (!bytes)
		return;

	/* Get where the DMA engine would have written this data */
	address = cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) +
				     channel * 8);

	ptr = cvmx_phys_to_ptr(address);
	cvmx_write64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index) + channel * 8,
			    address + bytes);

	/* Loop writing the FIFO data for this packet into memory */
	while (bytes > 0) {
		*ptr++ = cvmx_usb_read_csr32(usb,
					USB_FIFO_ADDRESS(channel, usb->index));
		bytes -= 4;
	}
	CVMX_SYNCW;
}

/**
 * Fill the TX hardware fifo with data out of the software
 * fifos
 *
 * @usb:	    USB device state populated by cvmx_usb_initialize().
 * @fifo:	    Software fifo to use
 * @available:	    Amount of space in the hardware fifo
 *
 * Returns: Non zero if the hardware fifo was too small and needs
 *	    to be serviced again.
 */
static int cvmx_usb_fill_tx_hw(struct octeon_hcd *usb,
			       struct cvmx_usb_tx_fifo *fifo, int available)
{
	/*
	 * We're done either when there isn't anymore space or the software FIFO
	 * is empty
	 */
	while (available && (fifo->head != fifo->tail)) {
		int i = fifo->tail;
		const u32 *ptr = cvmx_phys_to_ptr(fifo->entry[i].address);
		u64 csr_address = USB_FIFO_ADDRESS(fifo->entry[i].channel,
						   usb->index) ^ 4;
		int words = available;

		/* Limit the amount of data to what the SW fifo has */
		if (fifo->entry[i].size <= available) {
			words = fifo->entry[i].size;
			fifo->tail++;
			if (fifo->tail > MAX_CHANNELS)
				fifo->tail = 0;
		}

		/* Update the next locations and counts */
		available -= words;
		fifo->entry[i].address += words * 4;
		fifo->entry[i].size -= words;

		/*
		 * Write the HW fifo data. The read every three writes is due
		 * to an errata on CN3XXX chips
		 */
		while (words > 3) {
			cvmx_write64_uint32(csr_address, *ptr++);
			cvmx_write64_uint32(csr_address, *ptr++);
			cvmx_write64_uint32(csr_address, *ptr++);
			cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
			words -= 3;
		}
		cvmx_write64_uint32(csr_address, *ptr++);
		if (--words) {
			cvmx_write64_uint32(csr_address, *ptr++);
			if (--words)
				cvmx_write64_uint32(csr_address, *ptr++);
		}
		cvmx_read64_uint64(CVMX_USBNX_DMA0_INB_CHN0(usb->index));
	}
	return fifo->head != fifo->tail;
}

/**
 * Check the hardware FIFOs and fill them as needed
 *
 * @usb:	USB device state populated by cvmx_usb_initialize().
 */
static void cvmx_usb_poll_tx_fifo(struct octeon_hcd *usb)
{
	if (usb->periodic.head != usb->periodic.tail) {
		union cvmx_usbcx_hptxsts tx_status;

		tx_status.u32 = cvmx_usb_read_csr32(usb,
						    CVMX_USBCX_HPTXSTS(usb->index));
		if (cvmx_usb_fill_tx_hw(usb, &usb->periodic,
					tx_status.s.ptxfspcavail))
			USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
					cvmx_usbcx_gintmsk, ptxfempmsk, 1);
		else
			USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
					cvmx_usbcx_gintmsk, ptxfempmsk, 0);
	}

	if (usb->nonperiodic.head != usb->nonperiodic.tail) {
		union cvmx_usbcx_gnptxsts tx_status;

		tx_status.u32 = cvmx_usb_read_csr32(usb,
						    CVMX_USBCX_GNPTXSTS(usb->index));
		if (cvmx_usb_fill_tx_hw(usb, &usb->nonperiodic,
					tx_status.s.nptxfspcavail))
			USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
					cvmx_usbcx_gintmsk, nptxfempmsk, 1);
		else
			USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
					cvmx_usbcx_gintmsk, nptxfempmsk, 0);
	}
}

/**
 * Fill the TX FIFO with an outgoing packet
 *
 * @usb:	  USB device state populated by cvmx_usb_initialize().
 * @channel:	  Channel number to get packet from
 */
static void cvmx_usb_fill_tx_fifo(struct octeon_hcd *usb, int channel)
{
	union cvmx_usbcx_hccharx hcchar;
	union cvmx_usbcx_hcspltx usbc_hcsplt;
	union cvmx_usbcx_hctsizx usbc_hctsiz;
	struct cvmx_usb_tx_fifo *fifo;

	/* We only need to fill data on outbound channels */
	hcchar.u32 = cvmx_usb_read_csr32(usb,
					 CVMX_USBCX_HCCHARX(channel, usb->index));
	if (hcchar.s.epdir != CVMX_USB_DIRECTION_OUT)
		return;

	/* OUT Splits only have data on the start and not the complete */
	usbc_hcsplt.u32 = cvmx_usb_read_csr32(usb,
					      CVMX_USBCX_HCSPLTX(channel, usb->index));
	if (usbc_hcsplt.s.spltena && usbc_hcsplt.s.compsplt)
		return;

	/*
	 * Find out how many bytes we need to fill and convert it into 32bit
	 * words.
	 */
	usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
					      CVMX_USBCX_HCTSIZX(channel, usb->index));
	if (!usbc_hctsiz.s.xfersize)
		return;

	if ((hcchar.s.eptype == CVMX_USB_TRANSFER_INTERRUPT) ||
	    (hcchar.s.eptype == CVMX_USB_TRANSFER_ISOCHRONOUS))
		fifo = &usb->periodic;
	else
		fifo = &usb->nonperiodic;

	fifo->entry[fifo->head].channel = channel;
	fifo->entry[fifo->head].address =
		cvmx_read64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
				   channel * 8);
	fifo->entry[fifo->head].size = (usbc_hctsiz.s.xfersize + 3) >> 2;
	fifo->head++;
	if (fifo->head > MAX_CHANNELS)
		fifo->head = 0;

	cvmx_usb_poll_tx_fifo(usb);
}

/**
 * Perform channel specific setup for Control transactions. All
 * the generic stuff will already have been done in cvmx_usb_start_channel().
 *
 * @usb:	  USB device state populated by cvmx_usb_initialize().
 * @channel:	  Channel to setup
 * @pipe:	  Pipe for control transaction
 */
static void cvmx_usb_start_channel_control(struct octeon_hcd *usb,
					   int channel,
					   struct cvmx_usb_pipe *pipe)
{
	struct usb_hcd *hcd = octeon_to_hcd(usb);
	struct device *dev = hcd->self.controller;
	struct cvmx_usb_transaction *transaction =
		list_first_entry(&pipe->transactions, typeof(*transaction),
				 node);
	struct usb_ctrlrequest *header =
		cvmx_phys_to_ptr(transaction->control_header);
	int bytes_to_transfer = transaction->buffer_length -
		transaction->actual_bytes;
	int packets_to_transfer;
	union cvmx_usbcx_hctsizx usbc_hctsiz;

	usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
					      CVMX_USBCX_HCTSIZX(channel, usb->index));

	switch (transaction->stage) {
	case CVMX_USB_STAGE_NON_CONTROL:
	case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
		dev_err(dev, "%s: ERROR - Non control stage\n", __func__);
		break;
	case CVMX_USB_STAGE_SETUP:
		usbc_hctsiz.s.pid = 3; /* Setup */
		bytes_to_transfer = sizeof(*header);
		/* All Control operations start with a setup going OUT */
		USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
				cvmx_usbcx_hccharx, epdir,
				CVMX_USB_DIRECTION_OUT);
		/*
		 * Setup send the control header instead of the buffer data. The
		 * buffer data will be used in the next stage
		 */
		cvmx_write64_uint64(CVMX_USBNX_DMA0_OUTB_CHN0(usb->index) +
					channel * 8,
				    transaction->control_header);
		break;
	case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
		usbc_hctsiz.s.pid = 3; /* Setup */
		bytes_to_transfer = 0;
		/* All Control operations start with a setup going OUT */
		USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
				cvmx_usbcx_hccharx, epdir,
				CVMX_USB_DIRECTION_OUT);

		USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
				cvmx_usbcx_hcspltx, compsplt, 1);
		break;
	case CVMX_USB_STAGE_DATA:
		usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
		if (cvmx_usb_pipe_needs_split(usb, pipe)) {
			if (header->bRequestType & USB_DIR_IN)
				bytes_to_transfer = 0;
			else if (bytes_to_transfer > pipe->max_packet)
				bytes_to_transfer = pipe->max_packet;
		}
		USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
				cvmx_usbcx_hccharx, epdir,
				((header->bRequestType & USB_DIR_IN) ?
					CVMX_USB_DIRECTION_IN :
					CVMX_USB_DIRECTION_OUT));
		break;
	case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
		usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
		if (!(header->bRequestType & USB_DIR_IN))
			bytes_to_transfer = 0;
		USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
				cvmx_usbcx_hccharx, epdir,
				((header->bRequestType & USB_DIR_IN) ?
					CVMX_USB_DIRECTION_IN :
					CVMX_USB_DIRECTION_OUT));
		USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
				cvmx_usbcx_hcspltx, compsplt, 1);
		break;
	case CVMX_USB_STAGE_STATUS:
		usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
		bytes_to_transfer = 0;
		USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
				cvmx_usbcx_hccharx, epdir,
				((header->bRequestType & USB_DIR_IN) ?
					CVMX_USB_DIRECTION_OUT :
					CVMX_USB_DIRECTION_IN));
		break;
	case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
		usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
		bytes_to_transfer = 0;
		USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
				cvmx_usbcx_hccharx, epdir,
				((header->bRequestType & USB_DIR_IN) ?
					CVMX_USB_DIRECTION_OUT :
					CVMX_USB_DIRECTION_IN));
		USB_SET_FIELD32(CVMX_USBCX_HCSPLTX(channel, usb->index),
				cvmx_usbcx_hcspltx, compsplt, 1);
		break;
	}

	/*
	 * Make sure the transfer never exceeds the byte limit of the hardware.
	 * Further bytes will be sent as continued transactions
	 */
	if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
		/* Round MAX_TRANSFER_BYTES to a multiple of out packet size */
		bytes_to_transfer = MAX_TRANSFER_BYTES / pipe->max_packet;
		bytes_to_transfer *= pipe->max_packet;
	}

	/*
	 * Calculate the number of packets to transfer. If the length is zero
	 * we still need to transfer one packet
	 */
	packets_to_transfer = DIV_ROUND_UP(bytes_to_transfer,
					   pipe->max_packet);
	if (packets_to_transfer == 0) {
		packets_to_transfer = 1;
	} else if ((packets_to_transfer > 1) &&
			(usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
		/*
		 * Limit to one packet when not using DMA. Channels must be
		 * restarted between every packet for IN transactions, so there
		 * is no reason to do multiple packets in a row
		 */
		packets_to_transfer = 1;
		bytes_to_transfer = packets_to_transfer * pipe->max_packet;
	} else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
		/*
		 * Limit the number of packet and data transferred to what the
		 * hardware can handle
		 */
		packets_to_transfer = MAX_TRANSFER_PACKETS;
		bytes_to_transfer = packets_to_transfer * pipe->max_packet;
	}

	usbc_hctsiz.s.xfersize = bytes_to_transfer;
	usbc_hctsiz.s.pktcnt = packets_to_transfer;

	cvmx_usb_write_csr32(usb, CVMX_USBCX_HCTSIZX(channel, usb->index),
			     usbc_hctsiz.u32);
}

/**
 * Start a channel to perform the pipe's head transaction
 *
 * @usb:	  USB device state populated by cvmx_usb_initialize().
 * @channel:	  Channel to setup
 * @pipe:	  Pipe to start
 */
static void cvmx_usb_start_channel(struct octeon_hcd *usb, int channel,
				   struct cvmx_usb_pipe *pipe)
{
	struct cvmx_usb_transaction *transaction =
		list_first_entry(&pipe->transactions, typeof(*transaction),
				 node);

	/* Make sure all writes to the DMA region get flushed */
	CVMX_SYNCW;

	/* Attach the channel to the pipe */
	usb->pipe_for_channel[channel] = pipe;
	pipe->channel = channel;
	pipe->flags |= CVMX_USB_PIPE_FLAGS_SCHEDULED;

	/* Mark this channel as in use */
	usb->idle_hardware_channels &= ~(1 << channel);

	/* Enable the channel interrupt bits */
	{
		union cvmx_usbcx_hcintx usbc_hcint;
		union cvmx_usbcx_hcintmskx usbc_hcintmsk;
		union cvmx_usbcx_haintmsk usbc_haintmsk;

		/* Clear all channel status bits */
		usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
						     CVMX_USBCX_HCINTX(channel, usb->index));

		cvmx_usb_write_csr32(usb,
				     CVMX_USBCX_HCINTX(channel, usb->index),
				     usbc_hcint.u32);

		usbc_hcintmsk.u32 = 0;
		usbc_hcintmsk.s.chhltdmsk = 1;
		if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
			/*
			 * Channels need these extra interrupts when we aren't
			 * in DMA mode.
			 */
			usbc_hcintmsk.s.datatglerrmsk = 1;
			usbc_hcintmsk.s.frmovrunmsk = 1;
			usbc_hcintmsk.s.bblerrmsk = 1;
			usbc_hcintmsk.s.xacterrmsk = 1;
			if (cvmx_usb_pipe_needs_split(usb, pipe)) {
				/*
				 * Splits don't generate xfercompl, so we need
				 * ACK and NYET.
				 */
				usbc_hcintmsk.s.nyetmsk = 1;
				usbc_hcintmsk.s.ackmsk = 1;
			}
			usbc_hcintmsk.s.nakmsk = 1;
			usbc_hcintmsk.s.stallmsk = 1;
			usbc_hcintmsk.s.xfercomplmsk = 1;
		}
		cvmx_usb_write_csr32(usb,
				     CVMX_USBCX_HCINTMSKX(channel, usb->index),
				     usbc_hcintmsk.u32);

		/* Enable the channel interrupt to propagate */
		usbc_haintmsk.u32 = cvmx_usb_read_csr32(usb,
							CVMX_USBCX_HAINTMSK(usb->index));
		usbc_haintmsk.s.haintmsk |= 1 << channel;
		cvmx_usb_write_csr32(usb, CVMX_USBCX_HAINTMSK(usb->index),
				     usbc_haintmsk.u32);
	}

	/* Setup the location the DMA engine uses. */
	{
		u64 reg;
		u64 dma_address = transaction->buffer +
				  transaction->actual_bytes;

		if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
			dma_address = transaction->buffer +
					transaction->iso_packets[0].offset +
					transaction->actual_bytes;

		if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT)
			reg = CVMX_USBNX_DMA0_OUTB_CHN0(usb->index);
		else
			reg = CVMX_USBNX_DMA0_INB_CHN0(usb->index);
		cvmx_write64_uint64(reg + channel * 8, dma_address);
	}

	/* Setup both the size of the transfer and the SPLIT characteristics */
	{
		union cvmx_usbcx_hcspltx usbc_hcsplt = {.u32 = 0};
		union cvmx_usbcx_hctsizx usbc_hctsiz = {.u32 = 0};
		int packets_to_transfer;
		int bytes_to_transfer = transaction->buffer_length -
			transaction->actual_bytes;

		/*
		 * ISOCHRONOUS transactions store each individual transfer size
		 * in the packet structure, not the global buffer_length
		 */
		if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
			bytes_to_transfer =
				transaction->iso_packets[0].length -
				transaction->actual_bytes;

		/*
		 * We need to do split transactions when we are talking to non
		 * high speed devices that are behind a high speed hub
		 */
		if (cvmx_usb_pipe_needs_split(usb, pipe)) {
			/*
			 * On the start split phase (stage is even) record the
			 * frame number we will need to send the split complete.
			 * We only store the lower two bits since the time ahead
			 * can only be two frames
			 */
			if ((transaction->stage & 1) == 0) {
				if (transaction->type == CVMX_USB_TRANSFER_BULK)
					pipe->split_sc_frame =
						(usb->frame_number + 1) & 0x7f;
				else
					pipe->split_sc_frame =
						(usb->frame_number + 2) & 0x7f;
			} else {
				pipe->split_sc_frame = -1;
			}

			usbc_hcsplt.s.spltena = 1;
			usbc_hcsplt.s.hubaddr = pipe->hub_device_addr;
			usbc_hcsplt.s.prtaddr = pipe->hub_port;
			usbc_hcsplt.s.compsplt = (transaction->stage ==
				CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE);

			/*
			 * SPLIT transactions can only ever transmit one data
			 * packet so limit the transfer size to the max packet
			 * size
			 */
			if (bytes_to_transfer > pipe->max_packet)
				bytes_to_transfer = pipe->max_packet;

			/*
			 * ISOCHRONOUS OUT splits are unique in that they limit
			 * data transfers to 188 byte chunks representing the
			 * begin/middle/end of the data or all
			 */
			if (!usbc_hcsplt.s.compsplt &&
			    (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
			    (pipe->transfer_type ==
			     CVMX_USB_TRANSFER_ISOCHRONOUS)) {
				/*
				 * Clear the split complete frame number as
				 * there isn't going to be a split complete
				 */
				pipe->split_sc_frame = -1;
				/*
				 * See if we've started this transfer and sent
				 * data
				 */
				if (transaction->actual_bytes == 0) {
					/*
					 * Nothing sent yet, this is either a
					 * begin or the entire payload
					 */
					if (bytes_to_transfer <= 188)
						/* Entire payload in one go */
						usbc_hcsplt.s.xactpos = 3;
					else
						/* First part of payload */
						usbc_hcsplt.s.xactpos = 2;
				} else {
					/*
					 * Continuing the previous data, we must
					 * either be in the middle or at the end
					 */
					if (bytes_to_transfer <= 188)
						/* End of payload */
						usbc_hcsplt.s.xactpos = 1;
					else
						/* Middle of payload */
						usbc_hcsplt.s.xactpos = 0;
				}
				/*
				 * Again, the transfer size is limited to 188
				 * bytes
				 */
				if (bytes_to_transfer > 188)
					bytes_to_transfer = 188;
			}
		}

		/*
		 * Make sure the transfer never exceeds the byte limit of the
		 * hardware. Further bytes will be sent as continued
		 * transactions
		 */
		if (bytes_to_transfer > MAX_TRANSFER_BYTES) {
			/*
			 * Round MAX_TRANSFER_BYTES to a multiple of out packet
			 * size
			 */
			bytes_to_transfer = MAX_TRANSFER_BYTES /
				pipe->max_packet;
			bytes_to_transfer *= pipe->max_packet;
		}

		/*
		 * Calculate the number of packets to transfer. If the length is
		 * zero we still need to transfer one packet
		 */
		packets_to_transfer =
			DIV_ROUND_UP(bytes_to_transfer, pipe->max_packet);
		if (packets_to_transfer == 0) {
			packets_to_transfer = 1;
		} else if ((packets_to_transfer > 1) &&
			   (usb->init_flags &
			    CVMX_USB_INITIALIZE_FLAGS_NO_DMA)) {
			/*
			 * Limit to one packet when not using DMA. Channels must
			 * be restarted between every packet for IN
			 * transactions, so there is no reason to do multiple
			 * packets in a row
			 */
			packets_to_transfer = 1;
			bytes_to_transfer = packets_to_transfer *
				pipe->max_packet;
		} else if (packets_to_transfer > MAX_TRANSFER_PACKETS) {
			/*
			 * Limit the number of packet and data transferred to
			 * what the hardware can handle
			 */
			packets_to_transfer = MAX_TRANSFER_PACKETS;
			bytes_to_transfer = packets_to_transfer *
				pipe->max_packet;
		}

		usbc_hctsiz.s.xfersize = bytes_to_transfer;
		usbc_hctsiz.s.pktcnt = packets_to_transfer;

		/* Update the DATA0/DATA1 toggle */
		usbc_hctsiz.s.pid = cvmx_usb_get_data_pid(pipe);
		/*
		 * High speed pipes may need a hardware ping before they start
		 */
		if (pipe->flags & CVMX_USB_PIPE_FLAGS_NEED_PING)
			usbc_hctsiz.s.dopng = 1;

		cvmx_usb_write_csr32(usb,
				     CVMX_USBCX_HCSPLTX(channel, usb->index),
				     usbc_hcsplt.u32);
		cvmx_usb_write_csr32(usb,
				     CVMX_USBCX_HCTSIZX(channel, usb->index),
				     usbc_hctsiz.u32);
	}

	/* Setup the Host Channel Characteristics Register */
	{
		union cvmx_usbcx_hccharx usbc_hcchar = {.u32 = 0};

		/*
		 * Set the startframe odd/even properly. This is only used for
		 * periodic
		 */
		usbc_hcchar.s.oddfrm = usb->frame_number & 1;

		/*
		 * Set the number of back to back packets allowed by this
		 * endpoint. Split transactions interpret "ec" as the number of
		 * immediate retries of failure. These retries happen too
		 * quickly, so we disable these entirely for splits
		 */
		if (cvmx_usb_pipe_needs_split(usb, pipe))
			usbc_hcchar.s.ec = 1;
		else if (pipe->multi_count < 1)
			usbc_hcchar.s.ec = 1;
		else if (pipe->multi_count > 3)
			usbc_hcchar.s.ec = 3;
		else
			usbc_hcchar.s.ec = pipe->multi_count;

		/* Set the rest of the endpoint specific settings */
		usbc_hcchar.s.devaddr = pipe->device_addr;
		usbc_hcchar.s.eptype = transaction->type;
		usbc_hcchar.s.lspddev =
			(pipe->device_speed == CVMX_USB_SPEED_LOW);
		usbc_hcchar.s.epdir = pipe->transfer_dir;
		usbc_hcchar.s.epnum = pipe->endpoint_num;
		usbc_hcchar.s.mps = pipe->max_packet;
		cvmx_usb_write_csr32(usb,
				     CVMX_USBCX_HCCHARX(channel, usb->index),
				     usbc_hcchar.u32);
	}

	/* Do transaction type specific fixups as needed */
	switch (transaction->type) {
	case CVMX_USB_TRANSFER_CONTROL:
		cvmx_usb_start_channel_control(usb, channel, pipe);
		break;
	case CVMX_USB_TRANSFER_BULK:
	case CVMX_USB_TRANSFER_INTERRUPT:
		break;
	case CVMX_USB_TRANSFER_ISOCHRONOUS:
		if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
			/*
			 * ISO transactions require different PIDs depending on
			 * direction and how many packets are needed
			 */
			if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
				if (pipe->multi_count < 2) /* Need DATA0 */
					USB_SET_FIELD32(
						CVMX_USBCX_HCTSIZX(channel,
								   usb->index),
						cvmx_usbcx_hctsizx, pid, 0);
				else /* Need MDATA */
					USB_SET_FIELD32(
						CVMX_USBCX_HCTSIZX(channel,
								   usb->index),
						cvmx_usbcx_hctsizx, pid, 3);
			}
		}
		break;
	}
	{
		union cvmx_usbcx_hctsizx usbc_hctsiz = { .u32 =
			cvmx_usb_read_csr32(usb,
					    CVMX_USBCX_HCTSIZX(channel,
							       usb->index))
		};
		transaction->xfersize = usbc_hctsiz.s.xfersize;
		transaction->pktcnt = usbc_hctsiz.s.pktcnt;
	}
	/* Remember when we start a split transaction */
	if (cvmx_usb_pipe_needs_split(usb, pipe))
		usb->active_split = transaction;
	USB_SET_FIELD32(CVMX_USBCX_HCCHARX(channel, usb->index),
			cvmx_usbcx_hccharx, chena, 1);
	if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
		cvmx_usb_fill_tx_fifo(usb, channel);
}

/**
 * Find a pipe that is ready to be scheduled to hardware.
 * @usb:	 USB device state populated by cvmx_usb_initialize().
 * @xfer_type:	 Transfer type
 *
 * Returns: Pipe or NULL if none are ready
 */
static struct cvmx_usb_pipe *cvmx_usb_find_ready_pipe(struct octeon_hcd *usb,
						      enum cvmx_usb_transfer xfer_type)
{
	struct list_head *list = usb->active_pipes + xfer_type;
	u64 current_frame = usb->frame_number;
	struct cvmx_usb_pipe *pipe;

	list_for_each_entry(pipe, list, node) {
		struct cvmx_usb_transaction *t =
			list_first_entry(&pipe->transactions, typeof(*t),
					 node);
		if (!(pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED) && t &&
		    (pipe->next_tx_frame <= current_frame) &&
		    ((pipe->split_sc_frame == -1) ||
		     ((((int)current_frame - pipe->split_sc_frame) & 0x7f) <
		      0x40)) &&
		    (!usb->active_split || (usb->active_split == t))) {
			prefetch(t);
			return pipe;
		}
	}
	return NULL;
}

static struct cvmx_usb_pipe *cvmx_usb_next_pipe(struct octeon_hcd *usb,
						int is_sof)
{
	struct cvmx_usb_pipe *pipe;

	/* Find a pipe needing service. */
	if (is_sof) {
		/*
		 * Only process periodic pipes on SOF interrupts. This way we
		 * are sure that the periodic data is sent in the beginning of
		 * the frame.
		 */
		pipe = cvmx_usb_find_ready_pipe(usb,
						CVMX_USB_TRANSFER_ISOCHRONOUS);
		if (pipe)
			return pipe;
		pipe = cvmx_usb_find_ready_pipe(usb,
						CVMX_USB_TRANSFER_INTERRUPT);
		if (pipe)
			return pipe;
	}
	pipe = cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_CONTROL);
	if (pipe)
		return pipe;
	return cvmx_usb_find_ready_pipe(usb, CVMX_USB_TRANSFER_BULK);
}

/**
 * Called whenever a pipe might need to be scheduled to the
 * hardware.
 *
 * @usb:	 USB device state populated by cvmx_usb_initialize().
 * @is_sof:	 True if this schedule was called on a SOF interrupt.
 */
static void cvmx_usb_schedule(struct octeon_hcd *usb, int is_sof)
{
	int channel;
	struct cvmx_usb_pipe *pipe;
	int need_sof;
	enum cvmx_usb_transfer ttype;

	if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
		/*
		 * Without DMA we need to be careful to not schedule something
		 * at the end of a frame and cause an overrun.
		 */
		union cvmx_usbcx_hfnum hfnum = {
			.u32 = cvmx_usb_read_csr32(usb,
						CVMX_USBCX_HFNUM(usb->index))
		};

		union cvmx_usbcx_hfir hfir = {
			.u32 = cvmx_usb_read_csr32(usb,
						CVMX_USBCX_HFIR(usb->index))
		};

		if (hfnum.s.frrem < hfir.s.frint / 4)
			goto done;
	}

	while (usb->idle_hardware_channels) {
		/* Find an idle channel */
		channel = __fls(usb->idle_hardware_channels);
		if (unlikely(channel > 7))
			break;

		pipe = cvmx_usb_next_pipe(usb, is_sof);
		if (!pipe)
			break;

		cvmx_usb_start_channel(usb, channel, pipe);
	}

done:
	/*
	 * Only enable SOF interrupts when we have transactions pending in the
	 * future that might need to be scheduled
	 */
	need_sof = 0;
	for (ttype = CVMX_USB_TRANSFER_CONTROL;
	     ttype <= CVMX_USB_TRANSFER_INTERRUPT; ttype++) {
		list_for_each_entry(pipe, &usb->active_pipes[ttype], node) {
			if (pipe->next_tx_frame > usb->frame_number) {
				need_sof = 1;
				break;
			}
		}
	}
	USB_SET_FIELD32(CVMX_USBCX_GINTMSK(usb->index),
			cvmx_usbcx_gintmsk, sofmsk, need_sof);
}

static void octeon_usb_urb_complete_callback(struct octeon_hcd *usb,
					     enum cvmx_usb_status status,
					     struct cvmx_usb_pipe *pipe,
					     struct cvmx_usb_transaction
						*transaction,
					     int bytes_transferred,
					     struct urb *urb)
{
	struct usb_hcd *hcd = octeon_to_hcd(usb);
	struct device *dev = hcd->self.controller;

	if (likely(status == CVMX_USB_STATUS_OK))
		urb->actual_length = bytes_transferred;
	else
		urb->actual_length = 0;

	urb->hcpriv = NULL;

	/* For Isochronous transactions we need to update the URB packet status
	 * list from data in our private copy
	 */
	if (usb_pipetype(urb->pipe) == PIPE_ISOCHRONOUS) {
		int i;
		/*
		 * The pointer to the private list is stored in the setup_packet
		 * field.
		 */
		struct cvmx_usb_iso_packet *iso_packet =
			(struct cvmx_usb_iso_packet *)urb->setup_packet;
		/* Recalculate the transfer size by adding up each packet */
		urb->actual_length = 0;
		for (i = 0; i < urb->number_of_packets; i++) {
			if (iso_packet[i].status == CVMX_USB_STATUS_OK) {
				urb->iso_frame_desc[i].status = 0;
				urb->iso_frame_desc[i].actual_length =
					iso_packet[i].length;
				urb->actual_length +=
					urb->iso_frame_desc[i].actual_length;
			} else {
				dev_dbg(dev, "ISOCHRONOUS packet=%d of %d status=%d pipe=%p transaction=%p size=%d\n",
					i, urb->number_of_packets,
					iso_packet[i].status, pipe,
					transaction, iso_packet[i].length);
				urb->iso_frame_desc[i].status = -EREMOTEIO;
			}
		}
		/* Free the private list now that we don't need it anymore */
		kfree(iso_packet);
		urb->setup_packet = NULL;
	}

	switch (status) {
	case CVMX_USB_STATUS_OK:
		urb->status = 0;
		break;
	case CVMX_USB_STATUS_CANCEL:
		if (urb->status == 0)
			urb->status = -ENOENT;
		break;
	case CVMX_USB_STATUS_STALL:
		dev_dbg(dev, "status=stall pipe=%p transaction=%p size=%d\n",
			pipe, transaction, bytes_transferred);
		urb->status = -EPIPE;
		break;
	case CVMX_USB_STATUS_BABBLEERR:
		dev_dbg(dev, "status=babble pipe=%p transaction=%p size=%d\n",
			pipe, transaction, bytes_transferred);
		urb->status = -EPIPE;
		break;
	case CVMX_USB_STATUS_SHORT:
		dev_dbg(dev, "status=short pipe=%p transaction=%p size=%d\n",
			pipe, transaction, bytes_transferred);
		urb->status = -EREMOTEIO;
		break;
	case CVMX_USB_STATUS_ERROR:
	case CVMX_USB_STATUS_XACTERR:
	case CVMX_USB_STATUS_DATATGLERR:
	case CVMX_USB_STATUS_FRAMEERR:
		dev_dbg(dev, "status=%d pipe=%p transaction=%p size=%d\n",
			status, pipe, transaction, bytes_transferred);
		urb->status = -EPROTO;
		break;
	}
	usb_hcd_unlink_urb_from_ep(octeon_to_hcd(usb), urb);
	spin_unlock(&usb->lock);
	usb_hcd_giveback_urb(octeon_to_hcd(usb), urb, urb->status);
	spin_lock(&usb->lock);
}

/**
 * Signal the completion of a transaction and free it. The
 * transaction will be removed from the pipe transaction list.
 *
 * @usb:	 USB device state populated by cvmx_usb_initialize().
 * @pipe:	 Pipe the transaction is on
 * @transaction:
 *		 Transaction that completed
 * @complete_code:
 *		 Completion code
 */
static void cvmx_usb_complete(struct octeon_hcd *usb,
			      struct cvmx_usb_pipe *pipe,
			      struct cvmx_usb_transaction *transaction,
			      enum cvmx_usb_status complete_code)
{
	/* If this was a split then clear our split in progress marker */
	if (usb->active_split == transaction)
		usb->active_split = NULL;

	/*
	 * Isochronous transactions need extra processing as they might not be
	 * done after a single data transfer
	 */
	if (unlikely(transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)) {
		/* Update the number of bytes transferred in this ISO packet */
		transaction->iso_packets[0].length = transaction->actual_bytes;
		transaction->iso_packets[0].status = complete_code;

		/*
		 * If there are more ISOs pending and we succeeded, schedule the
		 * next one
		 */
		if ((transaction->iso_number_packets > 1) &&
		    (complete_code == CVMX_USB_STATUS_OK)) {
			/* No bytes transferred for this packet as of yet */
			transaction->actual_bytes = 0;
			/* One less ISO waiting to transfer */
			transaction->iso_number_packets--;
			/* Increment to the next location in our packet array */
			transaction->iso_packets++;
			transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
			return;
		}
	}

	/* Remove the transaction from the pipe list */
	list_del(&transaction->node);
	if (list_empty(&pipe->transactions))
		list_move_tail(&pipe->node, &usb->idle_pipes);
	octeon_usb_urb_complete_callback(usb, complete_code, pipe,
					 transaction,
					 transaction->actual_bytes,
					 transaction->urb);
	kfree(transaction);
}

/**
 * Submit a usb transaction to a pipe. Called for all types
 * of transactions.
 *
 * @usb:
 * @pipe:	    Which pipe to submit to.
 * @type:	    Transaction type
 * @buffer:	    User buffer for the transaction
 * @buffer_length:
 *		    User buffer's length in bytes
 * @control_header:
 *		    For control transactions, the 8 byte standard header
 * @iso_start_frame:
 *		    For ISO transactions, the start frame
 * @iso_number_packets:
 *		    For ISO, the number of packet in the transaction.
 * @iso_packets:
 *		    A description of each ISO packet
 * @urb:	    URB for the callback
 *
 * Returns: Transaction or NULL on failure.
 */
static struct cvmx_usb_transaction *cvmx_usb_submit_transaction(
				struct octeon_hcd *usb,
				struct cvmx_usb_pipe *pipe,
				enum cvmx_usb_transfer type,
				u64 buffer,
				int buffer_length,
				u64 control_header,
				int iso_start_frame,
				int iso_number_packets,
				struct cvmx_usb_iso_packet *iso_packets,
				struct urb *urb)
{
	struct cvmx_usb_transaction *transaction;

	if (unlikely(pipe->transfer_type != type))
		return NULL;

	transaction = kzalloc(sizeof(*transaction), GFP_ATOMIC);
	if (unlikely(!transaction))
		return NULL;

	transaction->type = type;
	transaction->buffer = buffer;
	transaction->buffer_length = buffer_length;
	transaction->control_header = control_header;
	/* FIXME: This is not used, implement it. */
	transaction->iso_start_frame = iso_start_frame;
	transaction->iso_number_packets = iso_number_packets;
	transaction->iso_packets = iso_packets;
	transaction->urb = urb;
	if (transaction->type == CVMX_USB_TRANSFER_CONTROL)
		transaction->stage = CVMX_USB_STAGE_SETUP;
	else
		transaction->stage = CVMX_USB_STAGE_NON_CONTROL;

	if (!list_empty(&pipe->transactions)) {
		list_add_tail(&transaction->node, &pipe->transactions);
	} else {
		list_add_tail(&transaction->node, &pipe->transactions);
		list_move_tail(&pipe->node,
			       &usb->active_pipes[pipe->transfer_type]);

		/*
		 * We may need to schedule the pipe if this was the head of the
		 * pipe.
		 */
		cvmx_usb_schedule(usb, 0);
	}

	return transaction;
}

/**
 * Call to submit a USB Bulk transfer to a pipe.
 *
 * @usb:	    USB device state populated by cvmx_usb_initialize().
 * @pipe:	    Handle to the pipe for the transfer.
 * @urb:	    URB.
 *
 * Returns: A submitted transaction or NULL on failure.
 */
static struct cvmx_usb_transaction *cvmx_usb_submit_bulk(
						struct octeon_hcd *usb,
						struct cvmx_usb_pipe *pipe,
						struct urb *urb)
{
	return cvmx_usb_submit_transaction(usb, pipe, CVMX_USB_TRANSFER_BULK,
					   urb->transfer_dma,
					   urb->transfer_buffer_length,
					   0, /* control_header */
					   0, /* iso_start_frame */
					   0, /* iso_number_packets */
					   NULL, /* iso_packets */
					   urb);
}

/**
 * Call to submit a USB Interrupt transfer to a pipe.
 *
 * @usb:	    USB device state populated by cvmx_usb_initialize().
 * @pipe:	    Handle to the pipe for the transfer.
 * @urb:	    URB returned when the callback is called.
 *
 * Returns: A submitted transaction or NULL on failure.
 */
static struct cvmx_usb_transaction *cvmx_usb_submit_interrupt(
						struct octeon_hcd *usb,
						struct cvmx_usb_pipe *pipe,
						struct urb *urb)
{
	return cvmx_usb_submit_transaction(usb, pipe,
					   CVMX_USB_TRANSFER_INTERRUPT,
					   urb->transfer_dma,
					   urb->transfer_buffer_length,
					   0, /* control_header */
					   0, /* iso_start_frame */
					   0, /* iso_number_packets */
					   NULL, /* iso_packets */
					   urb);
}

/**
 * Call to submit a USB Control transfer to a pipe.
 *
 * @usb:	    USB device state populated by cvmx_usb_initialize().
 * @pipe:	    Handle to the pipe for the transfer.
 * @urb:	    URB.
 *
 * Returns: A submitted transaction or NULL on failure.
 */
static struct cvmx_usb_transaction *cvmx_usb_submit_control(
						struct octeon_hcd *usb,
						struct cvmx_usb_pipe *pipe,
						struct urb *urb)
{
	int buffer_length = urb->transfer_buffer_length;
	u64 control_header = urb->setup_dma;
	struct usb_ctrlrequest *header = cvmx_phys_to_ptr(control_header);

	if ((header->bRequestType & USB_DIR_IN) == 0)
		buffer_length = le16_to_cpu(header->wLength);

	return cvmx_usb_submit_transaction(usb, pipe,
					   CVMX_USB_TRANSFER_CONTROL,
					   urb->transfer_dma, buffer_length,
					   control_header,
					   0, /* iso_start_frame */
					   0, /* iso_number_packets */
					   NULL, /* iso_packets */
					   urb);
}

/**
 * Call to submit a USB Isochronous transfer to a pipe.
 *
 * @usb:	    USB device state populated by cvmx_usb_initialize().
 * @pipe:	    Handle to the pipe for the transfer.
 * @urb:	    URB returned when the callback is called.
 *
 * Returns: A submitted transaction or NULL on failure.
 */
static struct cvmx_usb_transaction *cvmx_usb_submit_isochronous(
						struct octeon_hcd *usb,
						struct cvmx_usb_pipe *pipe,
						struct urb *urb)
{
	struct cvmx_usb_iso_packet *packets;

	packets = (struct cvmx_usb_iso_packet *)urb->setup_packet;
	return cvmx_usb_submit_transaction(usb, pipe,
					   CVMX_USB_TRANSFER_ISOCHRONOUS,
					   urb->transfer_dma,
					   urb->transfer_buffer_length,
					   0, /* control_header */
					   urb->start_frame,
					   urb->number_of_packets,
					   packets, urb);
}

/**
 * Cancel one outstanding request in a pipe. Canceling a request
 * can fail if the transaction has already completed before cancel
 * is called. Even after a successful cancel call, it may take
 * a frame or two for the cvmx_usb_poll() function to call the
 * associated callback.
 *
 * @usb:	 USB device state populated by cvmx_usb_initialize().
 * @pipe:	 Pipe to cancel requests in.
 * @transaction: Transaction to cancel, returned by the submit function.
 *
 * Returns: 0 or a negative error code.
 */
static int cvmx_usb_cancel(struct octeon_hcd *usb,
			   struct cvmx_usb_pipe *pipe,
			   struct cvmx_usb_transaction *transaction)
{
	/*
	 * If the transaction is the HEAD of the queue and scheduled. We need to
	 * treat it special
	 */
	if (list_first_entry(&pipe->transactions, typeof(*transaction), node) ==
	    transaction && (pipe->flags & CVMX_USB_PIPE_FLAGS_SCHEDULED)) {
		union cvmx_usbcx_hccharx usbc_hcchar;

		usb->pipe_for_channel[pipe->channel] = NULL;
		pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;

		CVMX_SYNCW;

		usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
						      CVMX_USBCX_HCCHARX(pipe->channel,
									 usb->index));
		/*
		 * If the channel isn't enabled then the transaction already
		 * completed.
		 */
		if (usbc_hcchar.s.chena) {
			usbc_hcchar.s.chdis = 1;
			cvmx_usb_write_csr32(usb,
					     CVMX_USBCX_HCCHARX(pipe->channel,
								usb->index),
					     usbc_hcchar.u32);
		}
	}
	cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_CANCEL);
	return 0;
}

/**
 * Cancel all outstanding requests in a pipe. Logically all this
 * does is call cvmx_usb_cancel() in a loop.
 *
 * @usb:	 USB device state populated by cvmx_usb_initialize().
 * @pipe:	 Pipe to cancel requests in.
 *
 * Returns: 0 or a negative error code.
 */
static int cvmx_usb_cancel_all(struct octeon_hcd *usb,
			       struct cvmx_usb_pipe *pipe)
{
	struct cvmx_usb_transaction *transaction, *next;

	/* Simply loop through and attempt to cancel each transaction */
	list_for_each_entry_safe(transaction, next, &pipe->transactions, node) {
		int result = cvmx_usb_cancel(usb, pipe, transaction);

		if (unlikely(result != 0))
			return result;
	}
	return 0;
}

/**
 * Close a pipe created with cvmx_usb_open_pipe().
 *
 * @usb:	 USB device state populated by cvmx_usb_initialize().
 * @pipe:	 Pipe to close.
 *
 * Returns: 0 or a negative error code. EBUSY is returned if the pipe has
 *	    outstanding transfers.
 */
static int cvmx_usb_close_pipe(struct octeon_hcd *usb,
			       struct cvmx_usb_pipe *pipe)
{
	/* Fail if the pipe has pending transactions */
	if (!list_empty(&pipe->transactions))
		return -EBUSY;

	list_del(&pipe->node);
	kfree(pipe);

	return 0;
}

/**
 * Get the current USB protocol level frame number. The frame
 * number is always in the range of 0-0x7ff.
 *
 * @usb: USB device state populated by cvmx_usb_initialize().
 *
 * Returns: USB frame number
 */
static int cvmx_usb_get_frame_number(struct octeon_hcd *usb)
{
	union cvmx_usbcx_hfnum usbc_hfnum;

	usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));

	return usbc_hfnum.s.frnum;
}

static void cvmx_usb_transfer_control(struct octeon_hcd *usb,
				      struct cvmx_usb_pipe *pipe,
				      struct cvmx_usb_transaction *transaction,
				      union cvmx_usbcx_hccharx usbc_hcchar,
				      int buffer_space_left,
				      int bytes_in_last_packet)
{
	switch (transaction->stage) {
	case CVMX_USB_STAGE_NON_CONTROL:
	case CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE:
		/* This should be impossible */
		cvmx_usb_complete(usb, pipe, transaction,
				  CVMX_USB_STATUS_ERROR);
		break;
	case CVMX_USB_STAGE_SETUP:
		pipe->pid_toggle = 1;
		if (cvmx_usb_pipe_needs_split(usb, pipe)) {
			transaction->stage =
				CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE;
		} else {
			struct usb_ctrlrequest *header =
				cvmx_phys_to_ptr(transaction->control_header);
			if (header->wLength)
				transaction->stage = CVMX_USB_STAGE_DATA;
			else
				transaction->stage = CVMX_USB_STAGE_STATUS;
		}
		break;
	case CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE:
		{
			struct usb_ctrlrequest *header =
				cvmx_phys_to_ptr(transaction->control_header);
			if (header->wLength)
				transaction->stage = CVMX_USB_STAGE_DATA;
			else
				transaction->stage = CVMX_USB_STAGE_STATUS;
		}
		break;
	case CVMX_USB_STAGE_DATA:
		if (cvmx_usb_pipe_needs_split(usb, pipe)) {
			transaction->stage = CVMX_USB_STAGE_DATA_SPLIT_COMPLETE;
			/*
			 * For setup OUT data that are splits,
			 * the hardware doesn't appear to count
			 * transferred data. Here we manually
			 * update the data transferred
			 */
			if (!usbc_hcchar.s.epdir) {
				if (buffer_space_left < pipe->max_packet)
					transaction->actual_bytes +=
						buffer_space_left;
				else
					transaction->actual_bytes +=
						pipe->max_packet;
			}
		} else if ((buffer_space_left == 0) ||
			   (bytes_in_last_packet < pipe->max_packet)) {
			pipe->pid_toggle = 1;
			transaction->stage = CVMX_USB_STAGE_STATUS;
		}
		break;
	case CVMX_USB_STAGE_DATA_SPLIT_COMPLETE:
		if ((buffer_space_left == 0) ||
		    (bytes_in_last_packet < pipe->max_packet)) {
			pipe->pid_toggle = 1;
			transaction->stage = CVMX_USB_STAGE_STATUS;
		} else {
			transaction->stage = CVMX_USB_STAGE_DATA;
		}
		break;
	case CVMX_USB_STAGE_STATUS:
		if (cvmx_usb_pipe_needs_split(usb, pipe))
			transaction->stage =
				CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE;
		else
			cvmx_usb_complete(usb, pipe, transaction,
					  CVMX_USB_STATUS_OK);
		break;
	case CVMX_USB_STAGE_STATUS_SPLIT_COMPLETE:
		cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
		break;
	}
}

static void cvmx_usb_transfer_bulk(struct octeon_hcd *usb,
				   struct cvmx_usb_pipe *pipe,
				   struct cvmx_usb_transaction *transaction,
				   union cvmx_usbcx_hcintx usbc_hcint,
				   int buffer_space_left,
				   int bytes_in_last_packet)
{
	/*
	 * The only time a bulk transfer isn't complete when it finishes with
	 * an ACK is during a split transaction. For splits we need to continue
	 * the transfer if more data is needed.
	 */
	if (cvmx_usb_pipe_needs_split(usb, pipe)) {
		if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL)
			transaction->stage =
				CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
		else if (buffer_space_left &&
			 (bytes_in_last_packet == pipe->max_packet))
			transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
		else
			cvmx_usb_complete(usb, pipe, transaction,
					  CVMX_USB_STATUS_OK);
	} else {
		if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
		    (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) &&
		    (usbc_hcint.s.nak))
			pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;
		if (!buffer_space_left ||
		    (bytes_in_last_packet < pipe->max_packet))
			cvmx_usb_complete(usb, pipe, transaction,
					  CVMX_USB_STATUS_OK);
	}
}

static void cvmx_usb_transfer_intr(struct octeon_hcd *usb,
				   struct cvmx_usb_pipe *pipe,
				   struct cvmx_usb_transaction *transaction,
				   int buffer_space_left,
				   int bytes_in_last_packet)
{
	if (cvmx_usb_pipe_needs_split(usb, pipe)) {
		if (transaction->stage == CVMX_USB_STAGE_NON_CONTROL) {
			transaction->stage =
				CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
		} else if (buffer_space_left &&
			   (bytes_in_last_packet == pipe->max_packet)) {
			transaction->stage = CVMX_USB_STAGE_NON_CONTROL;
		} else {
			pipe->next_tx_frame += pipe->interval;
			cvmx_usb_complete(usb, pipe, transaction,
					  CVMX_USB_STATUS_OK);
		}
	} else if (!buffer_space_left ||
		   (bytes_in_last_packet < pipe->max_packet)) {
		pipe->next_tx_frame += pipe->interval;
		cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
	}
}

static void cvmx_usb_transfer_isoc(struct octeon_hcd *usb,
				   struct cvmx_usb_pipe *pipe,
				   struct cvmx_usb_transaction *transaction,
				   int buffer_space_left,
				   int bytes_in_last_packet,
				   int bytes_this_transfer)
{
	if (cvmx_usb_pipe_needs_split(usb, pipe)) {
		/*
		 * ISOCHRONOUS OUT splits don't require a complete split stage.
		 * Instead they use a sequence of begin OUT splits to transfer
		 * the data 188 bytes at a time. Once the transfer is complete,
		 * the pipe sleeps until the next schedule interval.
		 */
		if (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT) {
			/*
			 * If no space left or this wasn't a max size packet
			 * then this transfer is complete. Otherwise start it
			 * again to send the next 188 bytes
			 */
			if (!buffer_space_left || (bytes_this_transfer < 188)) {
				pipe->next_tx_frame += pipe->interval;
				cvmx_usb_complete(usb, pipe, transaction,
						  CVMX_USB_STATUS_OK);
			}
			return;
		}
		if (transaction->stage ==
		    CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE) {
			/*
			 * We are in the incoming data phase. Keep getting data
			 * until we run out of space or get a small packet
			 */
			if ((buffer_space_left == 0) ||
			    (bytes_in_last_packet < pipe->max_packet)) {
				pipe->next_tx_frame += pipe->interval;
				cvmx_usb_complete(usb, pipe, transaction,
						  CVMX_USB_STATUS_OK);
			}
		} else {
			transaction->stage =
				CVMX_USB_STAGE_NON_CONTROL_SPLIT_COMPLETE;
		}
	} else {
		pipe->next_tx_frame += pipe->interval;
		cvmx_usb_complete(usb, pipe, transaction, CVMX_USB_STATUS_OK);
	}
}

/**
 * Poll a channel for status
 *
 * @usb:     USB device
 * @channel: Channel to poll
 *
 * Returns: Zero on success
 */
static int cvmx_usb_poll_channel(struct octeon_hcd *usb, int channel)
{
	struct usb_hcd *hcd = octeon_to_hcd(usb);
	struct device *dev = hcd->self.controller;
	union cvmx_usbcx_hcintx usbc_hcint;
	union cvmx_usbcx_hctsizx usbc_hctsiz;
	union cvmx_usbcx_hccharx usbc_hcchar;
	struct cvmx_usb_pipe *pipe;
	struct cvmx_usb_transaction *transaction;
	int bytes_this_transfer;
	int bytes_in_last_packet;
	int packets_processed;
	int buffer_space_left;

	/* Read the interrupt status bits for the channel */
	usbc_hcint.u32 = cvmx_usb_read_csr32(usb,
					     CVMX_USBCX_HCINTX(channel, usb->index));

	if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA) {
		usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
						      CVMX_USBCX_HCCHARX(channel,
									 usb->index));

		if (usbc_hcchar.s.chena && usbc_hcchar.s.chdis) {
			/*
			 * There seems to be a bug in CN31XX which can cause
			 * interrupt IN transfers to get stuck until we do a
			 * write of HCCHARX without changing things
			 */
			cvmx_usb_write_csr32(usb,
					     CVMX_USBCX_HCCHARX(channel,
								usb->index),
					     usbc_hcchar.u32);
			return 0;
		}

		/*
		 * In non DMA mode the channels don't halt themselves. We need
		 * to manually disable channels that are left running
		 */
		if (!usbc_hcint.s.chhltd) {
			if (usbc_hcchar.s.chena) {
				union cvmx_usbcx_hcintmskx hcintmsk;
				/* Disable all interrupts except CHHLTD */
				hcintmsk.u32 = 0;
				hcintmsk.s.chhltdmsk = 1;
				cvmx_usb_write_csr32(usb,
						     CVMX_USBCX_HCINTMSKX(channel, usb->index),
						     hcintmsk.u32);
				usbc_hcchar.s.chdis = 1;
				cvmx_usb_write_csr32(usb,
						     CVMX_USBCX_HCCHARX(channel, usb->index),
						     usbc_hcchar.u32);
				return 0;
			} else if (usbc_hcint.s.xfercompl) {
				/*
				 * Successful IN/OUT with transfer complete.
				 * Channel halt isn't needed.
				 */
			} else {
				dev_err(dev, "USB%d: Channel %d interrupt without halt\n",
					usb->index, channel);
				return 0;
			}
		}
	} else {
		/*
		 * There is are no interrupts that we need to process when the
		 * channel is still running
		 */
		if (!usbc_hcint.s.chhltd)
			return 0;
	}

	/* Disable the channel interrupts now that it is done */
	cvmx_usb_write_csr32(usb, CVMX_USBCX_HCINTMSKX(channel, usb->index), 0);
	usb->idle_hardware_channels |= (1 << channel);

	/* Make sure this channel is tied to a valid pipe */
	pipe = usb->pipe_for_channel[channel];
	prefetch(pipe);
	if (!pipe)
		return 0;
	transaction = list_first_entry(&pipe->transactions,
				       typeof(*transaction),
				       node);
	prefetch(transaction);

	/*
	 * Disconnect this pipe from the HW channel. Later the schedule
	 * function will figure out which pipe needs to go
	 */
	usb->pipe_for_channel[channel] = NULL;
	pipe->flags &= ~CVMX_USB_PIPE_FLAGS_SCHEDULED;

	/*
	 * Read the channel config info so we can figure out how much data
	 * transferred
	 */
	usbc_hcchar.u32 = cvmx_usb_read_csr32(usb,
					      CVMX_USBCX_HCCHARX(channel, usb->index));
	usbc_hctsiz.u32 = cvmx_usb_read_csr32(usb,
					      CVMX_USBCX_HCTSIZX(channel, usb->index));

	/*
	 * Calculating the number of bytes successfully transferred is dependent
	 * on the transfer direction
	 */
	packets_processed = transaction->pktcnt - usbc_hctsiz.s.pktcnt;
	if (usbc_hcchar.s.epdir) {
		/*
		 * IN transactions are easy. For every byte received the
		 * hardware decrements xfersize. All we need to do is subtract
		 * the current value of xfersize from its starting value and we
		 * know how many bytes were written to the buffer
		 */
		bytes_this_transfer = transaction->xfersize -
			usbc_hctsiz.s.xfersize;
	} else {
		/*
		 * OUT transaction don't decrement xfersize. Instead pktcnt is
		 * decremented on every successful packet send. The hardware
		 * does this when it receives an ACK, or NYET. If it doesn't
		 * receive one of these responses pktcnt doesn't change
		 */
		bytes_this_transfer = packets_processed * usbc_hcchar.s.mps;
		/*
		 * The last packet may not be a full transfer if we didn't have
		 * enough data
		 */
		if (bytes_this_transfer > transaction->xfersize)
			bytes_this_transfer = transaction->xfersize;
	}
	/* Figure out how many bytes were in the last packet of the transfer */
	if (packets_processed)
		bytes_in_last_packet = bytes_this_transfer -
			(packets_processed - 1) * usbc_hcchar.s.mps;
	else
		bytes_in_last_packet = bytes_this_transfer;

	/*
	 * As a special case, setup transactions output the setup header, not
	 * the user's data. For this reason we don't count setup data as bytes
	 * transferred
	 */
	if ((transaction->stage == CVMX_USB_STAGE_SETUP) ||
	    (transaction->stage == CVMX_USB_STAGE_SETUP_SPLIT_COMPLETE))
		bytes_this_transfer = 0;

	/*
	 * Add the bytes transferred to the running total. It is important that
	 * bytes_this_transfer doesn't count any data that needs to be
	 * retransmitted
	 */
	transaction->actual_bytes += bytes_this_transfer;
	if (transaction->type == CVMX_USB_TRANSFER_ISOCHRONOUS)
		buffer_space_left = transaction->iso_packets[0].length -
			transaction->actual_bytes;
	else
		buffer_space_left = transaction->buffer_length -
			transaction->actual_bytes;

	/*
	 * We need to remember the PID toggle state for the next transaction.
	 * The hardware already updated it for the next transaction
	 */
	pipe->pid_toggle = !(usbc_hctsiz.s.pid == 0);

	/*
	 * For high speed bulk out, assume the next transaction will need to do
	 * a ping before proceeding. If this isn't true the ACK processing below
	 * will clear this flag
	 */
	if ((pipe->device_speed == CVMX_USB_SPEED_HIGH) &&
	    (pipe->transfer_type == CVMX_USB_TRANSFER_BULK) &&
	    (pipe->transfer_dir == CVMX_USB_DIRECTION_OUT))
		pipe->flags |= CVMX_USB_PIPE_FLAGS_NEED_PING;

	if (WARN_ON_ONCE(bytes_this_transfer < 0)) {
		/*
		 * In some rare cases the DMA engine seems to get stuck and
		 * keeps substracting same byte count over and over again. In
		 * such case we just need to fail every transaction.
		 */
		cvmx_usb_complete(usb, pipe, transaction,
				  CVMX_USB_STATUS_ERROR);
		return 0;
	}

	if (usbc_hcint.s.stall) {
		/*
		 * STALL as a response means this transaction cannot be
		 * completed because the device can't process transactions. Tell
		 * the user. Any data that was transferred will be counted on
		 * the actual bytes transferred
		 */
		pipe->pid_toggle = 0;
		cvmx_usb_complete(usb, pipe, transaction,
				  CVMX_USB_STATUS_STALL);
	} else if (usbc_hcint.s.xacterr) {
		/*
		 * XactErr as a response means the device signaled
		 * something wrong with the transfer. For example, PID
		 * toggle errors cause these.
		 */
		cvmx_usb_complete(usb, pipe, transaction,
				  CVMX_USB_STATUS_XACTERR);
	} else if (usbc_hcint.s.bblerr) {
		/* Babble Error (BblErr) */
		cvmx_usb_complete(usb, pipe, transaction,
				  CVMX_USB_STATUS_BABBLEERR);
	} else if (usbc_hcint.s.datatglerr) {
		/* Data toggle error */
		cvmx_usb_complete(usb, pipe, transaction,
				  CVMX_USB_STATUS_DATATGLERR);
	} else if (usbc_hcint.s.nyet) {
		/*
		 * NYET as a response is only allowed in three cases: as a
		 * response to a ping, as a response to a split transaction, and
		 * as a response to a bulk out. The ping case is handled by
		 * hardware, so we only have splits and bulk out
		 */
		if (!cvmx_usb_pipe_needs_split(usb, pipe)) {
			transaction->retries = 0;
			/*
			 * If there is more data to go then we need to try
			 * again. Otherwise this transaction is complete
			 */
			if ((buffer_space_left == 0) ||
			    (bytes_in_last_packet < pipe->max_packet))
				cvmx_usb_complete(usb, pipe,
						  transaction,
						  CVMX_USB_STATUS_OK);
		} else {
			/*
			 * Split transactions retry the split complete 4 times
			 * then rewind to the start split and do the entire
			 * transactions again
			 */
			transaction->retries++;
			if ((transaction->retries & 0x3) == 0) {
				/*
				 * Rewind to the beginning of the transaction by
				 * anding off the split complete bit
				 */
				transaction->stage &= ~1;
				pipe->split_sc_frame = -1;
			}
		}
	} else if (usbc_hcint.s.ack) {
		transaction->retries = 0;
		/*
		 * The ACK bit can only be checked after the other error bits.
		 * This is because a multi packet transfer may succeed in a
		 * number of packets and then get a different response on the
		 * last packet. In this case both ACK and the last response bit
		 * will be set. If none of the other response bits is set, then
		 * the last packet must have been an ACK
		 *
		 * Since we got an ACK, we know we don't need to do a ping on
		 * this pipe
		 */
		pipe->flags &= ~CVMX_USB_PIPE_FLAGS_NEED_PING;

		switch (transaction->type) {
		case CVMX_USB_TRANSFER_CONTROL:
			cvmx_usb_transfer_control(usb, pipe, transaction,
						  usbc_hcchar,
						  buffer_space_left,
						  bytes_in_last_packet);
			break;
		case CVMX_USB_TRANSFER_BULK:
			cvmx_usb_transfer_bulk(usb, pipe, transaction,
					       usbc_hcint, buffer_space_left,
					       bytes_in_last_packet);
			break;
		case CVMX_USB_TRANSFER_INTERRUPT:
			cvmx_usb_transfer_intr(usb, pipe, transaction,
					       buffer_space_left,
					       bytes_in_last_packet);
			break;
		case CVMX_USB_TRANSFER_ISOCHRONOUS:
			cvmx_usb_transfer_isoc(usb, pipe, transaction,
					       buffer_space_left,
					       bytes_in_last_packet,
					       bytes_this_transfer);
			break;
		}
	} else if (usbc_hcint.s.nak) {
		/*
		 * If this was a split then clear our split in progress marker.
		 */
		if (usb->active_split == transaction)
			usb->active_split = NULL;
		/*
		 * NAK as a response means the device couldn't accept the
		 * transaction, but it should be retried in the future. Rewind
		 * to the beginning of the transaction by anding off the split
		 * complete bit. Retry in the next interval
		 */
		transaction->retries = 0;
		transaction->stage &= ~1;
		pipe->next_tx_frame += pipe->interval;
		if (pipe->next_tx_frame < usb->frame_number)
			pipe->next_tx_frame = usb->frame_number +
				pipe->interval -
				(usb->frame_number - pipe->next_tx_frame) %
				pipe->interval;
	} else {
		struct cvmx_usb_port_status port;

		port = cvmx_usb_get_status(usb);
		if (port.port_enabled) {
			/* We'll retry the exact same transaction again */
			transaction->retries++;
		} else {
			/*
			 * We get channel halted interrupts with no result bits
			 * sets when the cable is unplugged
			 */
			cvmx_usb_complete(usb, pipe, transaction,
					  CVMX_USB_STATUS_ERROR);
		}
	}
	return 0;
}

static void octeon_usb_port_callback(struct octeon_hcd *usb)
{
	spin_unlock(&usb->lock);
	usb_hcd_poll_rh_status(octeon_to_hcd(usb));
	spin_lock(&usb->lock);
}

/**
 * Poll the USB block for status and call all needed callback
 * handlers. This function is meant to be called in the interrupt
 * handler for the USB controller. It can also be called
 * periodically in a loop for non-interrupt based operation.
 *
 * @usb: USB device state populated by cvmx_usb_initialize().
 *
 * Returns: 0 or a negative error code.
 */
static int cvmx_usb_poll(struct octeon_hcd *usb)
{
	union cvmx_usbcx_hfnum usbc_hfnum;
	union cvmx_usbcx_gintsts usbc_gintsts;

	prefetch_range(usb, sizeof(*usb));

	/* Update the frame counter */
	usbc_hfnum.u32 = cvmx_usb_read_csr32(usb, CVMX_USBCX_HFNUM(usb->index));
	if ((usb->frame_number & 0x3fff) > usbc_hfnum.s.frnum)
		usb->frame_number += 0x4000;
	usb->frame_number &= ~0x3fffull;
	usb->frame_number |= usbc_hfnum.s.frnum;

	/* Read the pending interrupts */
	usbc_gintsts.u32 = cvmx_usb_read_csr32(usb,
					       CVMX_USBCX_GINTSTS(usb->index));

	/* Clear the interrupts now that we know about them */
	cvmx_usb_write_csr32(usb, CVMX_USBCX_GINTSTS(usb->index),
			     usbc_gintsts.u32);

	if (usbc_gintsts.s.rxflvl) {
		/*
		 * RxFIFO Non-Empty (RxFLvl)
		 * Indicates that there is at least one packet pending to be
		 * read from the RxFIFO.
		 *
		 * In DMA mode this is handled by hardware
		 */
		if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
			cvmx_usb_poll_rx_fifo(usb);
	}
	if (usbc_gintsts.s.ptxfemp || usbc_gintsts.s.nptxfemp) {
		/* Fill the Tx FIFOs when not in DMA mode */
		if (usb->init_flags & CVMX_USB_INITIALIZE_FLAGS_NO_DMA)
			cvmx_usb_poll_tx_fifo(usb);
	}
	if (usbc_gintsts.s.disconnint || usbc_gintsts.s.prtint) {
		union cvmx_usbcx_hprt usbc_hprt;
		/*
		 * Disconnect Detected Interrupt (DisconnInt)
		 * Asserted when a device disconnect is detected.
		 *
		 * Host Port Interrupt (PrtInt)
		 * The core sets this bit to indicate a change in port status of
		 * one of the O2P USB core ports in Host mode. The application
		 * must read the Host Port Control and Status (HPRT) register to
		 * determine the exact event that caused this interrupt. The
		 * application must clear the appropriate status bit in the Host
		 * Port Control and Status register to clear this bit.
		 *
		 * Call the user's port callback
		 */
		octeon_usb_port_callback(usb);
		/* Clear the port change bits */
		usbc_hprt.u32 =
			cvmx_usb_read_csr32(usb, CVMX_USBCX_HPRT(usb->index));
		usbc_hprt.s.prtena = 0;
		cvmx_usb_write_csr32(usb, CVMX_USBCX_HPRT(usb->index),
				     usbc_hprt.u32);
	}
	if (usbc_gintsts.s.hchint) {
		/*
		 * Host Channels Interrupt (HChInt)
		 * The core sets this bit to indicate that an interrupt is
		 * pending on one of the channels of the core (in Host mode).
		 * The application must read the Host All Channels Interrupt
		 * (HAINT) register to determine the exact number of the channel
		 * on which the interrupt occurred, and then read the
		 * corresponding Host Channel-n Interrupt (HCINTn) register to
		 * determine the exact cause of the interrupt. The application
		 * must clear the appropriate status bit in the HCINTn register
		 * to clear this bit.
		 */
		union cvmx_usbcx_haint usbc_haint;

		usbc_haint.u32 = cvmx_usb_read_csr32(usb,
						     CVMX_USBCX_HAINT(usb->index));
		while (usbc_haint.u32) {
			int channel;

			channel = __fls(usbc_haint.u32);
			cvmx_usb_poll_channel(usb, channel);
			usbc_haint.u32 ^= 1 << channel;
		}
	}

	cvmx_usb_schedule(usb, usbc_gintsts.s.sof);

	return 0;
}

/* convert between an HCD pointer and the corresponding struct octeon_hcd */
static inline struct octeon_hcd *hcd_to_octeon(struct usb_hcd *hcd)
{
	return (struct octeon_hcd *)(hcd->hcd_priv);
}

static irqreturn_t octeon_usb_irq(struct usb_hcd *hcd)
{
	struct octeon_hcd *usb = hcd_to_octeon(hcd);
	unsigned long flags;

	spin_lock_irqsave(&usb->lock, flags);
	cvmx_usb_poll(usb);
	spin_unlock_irqrestore(&usb->lock, flags);
	return IRQ_HANDLED;
}

static int octeon_usb_start(struct usb_hcd *hcd)
{
	hcd->state = HC_STATE_RUNNING;
	return 0;
}

static void octeon_usb_stop(struct usb_hcd *hcd)
{
	hcd->state = HC_STATE_HALT;
}

static int octeon_usb_get_frame_number(struct usb_hcd *hcd)
{
	struct octeon_hcd *usb = hcd_to_octeon(hcd);

	return cvmx_usb_get_frame_number(usb);
}

static int octeon_usb_urb_enqueue(struct usb_hcd *hcd,
				  struct urb *urb,
				  gfp_t mem_flags)
{
	struct octeon_hcd *usb = hcd_to_octeon(hcd);
	struct device *dev = hcd->self.controller;
	struct cvmx_usb_transaction *transaction = NULL;
	struct cvmx_usb_pipe *pipe;
	unsigned long flags;
	struct cvmx_usb_iso_packet *iso_packet;
	struct usb_host_endpoint *ep = urb->ep;
	int rc;

	urb->status = 0;
	spin_lock_irqsave(&usb->lock, flags);

	rc = usb_hcd_link_urb_to_ep(hcd, urb);
	if (rc) {
		spin_unlock_irqrestore(&usb->lock, flags);
		return rc;
	}

	if (!ep->hcpriv) {
		enum cvmx_usb_transfer transfer_type;
		enum cvmx_usb_speed speed;
		int split_device = 0;
		int split_port = 0;

		switch (usb_pipetype(urb->pipe)) {
		case PIPE_ISOCHRONOUS:
			transfer_type = CVMX_USB_TRANSFER_ISOCHRONOUS;
			break;
		case PIPE_INTERRUPT:
			transfer_type = CVMX_USB_TRANSFER_INTERRUPT;
			break;
		case PIPE_CONTROL:
			transfer_type = CVMX_USB_TRANSFER_CONTROL;
			break;
		default:
			transfer_type = CVMX_USB_TRANSFER_BULK;
			break;
		}
		switch (urb->dev->speed) {
		case USB_SPEED_LOW:
			speed = CVMX_USB_SPEED_LOW;
			break;
		case USB_SPEED_FULL:
			speed = CVMX_USB_SPEED_FULL;
			break;
		default:
			speed = CVMX_USB_SPEED_HIGH;
			break;
		}
		/*
		 * For slow devices on high speed ports we need to find the hub
		 * that does the speed translation so we know where to send the
		 * split transactions.
		 */
		if (speed != CVMX_USB_SPEED_HIGH) {
			/*
			 * Start at this device and work our way up the usb
			 * tree.
			 */
			struct usb_device *dev = urb->dev;

			while (dev->parent) {
				/*
				 * If our parent is high speed then he'll
				 * receive the splits.
				 */
				if (dev->parent->speed == USB_SPEED_HIGH) {
					split_device = dev->parent->devnum;
					split_port = dev->portnum;
					break;
				}
				/*
				 * Move up the tree one level. If we make it all
				 * the way up the tree, then the port must not
				 * be in high speed mode and we don't need a
				 * split.
				 */
				dev = dev->parent;
			}
		}
		pipe = cvmx_usb_open_pipe(usb, usb_pipedevice(urb->pipe),
					  usb_pipeendpoint(urb->pipe), speed,
					  le16_to_cpu(ep->desc.wMaxPacketSize)
					  & 0x7ff,
					  transfer_type,
					  usb_pipein(urb->pipe) ?
						CVMX_USB_DIRECTION_IN :
						CVMX_USB_DIRECTION_OUT,
					  urb->interval,
					  (le16_to_cpu(ep->desc.wMaxPacketSize)
					   >> 11) & 0x3,
					  split_device, split_port);
		if (!pipe) {
			usb_hcd_unlink_urb_from_ep(hcd, urb);
			spin_unlock_irqrestore(&usb->lock, flags);
			dev_dbg(dev, "Failed to create pipe\n");
			return -ENOMEM;
		}
		ep->hcpriv = pipe;
	} else {
		pipe = ep->hcpriv;
	}

	switch (usb_pipetype(urb->pipe)) {
	case PIPE_ISOCHRONOUS:
		dev_dbg(dev, "Submit isochronous to %d.%d\n",
			usb_pipedevice(urb->pipe),
			usb_pipeendpoint(urb->pipe));
		/*
		 * Allocate a structure to use for our private list of
		 * isochronous packets.
		 */
		iso_packet = kmalloc_array(urb->number_of_packets,
					   sizeof(struct cvmx_usb_iso_packet),
					   GFP_ATOMIC);
		if (iso_packet) {
			int i;
			/* Fill the list with the data from the URB */
			for (i = 0; i < urb->number_of_packets; i++) {
				iso_packet[i].offset =
					urb->iso_frame_desc[i].offset;
				iso_packet[i].length =
					urb->iso_frame_desc[i].length;
				iso_packet[i].status = CVMX_USB_STATUS_ERROR;
			}
			/*
			 * Store a pointer to the list in the URB setup_packet
			 * field. We know this currently isn't being used and
			 * this saves us a bunch of logic.
			 */
			urb->setup_packet = (char *)iso_packet;
			transaction = cvmx_usb_submit_isochronous(usb,
								  pipe, urb);
			/*
			 * If submit failed we need to free our private packet
			 * list.
			 */
			if (!transaction) {
				urb->setup_packet = NULL;
				kfree(iso_packet);
			}
		}
		break;
	case PIPE_INTERRUPT:
		dev_dbg(dev, "Submit interrupt to %d.%d\n",
			usb_pipedevice(urb->pipe),
			usb_pipeendpoint(urb->pipe));
		transaction = cvmx_usb_submit_interrupt(usb, pipe, urb);
		break;
	case PIPE_CONTROL:
		dev_dbg(dev, "Submit control to %d.%d\n",
			usb_pipedevice(urb->pipe),
			usb_pipeendpoint(urb->pipe));
		transaction = cvmx_usb_submit_control(usb, pipe, urb);
		break;
	case PIPE_BULK:
		dev_dbg(dev, "Submit bulk to %d.%d\n",
			usb_pipedevice(urb->pipe),
			usb_pipeendpoint(urb->pipe));
		transaction = cvmx_usb_submit_bulk(usb, pipe, urb);
		break;
	}
	if (!transaction) {
		usb_hcd_unlink_urb_from_ep(hcd, urb);
		spin_unlock_irqrestore(&usb->lock, flags);
		dev_dbg(dev, "Failed to submit\n");
		return -ENOMEM;
	}
	urb->hcpriv = transaction;
	spin_unlock_irqrestore(&usb->lock, flags);
	return 0;
}

static int octeon_usb_urb_dequeue(struct usb_hcd *hcd,
				  struct urb *urb,
				  int status)
{
	struct octeon_hcd *usb = hcd_to_octeon(hcd);
	unsigned long flags;
	int rc;

	if (!urb->dev)
		return -EINVAL;

	spin_lock_irqsave(&usb->lock, flags);

	rc = usb_hcd_check_unlink_urb(hcd, urb, status);
	if (rc)
		goto out;

	urb->status = status;
	cvmx_usb_cancel(usb, urb->ep->hcpriv, urb->hcpriv);

out:
	spin_unlock_irqrestore(&usb->lock, flags);

	return rc;
}

static void octeon_usb_endpoint_disable(struct usb_hcd *hcd,
					struct usb_host_endpoint *ep)
{
	struct device *dev = hcd->self.controller;

	if (ep->hcpriv) {
		struct octeon_hcd *usb = hcd_to_octeon(hcd);
		struct cvmx_usb_pipe *pipe = ep->hcpriv;
		unsigned long flags;

		spin_lock_irqsave(&usb->lock, flags);
		cvmx_usb_cancel_all(usb, pipe);
		if (cvmx_usb_close_pipe(usb, pipe))
			dev_dbg(dev, "Closing pipe %p failed\n", pipe);
		spin_unlock_irqrestore(&usb->lock, flags);
		ep->hcpriv = NULL;
	}
}

static int octeon_usb_hub_status_data(struct usb_hcd *hcd, char *buf)
{
	struct octeon_hcd *usb = hcd_to_octeon(hcd);
	struct cvmx_usb_port_status port_status;
	unsigned long flags;

	spin_lock_irqsave(&usb->lock, flags);
	port_status = cvmx_usb_get_status(usb);
	spin_unlock_irqrestore(&usb->lock, flags);
	buf[0] = port_status.connect_change << 1;

	return buf[0] != 0;
}

static int octeon_usb_hub_control(struct usb_hcd *hcd, u16 typeReq, u16 wValue,
				  u16 wIndex, char *buf, u16 wLength)
{
	struct octeon_hcd *usb = hcd_to_octeon(hcd);
	struct device *dev = hcd->self.controller;
	struct cvmx_usb_port_status usb_port_status;
	int port_status;
	struct usb_hub_descriptor *desc;
	unsigned long flags;

	switch (typeReq) {
	case ClearHubFeature:
		dev_dbg(dev, "ClearHubFeature\n");
		switch (wValue) {
		case C_HUB_LOCAL_POWER:
		case C_HUB_OVER_CURRENT:
			/* Nothing required here */
			break;
		default:
			return -EINVAL;
		}
		break;
	case ClearPortFeature:
		dev_dbg(dev, "ClearPortFeature\n");
		if (wIndex != 1) {
			dev_dbg(dev, " INVALID\n");
			return -EINVAL;
		}

		switch (wValue) {
		case USB_PORT_FEAT_ENABLE:
			dev_dbg(dev, " ENABLE\n");
			spin_lock_irqsave(&usb->lock, flags);
			cvmx_usb_disable(usb);
			spin_unlock_irqrestore(&usb->lock, flags);
			break;
		case USB_PORT_FEAT_SUSPEND:
			dev_dbg(dev, " SUSPEND\n");
			/* Not supported on Octeon */
			break;
		case USB_PORT_FEAT_POWER:
			dev_dbg(dev, " POWER\n");
			/* Not supported on Octeon */
			break;
		case USB_PORT_FEAT_INDICATOR:
			dev_dbg(dev, " INDICATOR\n");
			/* Port inidicator not supported */
			break;
		case USB_PORT_FEAT_C_CONNECTION:
			dev_dbg(dev, " C_CONNECTION\n");
			/* Clears drivers internal connect status change flag */
			spin_lock_irqsave(&usb->lock, flags);
			usb->port_status = cvmx_usb_get_status(usb);
			spin_unlock_irqrestore(&usb->lock, flags);
			break;
		case USB_PORT_FEAT_C_RESET:
			dev_dbg(dev, " C_RESET\n");
			/*
			 * Clears the driver's internal Port Reset Change flag.
			 */
			spin_lock_irqsave(&usb->lock, flags);
			usb->port_status = cvmx_usb_get_status(usb);
			spin_unlock_irqrestore(&usb->lock, flags);
			break;
		case USB_PORT_FEAT_C_ENABLE:
			dev_dbg(dev, " C_ENABLE\n");
			/*
			 * Clears the driver's internal Port Enable/Disable
			 * Change flag.
			 */
			spin_lock_irqsave(&usb->lock, flags);
			usb->port_status = cvmx_usb_get_status(usb);
			spin_unlock_irqrestore(&usb->lock, flags);
			break;
		case USB_PORT_FEAT_C_SUSPEND:
			dev_dbg(dev, " C_SUSPEND\n");
			/*
			 * Clears the driver's internal Port Suspend Change
			 * flag, which is set when resume signaling on the host
			 * port is complete.
			 */
			break;
		case USB_PORT_FEAT_C_OVER_CURRENT:
			dev_dbg(dev, " C_OVER_CURRENT\n");
			/* Clears the driver's overcurrent Change flag */
			spin_lock_irqsave(&usb->lock, flags);
			usb->port_status = cvmx_usb_get_status(usb);
			spin_unlock_irqrestore(&usb->lock, flags);
			break;
		default:
			dev_dbg(dev, " UNKNOWN\n");
			return -EINVAL;
		}
		break;
	case GetHubDescriptor:
		dev_dbg(dev, "GetHubDescriptor\n");
		desc = (struct usb_hub_descriptor *)buf;
		desc->bDescLength = 9;
		desc->bDescriptorType = 0x29;
		desc->bNbrPorts = 1;
		desc->wHubCharacteristics = cpu_to_le16(0x08);
		desc->bPwrOn2PwrGood = 1;
		desc->bHubContrCurrent = 0;
		desc->u.hs.DeviceRemovable[0] = 0;
		desc->u.hs.DeviceRemovable[1] = 0xff;
		break;
	case GetHubStatus:
		dev_dbg(dev, "GetHubStatus\n");
		*(__le32 *)buf = 0;
		break;
	case GetPortStatus:
		dev_dbg(dev, "GetPortStatus\n");
		if (wIndex != 1) {
			dev_dbg(dev, " INVALID\n");
			return -EINVAL;
		}

		spin_lock_irqsave(&usb->lock, flags);
		usb_port_status = cvmx_usb_get_status(usb);
		spin_unlock_irqrestore(&usb->lock, flags);
		port_status = 0;

		if (usb_port_status.connect_change) {
			port_status |= (1 << USB_PORT_FEAT_C_CONNECTION);
			dev_dbg(dev, " C_CONNECTION\n");
		}

		if (usb_port_status.port_enabled) {
			port_status |= (1 << USB_PORT_FEAT_C_ENABLE);
			dev_dbg(dev, " C_ENABLE\n");
		}

		if (usb_port_status.connected) {
			port_status |= (1 << USB_PORT_FEAT_CONNECTION);
			dev_dbg(dev, " CONNECTION\n");
		}

		if (usb_port_status.port_enabled) {
			port_status |= (1 << USB_PORT_FEAT_ENABLE);
			dev_dbg(dev, " ENABLE\n");
		}

		if (usb_port_status.port_over_current) {
			port_status |= (1 << USB_PORT_FEAT_OVER_CURRENT);
			dev_dbg(dev, " OVER_CURRENT\n");
		}

		if (usb_port_status.port_powered) {
			port_status |= (1 << USB_PORT_FEAT_POWER);
			dev_dbg(dev, " POWER\n");
		}

		if (usb_port_status.port_speed == CVMX_USB_SPEED_HIGH) {
			port_status |= USB_PORT_STAT_HIGH_SPEED;
			dev_dbg(dev, " HIGHSPEED\n");
		} else if (usb_port_status.port_speed == CVMX_USB_SPEED_LOW) {
			port_status |= (1 << USB_PORT_FEAT_LOWSPEED);
			dev_dbg(dev, " LOWSPEED\n");
		}

		*((__le32 *)buf) = cpu_to_le32(port_status);
		break;
	case SetHubFeature:
		dev_dbg(dev, "SetHubFeature\n");
		/* No HUB features supported */
		break;
	case SetPortFeature:
		dev_dbg(dev, "SetPortFeature\n");
		if (wIndex != 1) {
			dev_dbg(dev, " INVALID\n");
			return -EINVAL;
		}

		switch (wValue) {
		case USB_PORT_FEAT_SUSPEND:
			dev_dbg(dev, " SUSPEND\n");
			return -EINVAL;
		case USB_PORT_FEAT_POWER:
			dev_dbg(dev, " POWER\n");
			/*
			 * Program the port power bit to drive VBUS on the USB.
			 */
			spin_lock_irqsave(&usb->lock, flags);
			USB_SET_FIELD32(CVMX_USBCX_HPRT(usb->index),
					cvmx_usbcx_hprt, prtpwr, 1);
			spin_unlock_irqrestore(&usb->lock, flags);
			return 0;
		case USB_PORT_FEAT_RESET:
			dev_dbg(dev, " RESET\n");
			spin_lock_irqsave(&usb->lock, flags);
			cvmx_usb_reset_port(usb);
			spin_unlock_irqrestore(&usb->lock, flags);
			return 0;
		case USB_PORT_FEAT_INDICATOR:
			dev_dbg(dev, " INDICATOR\n");
			/* Not supported */
			break;
		default:
			dev_dbg(dev, " UNKNOWN\n");
			return -EINVAL;
		}
		break;
	default:
		dev_dbg(dev, "Unknown root hub request\n");
		return -EINVAL;
	}
	return 0;
}

static const struct hc_driver octeon_hc_driver = {
	.description		= "Octeon USB",
	.product_desc		= "Octeon Host Controller",
	.hcd_priv_size		= sizeof(struct octeon_hcd),
	.irq			= octeon_usb_irq,
	.flags			= HCD_MEMORY | HCD_DMA | HCD_USB2,
	.start			= octeon_usb_start,
	.stop			= octeon_usb_stop,
	.urb_enqueue		= octeon_usb_urb_enqueue,
	.urb_dequeue		= octeon_usb_urb_dequeue,
	.endpoint_disable	= octeon_usb_endpoint_disable,
	.get_frame_number	= octeon_usb_get_frame_number,
	.hub_status_data	= octeon_usb_hub_status_data,
	.hub_control		= octeon_usb_hub_control,
	.map_urb_for_dma	= octeon_map_urb_for_dma,
	.unmap_urb_for_dma	= octeon_unmap_urb_for_dma,
};

static int octeon_usb_probe(struct platform_device *pdev)
{
	int status;
	int initialize_flags;
	int usb_num;
	struct resource *res_mem;
	struct device_node *usbn_node;
	int irq = platform_get_irq(pdev, 0);
	struct device *dev = &pdev->dev;
	struct octeon_hcd *usb;
	struct usb_hcd *hcd;
	u32 clock_rate = 48000000;
	bool is_crystal_clock = false;
	const char *clock_type;
	int i;

	if (!dev->of_node) {
		dev_err(dev, "Error: empty of_node\n");
		return -ENXIO;
	}
	usbn_node = dev->of_node->parent;

	i = of_property_read_u32(usbn_node,
				 "clock-frequency", &clock_rate);
	if (i)
		i = of_property_read_u32(usbn_node,
					 "refclk-frequency", &clock_rate);
	if (i) {
		dev_err(dev, "No USBN \"clock-frequency\"\n");
		return -ENXIO;
	}
	switch (clock_rate) {
	case 12000000:
		initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_12MHZ;
		break;
	case 24000000:
		initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_24MHZ;
		break;
	case 48000000:
		initialize_flags = CVMX_USB_INITIALIZE_FLAGS_CLOCK_48MHZ;
		break;
	default:
		dev_err(dev, "Illegal USBN \"clock-frequency\" %u\n",
			clock_rate);
		return -ENXIO;
	}

	i = of_property_read_string(usbn_node,
				    "cavium,refclk-type", &clock_type);
	if (i)
		i = of_property_read_string(usbn_node,
					    "refclk-type", &clock_type);

	if (!i && strcmp("crystal", clock_type) == 0)
		is_crystal_clock = true;

	if (is_crystal_clock)
		initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_XI;
	else
		initialize_flags |= CVMX_USB_INITIALIZE_FLAGS_CLOCK_XO_GND;

	res_mem = platform_get_resource(pdev, IORESOURCE_MEM, 0);
	if (!res_mem) {
		dev_err(dev, "found no memory resource\n");
		return -ENXIO;
	}
	usb_num = (res_mem->start >> 44) & 1;

	if (irq < 0) {
		/* Defective device tree, but we know how to fix it. */
		irq_hw_number_t hwirq = usb_num ? (1 << 6) + 17 : 56;

		irq = irq_create_mapping(NULL, hwirq);
	}

	/*
	 * Set the DMA mask to 64bits so we get buffers already translated for
	 * DMA.
	 */
	i = dma_coerce_mask_and_coherent(dev, DMA_BIT_MASK(64));
	if (i)
		return i;

	/*
	 * Only cn52XX and cn56XX have DWC_OTG USB hardware and the
	 * IOB priority registers.  Under heavy network load USB
	 * hardware can be starved by the IOB causing a crash.  Give
	 * it a priority boost if it has been waiting more than 400
	 * cycles to avoid this situation.
	 *
	 * Testing indicates that a cnt_val of 8192 is not sufficient,
	 * but no failures are seen with 4096.  We choose a value of
	 * 400 to give a safety factor of 10.
	 */
	if (OCTEON_IS_MODEL(OCTEON_CN52XX) || OCTEON_IS_MODEL(OCTEON_CN56XX)) {
		union cvmx_iob_n2c_l2c_pri_cnt pri_cnt;

		pri_cnt.u64 = 0;
		pri_cnt.s.cnt_enb = 1;
		pri_cnt.s.cnt_val = 400;
		cvmx_write_csr(CVMX_IOB_N2C_L2C_PRI_CNT, pri_cnt.u64);
	}

	hcd = usb_create_hcd(&octeon_hc_driver, dev, dev_name(dev));
	if (!hcd) {
		dev_dbg(dev, "Failed to allocate memory for HCD\n");
		return -1;
	}
	hcd->uses_new_polling = 1;
	usb = (struct octeon_hcd *)hcd->hcd_priv;

	spin_lock_init(&usb->lock);

	usb->init_flags = initialize_flags;

	/* Initialize the USB state structure */
	usb->index = usb_num;
	INIT_LIST_HEAD(&usb->idle_pipes);
	for (i = 0; i < ARRAY_SIZE(usb->active_pipes); i++)
		INIT_LIST_HEAD(&usb->active_pipes[i]);

	/* Due to an errata, CN31XX doesn't support DMA */
	if (OCTEON_IS_MODEL(OCTEON_CN31XX)) {
		usb->init_flags |= CVMX_USB_INITIALIZE_FLAGS_NO_DMA;
		/* Only use one channel with non DMA */
		usb->idle_hardware_channels = 0x1;
	} else if (OCTEON_IS_MODEL(OCTEON_CN5XXX)) {
		/* CN5XXX have an errata with channel 3 */
		usb->idle_hardware_channels = 0xf7;
	} else {
		usb->idle_hardware_channels = 0xff;
	}

	status = cvmx_usb_initialize(dev, usb);
	if (status) {
		dev_dbg(dev, "USB initialization failed with %d\n", status);
		usb_put_hcd(hcd);
		return -1;
	}

	status = usb_add_hcd(hcd, irq, 0);
	if (status) {
		dev_dbg(dev, "USB add HCD failed with %d\n", status);
		usb_put_hcd(hcd);
		return -1;
	}
	device_wakeup_enable(hcd->self.controller);

	dev_info(dev, "Registered HCD for port %d on irq %d\n", usb_num, irq);

	return 0;
}

static int octeon_usb_remove(struct platform_device *pdev)
{
	int status;
	struct device *dev = &pdev->dev;
	struct usb_hcd *hcd = dev_get_drvdata(dev);
	struct octeon_hcd *usb = hcd_to_octeon(hcd);
	unsigned long flags;

	usb_remove_hcd(hcd);
	spin_lock_irqsave(&usb->lock, flags);
	status = cvmx_usb_shutdown(usb);
	spin_unlock_irqrestore(&usb->lock, flags);
	if (status)
		dev_dbg(dev, "USB shutdown failed with %d\n", status);

	usb_put_hcd(hcd);

	return 0;
}

static const struct of_device_id octeon_usb_match[] = {
	{
		.compatible = "cavium,octeon-5750-usbc",
	},
	{},
};
MODULE_DEVICE_TABLE(of, octeon_usb_match);

static struct platform_driver octeon_usb_driver = {
	.driver = {
		.name		= "octeon-hcd",
		.of_match_table = octeon_usb_match,
	},
	.probe      = octeon_usb_probe,
	.remove     = octeon_usb_remove,
};

static int __init octeon_usb_driver_init(void)
{
	if (usb_disabled())
		return 0;

	return platform_driver_register(&octeon_usb_driver);
}
module_init(octeon_usb_driver_init);

static void __exit octeon_usb_driver_exit(void)
{
	if (usb_disabled())
		return;

	platform_driver_unregister(&octeon_usb_driver);
}
module_exit(octeon_usb_driver_exit);

MODULE_LICENSE("GPL");
MODULE_AUTHOR("Cavium, Inc. <support@cavium.com>");
MODULE_DESCRIPTION("Cavium Inc. OCTEON USB Host driver.");