116603153SAndreas Noever /* 216603153SAndreas Noever * Thunderbolt Cactus Ridge driver - NHI driver 316603153SAndreas Noever * 416603153SAndreas Noever * The NHI (native host interface) is the pci device that allows us to send and 516603153SAndreas Noever * receive frames from the thunderbolt bus. 616603153SAndreas Noever * 716603153SAndreas Noever * Copyright (c) 2014 Andreas Noever <andreas.noever@gmail.com> 816603153SAndreas Noever */ 916603153SAndreas Noever 1016603153SAndreas Noever #include <linux/slab.h> 1116603153SAndreas Noever #include <linux/errno.h> 1216603153SAndreas Noever #include <linux/pci.h> 1316603153SAndreas Noever #include <linux/interrupt.h> 1416603153SAndreas Noever #include <linux/module.h> 1516603153SAndreas Noever #include <linux/dmi.h> 1616603153SAndreas Noever 1716603153SAndreas Noever #include "nhi.h" 1816603153SAndreas Noever #include "nhi_regs.h" 19*d6cc51cdSAndreas Noever #include "tb.h" 2016603153SAndreas Noever 2116603153SAndreas Noever #define RING_TYPE(ring) ((ring)->is_tx ? "TX ring" : "RX ring") 2216603153SAndreas Noever 2316603153SAndreas Noever 2416603153SAndreas Noever static int ring_interrupt_index(struct tb_ring *ring) 2516603153SAndreas Noever { 2616603153SAndreas Noever int bit = ring->hop; 2716603153SAndreas Noever if (!ring->is_tx) 2816603153SAndreas Noever bit += ring->nhi->hop_count; 2916603153SAndreas Noever return bit; 3016603153SAndreas Noever } 3116603153SAndreas Noever 3216603153SAndreas Noever /** 3316603153SAndreas Noever * ring_interrupt_active() - activate/deactivate interrupts for a single ring 3416603153SAndreas Noever * 3516603153SAndreas Noever * ring->nhi->lock must be held. 3616603153SAndreas Noever */ 3716603153SAndreas Noever static void ring_interrupt_active(struct tb_ring *ring, bool active) 3816603153SAndreas Noever { 3916603153SAndreas Noever int reg = REG_RING_INTERRUPT_BASE + ring_interrupt_index(ring) / 32; 4016603153SAndreas Noever int bit = ring_interrupt_index(ring) & 31; 4116603153SAndreas Noever int mask = 1 << bit; 4216603153SAndreas Noever u32 old, new; 4316603153SAndreas Noever old = ioread32(ring->nhi->iobase + reg); 4416603153SAndreas Noever if (active) 4516603153SAndreas Noever new = old | mask; 4616603153SAndreas Noever else 4716603153SAndreas Noever new = old & ~mask; 4816603153SAndreas Noever 4916603153SAndreas Noever dev_info(&ring->nhi->pdev->dev, 5016603153SAndreas Noever "%s interrupt at register %#x bit %d (%#x -> %#x)\n", 5116603153SAndreas Noever active ? "enabling" : "disabling", reg, bit, old, new); 5216603153SAndreas Noever 5316603153SAndreas Noever if (new == old) 5416603153SAndreas Noever dev_WARN(&ring->nhi->pdev->dev, 5516603153SAndreas Noever "interrupt for %s %d is already %s\n", 5616603153SAndreas Noever RING_TYPE(ring), ring->hop, 5716603153SAndreas Noever active ? "enabled" : "disabled"); 5816603153SAndreas Noever iowrite32(new, ring->nhi->iobase + reg); 5916603153SAndreas Noever } 6016603153SAndreas Noever 6116603153SAndreas Noever /** 6216603153SAndreas Noever * nhi_disable_interrupts() - disable interrupts for all rings 6316603153SAndreas Noever * 6416603153SAndreas Noever * Use only during init and shutdown. 6516603153SAndreas Noever */ 6616603153SAndreas Noever static void nhi_disable_interrupts(struct tb_nhi *nhi) 6716603153SAndreas Noever { 6816603153SAndreas Noever int i = 0; 6916603153SAndreas Noever /* disable interrupts */ 7016603153SAndreas Noever for (i = 0; i < RING_INTERRUPT_REG_COUNT(nhi); i++) 7116603153SAndreas Noever iowrite32(0, nhi->iobase + REG_RING_INTERRUPT_BASE + 4 * i); 7216603153SAndreas Noever 7316603153SAndreas Noever /* clear interrupt status bits */ 7416603153SAndreas Noever for (i = 0; i < RING_NOTIFY_REG_COUNT(nhi); i++) 7516603153SAndreas Noever ioread32(nhi->iobase + REG_RING_NOTIFY_BASE + 4 * i); 7616603153SAndreas Noever } 7716603153SAndreas Noever 7816603153SAndreas Noever /* ring helper methods */ 7916603153SAndreas Noever 8016603153SAndreas Noever static void __iomem *ring_desc_base(struct tb_ring *ring) 8116603153SAndreas Noever { 8216603153SAndreas Noever void __iomem *io = ring->nhi->iobase; 8316603153SAndreas Noever io += ring->is_tx ? REG_TX_RING_BASE : REG_RX_RING_BASE; 8416603153SAndreas Noever io += ring->hop * 16; 8516603153SAndreas Noever return io; 8616603153SAndreas Noever } 8716603153SAndreas Noever 8816603153SAndreas Noever static void __iomem *ring_options_base(struct tb_ring *ring) 8916603153SAndreas Noever { 9016603153SAndreas Noever void __iomem *io = ring->nhi->iobase; 9116603153SAndreas Noever io += ring->is_tx ? REG_TX_OPTIONS_BASE : REG_RX_OPTIONS_BASE; 9216603153SAndreas Noever io += ring->hop * 32; 9316603153SAndreas Noever return io; 9416603153SAndreas Noever } 9516603153SAndreas Noever 9616603153SAndreas Noever static void ring_iowrite16desc(struct tb_ring *ring, u32 value, u32 offset) 9716603153SAndreas Noever { 9816603153SAndreas Noever iowrite16(value, ring_desc_base(ring) + offset); 9916603153SAndreas Noever } 10016603153SAndreas Noever 10116603153SAndreas Noever static void ring_iowrite32desc(struct tb_ring *ring, u32 value, u32 offset) 10216603153SAndreas Noever { 10316603153SAndreas Noever iowrite32(value, ring_desc_base(ring) + offset); 10416603153SAndreas Noever } 10516603153SAndreas Noever 10616603153SAndreas Noever static void ring_iowrite64desc(struct tb_ring *ring, u64 value, u32 offset) 10716603153SAndreas Noever { 10816603153SAndreas Noever iowrite32(value, ring_desc_base(ring) + offset); 10916603153SAndreas Noever iowrite32(value >> 32, ring_desc_base(ring) + offset + 4); 11016603153SAndreas Noever } 11116603153SAndreas Noever 11216603153SAndreas Noever static void ring_iowrite32options(struct tb_ring *ring, u32 value, u32 offset) 11316603153SAndreas Noever { 11416603153SAndreas Noever iowrite32(value, ring_options_base(ring) + offset); 11516603153SAndreas Noever } 11616603153SAndreas Noever 11716603153SAndreas Noever static bool ring_full(struct tb_ring *ring) 11816603153SAndreas Noever { 11916603153SAndreas Noever return ((ring->head + 1) % ring->size) == ring->tail; 12016603153SAndreas Noever } 12116603153SAndreas Noever 12216603153SAndreas Noever static bool ring_empty(struct tb_ring *ring) 12316603153SAndreas Noever { 12416603153SAndreas Noever return ring->head == ring->tail; 12516603153SAndreas Noever } 12616603153SAndreas Noever 12716603153SAndreas Noever /** 12816603153SAndreas Noever * ring_write_descriptors() - post frames from ring->queue to the controller 12916603153SAndreas Noever * 13016603153SAndreas Noever * ring->lock is held. 13116603153SAndreas Noever */ 13216603153SAndreas Noever static void ring_write_descriptors(struct tb_ring *ring) 13316603153SAndreas Noever { 13416603153SAndreas Noever struct ring_frame *frame, *n; 13516603153SAndreas Noever struct ring_desc *descriptor; 13616603153SAndreas Noever list_for_each_entry_safe(frame, n, &ring->queue, list) { 13716603153SAndreas Noever if (ring_full(ring)) 13816603153SAndreas Noever break; 13916603153SAndreas Noever list_move_tail(&frame->list, &ring->in_flight); 14016603153SAndreas Noever descriptor = &ring->descriptors[ring->head]; 14116603153SAndreas Noever descriptor->phys = frame->buffer_phy; 14216603153SAndreas Noever descriptor->time = 0; 14316603153SAndreas Noever descriptor->flags = RING_DESC_POSTED | RING_DESC_INTERRUPT; 14416603153SAndreas Noever if (ring->is_tx) { 14516603153SAndreas Noever descriptor->length = frame->size; 14616603153SAndreas Noever descriptor->eof = frame->eof; 14716603153SAndreas Noever descriptor->sof = frame->sof; 14816603153SAndreas Noever } 14916603153SAndreas Noever ring->head = (ring->head + 1) % ring->size; 15016603153SAndreas Noever ring_iowrite16desc(ring, ring->head, ring->is_tx ? 10 : 8); 15116603153SAndreas Noever } 15216603153SAndreas Noever } 15316603153SAndreas Noever 15416603153SAndreas Noever /** 15516603153SAndreas Noever * ring_work() - progress completed frames 15616603153SAndreas Noever * 15716603153SAndreas Noever * If the ring is shutting down then all frames are marked as canceled and 15816603153SAndreas Noever * their callbacks are invoked. 15916603153SAndreas Noever * 16016603153SAndreas Noever * Otherwise we collect all completed frame from the ring buffer, write new 16116603153SAndreas Noever * frame to the ring buffer and invoke the callbacks for the completed frames. 16216603153SAndreas Noever */ 16316603153SAndreas Noever static void ring_work(struct work_struct *work) 16416603153SAndreas Noever { 16516603153SAndreas Noever struct tb_ring *ring = container_of(work, typeof(*ring), work); 16616603153SAndreas Noever struct ring_frame *frame; 16716603153SAndreas Noever bool canceled = false; 16816603153SAndreas Noever LIST_HEAD(done); 16916603153SAndreas Noever mutex_lock(&ring->lock); 17016603153SAndreas Noever 17116603153SAndreas Noever if (!ring->running) { 17216603153SAndreas Noever /* Move all frames to done and mark them as canceled. */ 17316603153SAndreas Noever list_splice_tail_init(&ring->in_flight, &done); 17416603153SAndreas Noever list_splice_tail_init(&ring->queue, &done); 17516603153SAndreas Noever canceled = true; 17616603153SAndreas Noever goto invoke_callback; 17716603153SAndreas Noever } 17816603153SAndreas Noever 17916603153SAndreas Noever while (!ring_empty(ring)) { 18016603153SAndreas Noever if (!(ring->descriptors[ring->tail].flags 18116603153SAndreas Noever & RING_DESC_COMPLETED)) 18216603153SAndreas Noever break; 18316603153SAndreas Noever frame = list_first_entry(&ring->in_flight, typeof(*frame), 18416603153SAndreas Noever list); 18516603153SAndreas Noever list_move_tail(&frame->list, &done); 18616603153SAndreas Noever if (!ring->is_tx) { 18716603153SAndreas Noever frame->size = ring->descriptors[ring->tail].length; 18816603153SAndreas Noever frame->eof = ring->descriptors[ring->tail].eof; 18916603153SAndreas Noever frame->sof = ring->descriptors[ring->tail].sof; 19016603153SAndreas Noever frame->flags = ring->descriptors[ring->tail].flags; 19116603153SAndreas Noever if (frame->sof != 0) 19216603153SAndreas Noever dev_WARN(&ring->nhi->pdev->dev, 19316603153SAndreas Noever "%s %d got unexpected SOF: %#x\n", 19416603153SAndreas Noever RING_TYPE(ring), ring->hop, 19516603153SAndreas Noever frame->sof); 19616603153SAndreas Noever /* 19716603153SAndreas Noever * known flags: 19816603153SAndreas Noever * raw not enabled, interupt not set: 0x2=0010 19916603153SAndreas Noever * raw enabled: 0xa=1010 20016603153SAndreas Noever * raw not enabled: 0xb=1011 20116603153SAndreas Noever * partial frame (>MAX_FRAME_SIZE): 0xe=1110 20216603153SAndreas Noever */ 20316603153SAndreas Noever if (frame->flags != 0xa) 20416603153SAndreas Noever dev_WARN(&ring->nhi->pdev->dev, 20516603153SAndreas Noever "%s %d got unexpected flags: %#x\n", 20616603153SAndreas Noever RING_TYPE(ring), ring->hop, 20716603153SAndreas Noever frame->flags); 20816603153SAndreas Noever } 20916603153SAndreas Noever ring->tail = (ring->tail + 1) % ring->size; 21016603153SAndreas Noever } 21116603153SAndreas Noever ring_write_descriptors(ring); 21216603153SAndreas Noever 21316603153SAndreas Noever invoke_callback: 21416603153SAndreas Noever mutex_unlock(&ring->lock); /* allow callbacks to schedule new work */ 21516603153SAndreas Noever while (!list_empty(&done)) { 21616603153SAndreas Noever frame = list_first_entry(&done, typeof(*frame), list); 21716603153SAndreas Noever /* 21816603153SAndreas Noever * The callback may reenqueue or delete frame. 21916603153SAndreas Noever * Do not hold on to it. 22016603153SAndreas Noever */ 22116603153SAndreas Noever list_del_init(&frame->list); 22216603153SAndreas Noever frame->callback(ring, frame, canceled); 22316603153SAndreas Noever } 22416603153SAndreas Noever } 22516603153SAndreas Noever 22616603153SAndreas Noever int __ring_enqueue(struct tb_ring *ring, struct ring_frame *frame) 22716603153SAndreas Noever { 22816603153SAndreas Noever int ret = 0; 22916603153SAndreas Noever mutex_lock(&ring->lock); 23016603153SAndreas Noever if (ring->running) { 23116603153SAndreas Noever list_add_tail(&frame->list, &ring->queue); 23216603153SAndreas Noever ring_write_descriptors(ring); 23316603153SAndreas Noever } else { 23416603153SAndreas Noever ret = -ESHUTDOWN; 23516603153SAndreas Noever } 23616603153SAndreas Noever mutex_unlock(&ring->lock); 23716603153SAndreas Noever return ret; 23816603153SAndreas Noever } 23916603153SAndreas Noever 24016603153SAndreas Noever static struct tb_ring *ring_alloc(struct tb_nhi *nhi, u32 hop, int size, 24116603153SAndreas Noever bool transmit) 24216603153SAndreas Noever { 24316603153SAndreas Noever struct tb_ring *ring = NULL; 24416603153SAndreas Noever dev_info(&nhi->pdev->dev, "allocating %s ring %d of size %d\n", 24516603153SAndreas Noever transmit ? "TX" : "RX", hop, size); 24616603153SAndreas Noever 24716603153SAndreas Noever mutex_lock(&nhi->lock); 24816603153SAndreas Noever if (hop >= nhi->hop_count) { 24916603153SAndreas Noever dev_WARN(&nhi->pdev->dev, "invalid hop: %d\n", hop); 25016603153SAndreas Noever goto err; 25116603153SAndreas Noever } 25216603153SAndreas Noever if (transmit && nhi->tx_rings[hop]) { 25316603153SAndreas Noever dev_WARN(&nhi->pdev->dev, "TX hop %d already allocated\n", hop); 25416603153SAndreas Noever goto err; 25516603153SAndreas Noever } else if (!transmit && nhi->rx_rings[hop]) { 25616603153SAndreas Noever dev_WARN(&nhi->pdev->dev, "RX hop %d already allocated\n", hop); 25716603153SAndreas Noever goto err; 25816603153SAndreas Noever } 25916603153SAndreas Noever ring = kzalloc(sizeof(*ring), GFP_KERNEL); 26016603153SAndreas Noever if (!ring) 26116603153SAndreas Noever goto err; 26216603153SAndreas Noever 26316603153SAndreas Noever mutex_init(&ring->lock); 26416603153SAndreas Noever INIT_LIST_HEAD(&ring->queue); 26516603153SAndreas Noever INIT_LIST_HEAD(&ring->in_flight); 26616603153SAndreas Noever INIT_WORK(&ring->work, ring_work); 26716603153SAndreas Noever 26816603153SAndreas Noever ring->nhi = nhi; 26916603153SAndreas Noever ring->hop = hop; 27016603153SAndreas Noever ring->is_tx = transmit; 27116603153SAndreas Noever ring->size = size; 27216603153SAndreas Noever ring->head = 0; 27316603153SAndreas Noever ring->tail = 0; 27416603153SAndreas Noever ring->running = false; 27516603153SAndreas Noever ring->descriptors = dma_alloc_coherent(&ring->nhi->pdev->dev, 27616603153SAndreas Noever size * sizeof(*ring->descriptors), 27716603153SAndreas Noever &ring->descriptors_dma, GFP_KERNEL | __GFP_ZERO); 27816603153SAndreas Noever if (!ring->descriptors) 27916603153SAndreas Noever goto err; 28016603153SAndreas Noever 28116603153SAndreas Noever if (transmit) 28216603153SAndreas Noever nhi->tx_rings[hop] = ring; 28316603153SAndreas Noever else 28416603153SAndreas Noever nhi->rx_rings[hop] = ring; 28516603153SAndreas Noever mutex_unlock(&nhi->lock); 28616603153SAndreas Noever return ring; 28716603153SAndreas Noever 28816603153SAndreas Noever err: 28916603153SAndreas Noever if (ring) 29016603153SAndreas Noever mutex_destroy(&ring->lock); 29116603153SAndreas Noever kfree(ring); 29216603153SAndreas Noever mutex_unlock(&nhi->lock); 29316603153SAndreas Noever return NULL; 29416603153SAndreas Noever } 29516603153SAndreas Noever 29616603153SAndreas Noever struct tb_ring *ring_alloc_tx(struct tb_nhi *nhi, int hop, int size) 29716603153SAndreas Noever { 29816603153SAndreas Noever return ring_alloc(nhi, hop, size, true); 29916603153SAndreas Noever } 30016603153SAndreas Noever 30116603153SAndreas Noever struct tb_ring *ring_alloc_rx(struct tb_nhi *nhi, int hop, int size) 30216603153SAndreas Noever { 30316603153SAndreas Noever return ring_alloc(nhi, hop, size, false); 30416603153SAndreas Noever } 30516603153SAndreas Noever 30616603153SAndreas Noever /** 30716603153SAndreas Noever * ring_start() - enable a ring 30816603153SAndreas Noever * 30916603153SAndreas Noever * Must not be invoked in parallel with ring_stop(). 31016603153SAndreas Noever */ 31116603153SAndreas Noever void ring_start(struct tb_ring *ring) 31216603153SAndreas Noever { 31316603153SAndreas Noever mutex_lock(&ring->nhi->lock); 31416603153SAndreas Noever mutex_lock(&ring->lock); 31516603153SAndreas Noever if (ring->running) { 31616603153SAndreas Noever dev_WARN(&ring->nhi->pdev->dev, "ring already started\n"); 31716603153SAndreas Noever goto err; 31816603153SAndreas Noever } 31916603153SAndreas Noever dev_info(&ring->nhi->pdev->dev, "starting %s %d\n", 32016603153SAndreas Noever RING_TYPE(ring), ring->hop); 32116603153SAndreas Noever 32216603153SAndreas Noever ring_iowrite64desc(ring, ring->descriptors_dma, 0); 32316603153SAndreas Noever if (ring->is_tx) { 32416603153SAndreas Noever ring_iowrite32desc(ring, ring->size, 12); 32516603153SAndreas Noever ring_iowrite32options(ring, 0, 4); /* time releated ? */ 32616603153SAndreas Noever ring_iowrite32options(ring, 32716603153SAndreas Noever RING_FLAG_ENABLE | RING_FLAG_RAW, 0); 32816603153SAndreas Noever } else { 32916603153SAndreas Noever ring_iowrite32desc(ring, 33016603153SAndreas Noever (TB_FRAME_SIZE << 16) | ring->size, 12); 33116603153SAndreas Noever ring_iowrite32options(ring, 0xffffffff, 4); /* SOF EOF mask */ 33216603153SAndreas Noever ring_iowrite32options(ring, 33316603153SAndreas Noever RING_FLAG_ENABLE | RING_FLAG_RAW, 0); 33416603153SAndreas Noever } 33516603153SAndreas Noever ring_interrupt_active(ring, true); 33616603153SAndreas Noever ring->running = true; 33716603153SAndreas Noever err: 33816603153SAndreas Noever mutex_unlock(&ring->lock); 33916603153SAndreas Noever mutex_unlock(&ring->nhi->lock); 34016603153SAndreas Noever } 34116603153SAndreas Noever 34216603153SAndreas Noever 34316603153SAndreas Noever /** 34416603153SAndreas Noever * ring_stop() - shutdown a ring 34516603153SAndreas Noever * 34616603153SAndreas Noever * Must not be invoked from a callback. 34716603153SAndreas Noever * 34816603153SAndreas Noever * This method will disable the ring. Further calls to ring_tx/ring_rx will 34916603153SAndreas Noever * return -ESHUTDOWN until ring_stop has been called. 35016603153SAndreas Noever * 35116603153SAndreas Noever * All enqueued frames will be canceled and their callbacks will be executed 35216603153SAndreas Noever * with frame->canceled set to true (on the callback thread). This method 35316603153SAndreas Noever * returns only after all callback invocations have finished. 35416603153SAndreas Noever */ 35516603153SAndreas Noever void ring_stop(struct tb_ring *ring) 35616603153SAndreas Noever { 35716603153SAndreas Noever mutex_lock(&ring->nhi->lock); 35816603153SAndreas Noever mutex_lock(&ring->lock); 35916603153SAndreas Noever dev_info(&ring->nhi->pdev->dev, "stopping %s %d\n", 36016603153SAndreas Noever RING_TYPE(ring), ring->hop); 36116603153SAndreas Noever if (!ring->running) { 36216603153SAndreas Noever dev_WARN(&ring->nhi->pdev->dev, "%s %d already stopped\n", 36316603153SAndreas Noever RING_TYPE(ring), ring->hop); 36416603153SAndreas Noever goto err; 36516603153SAndreas Noever } 36616603153SAndreas Noever ring_interrupt_active(ring, false); 36716603153SAndreas Noever 36816603153SAndreas Noever ring_iowrite32options(ring, 0, 0); 36916603153SAndreas Noever ring_iowrite64desc(ring, 0, 0); 37016603153SAndreas Noever ring_iowrite16desc(ring, 0, ring->is_tx ? 10 : 8); 37116603153SAndreas Noever ring_iowrite32desc(ring, 0, 12); 37216603153SAndreas Noever ring->head = 0; 37316603153SAndreas Noever ring->tail = 0; 37416603153SAndreas Noever ring->running = false; 37516603153SAndreas Noever 37616603153SAndreas Noever err: 37716603153SAndreas Noever mutex_unlock(&ring->lock); 37816603153SAndreas Noever mutex_unlock(&ring->nhi->lock); 37916603153SAndreas Noever 38016603153SAndreas Noever /* 38116603153SAndreas Noever * schedule ring->work to invoke callbacks on all remaining frames. 38216603153SAndreas Noever */ 38316603153SAndreas Noever schedule_work(&ring->work); 38416603153SAndreas Noever flush_work(&ring->work); 38516603153SAndreas Noever } 38616603153SAndreas Noever 38716603153SAndreas Noever /* 38816603153SAndreas Noever * ring_free() - free ring 38916603153SAndreas Noever * 39016603153SAndreas Noever * When this method returns all invocations of ring->callback will have 39116603153SAndreas Noever * finished. 39216603153SAndreas Noever * 39316603153SAndreas Noever * Ring must be stopped. 39416603153SAndreas Noever * 39516603153SAndreas Noever * Must NOT be called from ring_frame->callback! 39616603153SAndreas Noever */ 39716603153SAndreas Noever void ring_free(struct tb_ring *ring) 39816603153SAndreas Noever { 39916603153SAndreas Noever mutex_lock(&ring->nhi->lock); 40016603153SAndreas Noever /* 40116603153SAndreas Noever * Dissociate the ring from the NHI. This also ensures that 40216603153SAndreas Noever * nhi_interrupt_work cannot reschedule ring->work. 40316603153SAndreas Noever */ 40416603153SAndreas Noever if (ring->is_tx) 40516603153SAndreas Noever ring->nhi->tx_rings[ring->hop] = NULL; 40616603153SAndreas Noever else 40716603153SAndreas Noever ring->nhi->rx_rings[ring->hop] = NULL; 40816603153SAndreas Noever 40916603153SAndreas Noever if (ring->running) { 41016603153SAndreas Noever dev_WARN(&ring->nhi->pdev->dev, "%s %d still running\n", 41116603153SAndreas Noever RING_TYPE(ring), ring->hop); 41216603153SAndreas Noever } 41316603153SAndreas Noever 41416603153SAndreas Noever dma_free_coherent(&ring->nhi->pdev->dev, 41516603153SAndreas Noever ring->size * sizeof(*ring->descriptors), 41616603153SAndreas Noever ring->descriptors, ring->descriptors_dma); 41716603153SAndreas Noever 41816603153SAndreas Noever ring->descriptors = 0; 41916603153SAndreas Noever ring->descriptors_dma = 0; 42016603153SAndreas Noever 42116603153SAndreas Noever 42216603153SAndreas Noever dev_info(&ring->nhi->pdev->dev, 42316603153SAndreas Noever "freeing %s %d\n", 42416603153SAndreas Noever RING_TYPE(ring), 42516603153SAndreas Noever ring->hop); 42616603153SAndreas Noever 42716603153SAndreas Noever mutex_unlock(&ring->nhi->lock); 42816603153SAndreas Noever /** 42916603153SAndreas Noever * ring->work can no longer be scheduled (it is scheduled only by 43016603153SAndreas Noever * nhi_interrupt_work and ring_stop). Wait for it to finish before 43116603153SAndreas Noever * freeing the ring. 43216603153SAndreas Noever */ 43316603153SAndreas Noever flush_work(&ring->work); 43416603153SAndreas Noever mutex_destroy(&ring->lock); 43516603153SAndreas Noever kfree(ring); 43616603153SAndreas Noever } 43716603153SAndreas Noever 43816603153SAndreas Noever static void nhi_interrupt_work(struct work_struct *work) 43916603153SAndreas Noever { 44016603153SAndreas Noever struct tb_nhi *nhi = container_of(work, typeof(*nhi), interrupt_work); 44116603153SAndreas Noever int value = 0; /* Suppress uninitialized usage warning. */ 44216603153SAndreas Noever int bit; 44316603153SAndreas Noever int hop = -1; 44416603153SAndreas Noever int type = 0; /* current interrupt type 0: TX, 1: RX, 2: RX overflow */ 44516603153SAndreas Noever struct tb_ring *ring; 44616603153SAndreas Noever 44716603153SAndreas Noever mutex_lock(&nhi->lock); 44816603153SAndreas Noever 44916603153SAndreas Noever /* 45016603153SAndreas Noever * Starting at REG_RING_NOTIFY_BASE there are three status bitfields 45116603153SAndreas Noever * (TX, RX, RX overflow). We iterate over the bits and read a new 45216603153SAndreas Noever * dwords as required. The registers are cleared on read. 45316603153SAndreas Noever */ 45416603153SAndreas Noever for (bit = 0; bit < 3 * nhi->hop_count; bit++) { 45516603153SAndreas Noever if (bit % 32 == 0) 45616603153SAndreas Noever value = ioread32(nhi->iobase 45716603153SAndreas Noever + REG_RING_NOTIFY_BASE 45816603153SAndreas Noever + 4 * (bit / 32)); 45916603153SAndreas Noever if (++hop == nhi->hop_count) { 46016603153SAndreas Noever hop = 0; 46116603153SAndreas Noever type++; 46216603153SAndreas Noever } 46316603153SAndreas Noever if ((value & (1 << (bit % 32))) == 0) 46416603153SAndreas Noever continue; 46516603153SAndreas Noever if (type == 2) { 46616603153SAndreas Noever dev_warn(&nhi->pdev->dev, 46716603153SAndreas Noever "RX overflow for ring %d\n", 46816603153SAndreas Noever hop); 46916603153SAndreas Noever continue; 47016603153SAndreas Noever } 47116603153SAndreas Noever if (type == 0) 47216603153SAndreas Noever ring = nhi->tx_rings[hop]; 47316603153SAndreas Noever else 47416603153SAndreas Noever ring = nhi->rx_rings[hop]; 47516603153SAndreas Noever if (ring == NULL) { 47616603153SAndreas Noever dev_warn(&nhi->pdev->dev, 47716603153SAndreas Noever "got interrupt for inactive %s ring %d\n", 47816603153SAndreas Noever type ? "RX" : "TX", 47916603153SAndreas Noever hop); 48016603153SAndreas Noever continue; 48116603153SAndreas Noever } 48216603153SAndreas Noever /* we do not check ring->running, this is done in ring->work */ 48316603153SAndreas Noever schedule_work(&ring->work); 48416603153SAndreas Noever } 48516603153SAndreas Noever mutex_unlock(&nhi->lock); 48616603153SAndreas Noever } 48716603153SAndreas Noever 48816603153SAndreas Noever static irqreturn_t nhi_msi(int irq, void *data) 48916603153SAndreas Noever { 49016603153SAndreas Noever struct tb_nhi *nhi = data; 49116603153SAndreas Noever schedule_work(&nhi->interrupt_work); 49216603153SAndreas Noever return IRQ_HANDLED; 49316603153SAndreas Noever } 49416603153SAndreas Noever 49516603153SAndreas Noever static void nhi_shutdown(struct tb_nhi *nhi) 49616603153SAndreas Noever { 49716603153SAndreas Noever int i; 49816603153SAndreas Noever dev_info(&nhi->pdev->dev, "shutdown\n"); 49916603153SAndreas Noever 50016603153SAndreas Noever for (i = 0; i < nhi->hop_count; i++) { 50116603153SAndreas Noever if (nhi->tx_rings[i]) 50216603153SAndreas Noever dev_WARN(&nhi->pdev->dev, 50316603153SAndreas Noever "TX ring %d is still active\n", i); 50416603153SAndreas Noever if (nhi->rx_rings[i]) 50516603153SAndreas Noever dev_WARN(&nhi->pdev->dev, 50616603153SAndreas Noever "RX ring %d is still active\n", i); 50716603153SAndreas Noever } 50816603153SAndreas Noever nhi_disable_interrupts(nhi); 50916603153SAndreas Noever /* 51016603153SAndreas Noever * We have to release the irq before calling flush_work. Otherwise an 51116603153SAndreas Noever * already executing IRQ handler could call schedule_work again. 51216603153SAndreas Noever */ 51316603153SAndreas Noever devm_free_irq(&nhi->pdev->dev, nhi->pdev->irq, nhi); 51416603153SAndreas Noever flush_work(&nhi->interrupt_work); 51516603153SAndreas Noever mutex_destroy(&nhi->lock); 51616603153SAndreas Noever } 51716603153SAndreas Noever 51816603153SAndreas Noever static int nhi_probe(struct pci_dev *pdev, const struct pci_device_id *id) 51916603153SAndreas Noever { 52016603153SAndreas Noever struct tb_nhi *nhi; 521*d6cc51cdSAndreas Noever struct tb *tb; 52216603153SAndreas Noever int res; 52316603153SAndreas Noever 52416603153SAndreas Noever res = pcim_enable_device(pdev); 52516603153SAndreas Noever if (res) { 52616603153SAndreas Noever dev_err(&pdev->dev, "cannot enable PCI device, aborting\n"); 52716603153SAndreas Noever return res; 52816603153SAndreas Noever } 52916603153SAndreas Noever 53016603153SAndreas Noever res = pci_enable_msi(pdev); 53116603153SAndreas Noever if (res) { 53216603153SAndreas Noever dev_err(&pdev->dev, "cannot enable MSI, aborting\n"); 53316603153SAndreas Noever return res; 53416603153SAndreas Noever } 53516603153SAndreas Noever 53616603153SAndreas Noever res = pcim_iomap_regions(pdev, 1 << 0, "thunderbolt"); 53716603153SAndreas Noever if (res) { 53816603153SAndreas Noever dev_err(&pdev->dev, "cannot obtain PCI resources, aborting\n"); 53916603153SAndreas Noever return res; 54016603153SAndreas Noever } 54116603153SAndreas Noever 54216603153SAndreas Noever nhi = devm_kzalloc(&pdev->dev, sizeof(*nhi), GFP_KERNEL); 54316603153SAndreas Noever if (!nhi) 54416603153SAndreas Noever return -ENOMEM; 54516603153SAndreas Noever 54616603153SAndreas Noever nhi->pdev = pdev; 54716603153SAndreas Noever /* cannot fail - table is allocated bin pcim_iomap_regions */ 54816603153SAndreas Noever nhi->iobase = pcim_iomap_table(pdev)[0]; 54916603153SAndreas Noever nhi->hop_count = ioread32(nhi->iobase + REG_HOP_COUNT) & 0x3ff; 55016603153SAndreas Noever if (nhi->hop_count != 12) 55116603153SAndreas Noever dev_warn(&pdev->dev, "unexpected hop count: %d\n", 55216603153SAndreas Noever nhi->hop_count); 55316603153SAndreas Noever INIT_WORK(&nhi->interrupt_work, nhi_interrupt_work); 55416603153SAndreas Noever 55516603153SAndreas Noever nhi->tx_rings = devm_kzalloc(&pdev->dev, 55616603153SAndreas Noever nhi->hop_count * sizeof(struct tb_ring), 55716603153SAndreas Noever GFP_KERNEL); 55816603153SAndreas Noever nhi->rx_rings = devm_kzalloc(&pdev->dev, 55916603153SAndreas Noever nhi->hop_count * sizeof(struct tb_ring), 56016603153SAndreas Noever GFP_KERNEL); 56116603153SAndreas Noever if (!nhi->tx_rings || !nhi->rx_rings) 56216603153SAndreas Noever return -ENOMEM; 56316603153SAndreas Noever 56416603153SAndreas Noever nhi_disable_interrupts(nhi); /* In case someone left them on. */ 56516603153SAndreas Noever res = devm_request_irq(&pdev->dev, pdev->irq, nhi_msi, 56616603153SAndreas Noever IRQF_NO_SUSPEND, /* must work during _noirq */ 56716603153SAndreas Noever "thunderbolt", nhi); 56816603153SAndreas Noever if (res) { 56916603153SAndreas Noever dev_err(&pdev->dev, "request_irq failed, aborting\n"); 57016603153SAndreas Noever return res; 57116603153SAndreas Noever } 57216603153SAndreas Noever 57316603153SAndreas Noever mutex_init(&nhi->lock); 57416603153SAndreas Noever 57516603153SAndreas Noever pci_set_master(pdev); 57616603153SAndreas Noever 57716603153SAndreas Noever /* magic value - clock related? */ 57816603153SAndreas Noever iowrite32(3906250 / 10000, nhi->iobase + 0x38c00); 57916603153SAndreas Noever 580*d6cc51cdSAndreas Noever dev_info(&nhi->pdev->dev, "NHI initialized, starting thunderbolt\n"); 581*d6cc51cdSAndreas Noever tb = thunderbolt_alloc_and_start(nhi); 582*d6cc51cdSAndreas Noever if (!tb) { 583*d6cc51cdSAndreas Noever /* 584*d6cc51cdSAndreas Noever * At this point the RX/TX rings might already have been 585*d6cc51cdSAndreas Noever * activated. Do a proper shutdown. 586*d6cc51cdSAndreas Noever */ 587*d6cc51cdSAndreas Noever nhi_shutdown(nhi); 588*d6cc51cdSAndreas Noever return -EIO; 589*d6cc51cdSAndreas Noever } 590*d6cc51cdSAndreas Noever pci_set_drvdata(pdev, tb); 59116603153SAndreas Noever 59216603153SAndreas Noever return 0; 59316603153SAndreas Noever } 59416603153SAndreas Noever 59516603153SAndreas Noever static void nhi_remove(struct pci_dev *pdev) 59616603153SAndreas Noever { 597*d6cc51cdSAndreas Noever struct tb *tb = pci_get_drvdata(pdev); 598*d6cc51cdSAndreas Noever struct tb_nhi *nhi = tb->nhi; 599*d6cc51cdSAndreas Noever thunderbolt_shutdown_and_free(tb); 60016603153SAndreas Noever nhi_shutdown(nhi); 60116603153SAndreas Noever } 60216603153SAndreas Noever 60316603153SAndreas Noever struct pci_device_id nhi_ids[] = { 60416603153SAndreas Noever /* 60516603153SAndreas Noever * We have to specify class, the TB bridges use the same device and 60616603153SAndreas Noever * vendor (sub)id. 60716603153SAndreas Noever */ 60816603153SAndreas Noever { 60916603153SAndreas Noever .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, 61016603153SAndreas Noever .vendor = PCI_VENDOR_ID_INTEL, .device = 0x1547, 61116603153SAndreas Noever .subvendor = 0x2222, .subdevice = 0x1111, 61216603153SAndreas Noever }, 61316603153SAndreas Noever { 61416603153SAndreas Noever .class = PCI_CLASS_SYSTEM_OTHER << 8, .class_mask = ~0, 61516603153SAndreas Noever .vendor = PCI_VENDOR_ID_INTEL, .device = 0x156c, 61616603153SAndreas Noever .subvendor = 0x2222, .subdevice = 0x1111, 61716603153SAndreas Noever }, 61816603153SAndreas Noever { 0,} 61916603153SAndreas Noever }; 62016603153SAndreas Noever 62116603153SAndreas Noever MODULE_DEVICE_TABLE(pci, nhi_ids); 62216603153SAndreas Noever MODULE_LICENSE("GPL"); 62316603153SAndreas Noever 62416603153SAndreas Noever static struct pci_driver nhi_driver = { 62516603153SAndreas Noever .name = "thunderbolt", 62616603153SAndreas Noever .id_table = nhi_ids, 62716603153SAndreas Noever .probe = nhi_probe, 62816603153SAndreas Noever .remove = nhi_remove, 62916603153SAndreas Noever }; 63016603153SAndreas Noever 63116603153SAndreas Noever static int __init nhi_init(void) 63216603153SAndreas Noever { 63316603153SAndreas Noever if (!dmi_match(DMI_BOARD_VENDOR, "Apple Inc.")) 63416603153SAndreas Noever return -ENOSYS; 63516603153SAndreas Noever return pci_register_driver(&nhi_driver); 63616603153SAndreas Noever } 63716603153SAndreas Noever 63816603153SAndreas Noever static void __exit nhi_unload(void) 63916603153SAndreas Noever { 64016603153SAndreas Noever pci_unregister_driver(&nhi_driver); 64116603153SAndreas Noever } 64216603153SAndreas Noever 64316603153SAndreas Noever module_init(nhi_init); 64416603153SAndreas Noever module_exit(nhi_unload); 645