Lines Matching +full:fifo +full:- +full:size

1 // SPDX-License-Identifier: GPL-2.0+
24 #include "mlxbf-tmfifo-regs.h"
26 /* Vring size. */
29 /* Console Tx buffer size. */
35 /* House-keeping timer interval. */
38 /* Virtual devices sharing the TM FIFO. */
53 /* ACPI UID for BlueField-3. */
59 * struct mlxbf_tmfifo_vring - Structure of the TmFifo virtual ring
71 * @num: vring size (number of descriptors)
72 * @align: vring alignment size
76 * @fifo: pointer to the tmfifo structure
95 struct mlxbf_tmfifo *fifo; member
101 (r->desc_head == &r->drop_desc ? true : false); })
123 * struct mlxbf_tmfifo_vdev - Structure of the TmFifo virtual device
129 * @config: non-anonymous union for cons and net
130 * @config.cons: virtual console config -
132 * @config.net: virtual network config -
134 * @tx_buf: tx buffer used to buffer data before writing into the FIFO
149 * struct mlxbf_tmfifo_irq_info - Structure of the interrupt information
150 * @fifo: pointer to the tmfifo structure
155 struct mlxbf_tmfifo *fifo; member
161 * struct mlxbf_tmfifo_io - Structure of the TmFifo IO resource (for both rx & tx)
173 * struct mlxbf_tmfifo - Structure of the TmFifo
180 * @rx_fifo_size: number of entries of the Rx FIFO
181 * @tx_fifo_size: number of entries of the Tx FIFO
209 * struct mlxbf_tmfifo_msg_hdr - Structure of the TmFifo message header
211 * @len: payload length in network byte order. Messages sent into the FIFO
238 /* Supported virtio-net features. */
245 /* Free vrings of the FIFO device. */
246 static void mlxbf_tmfifo_free_vrings(struct mlxbf_tmfifo *fifo, in mlxbf_tmfifo_free_vrings() argument
250 int i, size; in mlxbf_tmfifo_free_vrings() local
252 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { in mlxbf_tmfifo_free_vrings()
253 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_free_vrings()
254 if (vring->va) { in mlxbf_tmfifo_free_vrings()
255 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_free_vrings()
256 dma_free_coherent(tm_vdev->vdev.dev.parent, size, in mlxbf_tmfifo_free_vrings()
257 vring->va, vring->dma); in mlxbf_tmfifo_free_vrings()
258 vring->va = NULL; in mlxbf_tmfifo_free_vrings()
259 if (vring->vq) { in mlxbf_tmfifo_free_vrings()
260 vring_del_virtqueue(vring->vq); in mlxbf_tmfifo_free_vrings()
261 vring->vq = NULL; in mlxbf_tmfifo_free_vrings()
267 /* Allocate vrings for the FIFO. */
268 static int mlxbf_tmfifo_alloc_vrings(struct mlxbf_tmfifo *fifo, in mlxbf_tmfifo_alloc_vrings() argument
274 int i, size; in mlxbf_tmfifo_alloc_vrings() local
277 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { in mlxbf_tmfifo_alloc_vrings()
278 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_alloc_vrings()
279 vring->fifo = fifo; in mlxbf_tmfifo_alloc_vrings()
280 vring->num = MLXBF_TMFIFO_VRING_SIZE; in mlxbf_tmfifo_alloc_vrings()
281 vring->align = SMP_CACHE_BYTES; in mlxbf_tmfifo_alloc_vrings()
282 vring->index = i; in mlxbf_tmfifo_alloc_vrings()
283 vring->vdev_id = tm_vdev->vdev.id.device; in mlxbf_tmfifo_alloc_vrings()
284 vring->drop_desc.len = VRING_DROP_DESC_MAX_LEN; in mlxbf_tmfifo_alloc_vrings()
285 dev = &tm_vdev->vdev.dev; in mlxbf_tmfifo_alloc_vrings()
287 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_alloc_vrings()
288 va = dma_alloc_coherent(dev->parent, size, &dma, GFP_KERNEL); in mlxbf_tmfifo_alloc_vrings()
290 mlxbf_tmfifo_free_vrings(fifo, tm_vdev); in mlxbf_tmfifo_alloc_vrings()
291 dev_err(dev->parent, "dma_alloc_coherent failed\n"); in mlxbf_tmfifo_alloc_vrings()
292 return -ENOMEM; in mlxbf_tmfifo_alloc_vrings()
295 vring->va = va; in mlxbf_tmfifo_alloc_vrings()
296 vring->dma = dma; in mlxbf_tmfifo_alloc_vrings()
302 /* Disable interrupts of the FIFO device. */
303 static void mlxbf_tmfifo_disable_irqs(struct mlxbf_tmfifo *fifo) in mlxbf_tmfifo_disable_irqs() argument
308 irq = fifo->irq_info[i].irq; in mlxbf_tmfifo_disable_irqs()
309 fifo->irq_info[i].irq = 0; in mlxbf_tmfifo_disable_irqs()
319 if (!test_and_set_bit(irq_info->index, &irq_info->fifo->pend_events)) in mlxbf_tmfifo_irq_handler()
320 schedule_work(&irq_info->fifo->work); in mlxbf_tmfifo_irq_handler()
329 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_get_next_desc()
330 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_get_next_desc()
333 if (vring->next_avail == virtio16_to_cpu(vdev, vr->avail->idx)) in mlxbf_tmfifo_get_next_desc()
336 /* Make sure 'avail->idx' is visible already. */ in mlxbf_tmfifo_get_next_desc()
339 idx = vring->next_avail % vr->num; in mlxbf_tmfifo_get_next_desc()
340 head = virtio16_to_cpu(vdev, vr->avail->ring[idx]); in mlxbf_tmfifo_get_next_desc()
341 if (WARN_ON(head >= vr->num)) in mlxbf_tmfifo_get_next_desc()
344 vring->next_avail++; in mlxbf_tmfifo_get_next_desc()
346 return &vr->desc[head]; in mlxbf_tmfifo_get_next_desc()
353 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_release_desc()
354 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_release_desc()
357 vr_idx = virtio16_to_cpu(vdev, vr->used->idx); in mlxbf_tmfifo_release_desc()
358 idx = vr_idx % vr->num; in mlxbf_tmfifo_release_desc()
359 vr->used->ring[idx].id = cpu_to_virtio32(vdev, desc - vr->desc); in mlxbf_tmfifo_release_desc()
360 vr->used->ring[idx].len = cpu_to_virtio32(vdev, len); in mlxbf_tmfifo_release_desc()
368 vr->used->idx = cpu_to_virtio16(vdev, vr_idx + 1); in mlxbf_tmfifo_release_desc()
375 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_get_pkt_len()
376 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_get_pkt_len()
380 len += virtio32_to_cpu(vdev, desc->len); in mlxbf_tmfifo_get_pkt_len()
381 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) in mlxbf_tmfifo_get_pkt_len()
383 idx = virtio16_to_cpu(vdev, desc->next); in mlxbf_tmfifo_get_pkt_len()
384 desc = &vr->desc[idx]; in mlxbf_tmfifo_get_pkt_len()
395 if (vring->desc_head) { in mlxbf_tmfifo_release_pkt()
396 desc_head = vring->desc_head; in mlxbf_tmfifo_release_pkt()
397 len = vring->pkt_len; in mlxbf_tmfifo_release_pkt()
406 vring->pkt_len = 0; in mlxbf_tmfifo_release_pkt()
407 vring->desc = NULL; in mlxbf_tmfifo_release_pkt()
408 vring->desc_head = NULL; in mlxbf_tmfifo_release_pkt()
414 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_init_net_desc()
417 net_hdr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr)); in mlxbf_tmfifo_init_net_desc()
428 if (desc && is_rx && vring->vdev_id == VIRTIO_ID_NET) in mlxbf_tmfifo_get_next_pkt()
431 vring->desc_head = desc; in mlxbf_tmfifo_get_next_pkt()
432 vring->desc = desc; in mlxbf_tmfifo_get_next_pkt()
437 /* House-keeping timer. */
440 struct mlxbf_tmfifo *fifo = container_of(t, struct mlxbf_tmfifo, timer); in mlxbf_tmfifo_timer() local
443 rx = !test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events); in mlxbf_tmfifo_timer()
444 tx = !test_and_set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events); in mlxbf_tmfifo_timer()
447 schedule_work(&fifo->work); in mlxbf_tmfifo_timer()
449 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL); in mlxbf_tmfifo_timer()
457 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_console_output_one()
458 struct virtio_device *vdev = &cons->vdev; in mlxbf_tmfifo_console_output_one()
463 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr)); in mlxbf_tmfifo_console_output_one()
464 len = virtio32_to_cpu(vdev, desc->len); in mlxbf_tmfifo_console_output_one()
466 seg = CIRC_SPACE_TO_END(cons->tx_buf.head, cons->tx_buf.tail, in mlxbf_tmfifo_console_output_one()
469 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, len); in mlxbf_tmfifo_console_output_one()
471 memcpy(cons->tx_buf.buf + cons->tx_buf.head, addr, seg); in mlxbf_tmfifo_console_output_one()
473 memcpy(cons->tx_buf.buf, addr, len - seg); in mlxbf_tmfifo_console_output_one()
475 cons->tx_buf.head = (cons->tx_buf.head + len) % in mlxbf_tmfifo_console_output_one()
478 if (!(virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) in mlxbf_tmfifo_console_output_one()
480 idx = virtio16_to_cpu(vdev, desc->next); in mlxbf_tmfifo_console_output_one()
481 desc = &vr->desc[idx]; in mlxbf_tmfifo_console_output_one()
496 avail = CIRC_SPACE(cons->tx_buf.head, cons->tx_buf.tail, in mlxbf_tmfifo_console_output()
509 /* Get the number of available words in Rx FIFO for receiving. */
510 static int mlxbf_tmfifo_get_rx_avail(struct mlxbf_tmfifo *fifo) in mlxbf_tmfifo_get_rx_avail() argument
514 sts = readq(fifo->rx.sts); in mlxbf_tmfifo_get_rx_avail()
519 static int mlxbf_tmfifo_get_tx_avail(struct mlxbf_tmfifo *fifo, int vdev_id) in mlxbf_tmfifo_get_tx_avail() argument
525 /* Reserve some room in FIFO for console messages. */ in mlxbf_tmfifo_get_tx_avail()
527 tx_reserve = fifo->tx_fifo_size / MLXBF_TMFIFO_RESERVE_RATIO; in mlxbf_tmfifo_get_tx_avail()
531 sts = readq(fifo->tx.sts); in mlxbf_tmfifo_get_tx_avail()
533 return fifo->tx_fifo_size - tx_reserve - count; in mlxbf_tmfifo_get_tx_avail()
537 static void mlxbf_tmfifo_console_tx(struct mlxbf_tmfifo *fifo, int avail) in mlxbf_tmfifo_console_tx() argument
542 int size, seg; in mlxbf_tmfifo_console_tx() local
550 cons = fifo->vdev[VIRTIO_ID_CONSOLE]; in mlxbf_tmfifo_console_tx()
551 if (!cons || !cons->tx_buf.buf) in mlxbf_tmfifo_console_tx()
555 size = CIRC_CNT(cons->tx_buf.head, cons->tx_buf.tail, in mlxbf_tmfifo_console_tx()
557 if (size == 0) in mlxbf_tmfifo_console_tx()
560 /* Adjust the size to available space. */ in mlxbf_tmfifo_console_tx()
561 if (size + sizeof(hdr) > avail * sizeof(u64)) in mlxbf_tmfifo_console_tx()
562 size = avail * sizeof(u64) - sizeof(hdr); in mlxbf_tmfifo_console_tx()
566 hdr.len = htons(size); in mlxbf_tmfifo_console_tx()
567 writeq(*(u64 *)&hdr, fifo->tx.data); in mlxbf_tmfifo_console_tx()
569 /* Use spin-lock to protect the 'cons->tx_buf'. */ in mlxbf_tmfifo_console_tx()
570 spin_lock_irqsave(&fifo->spin_lock[0], flags); in mlxbf_tmfifo_console_tx()
572 while (size > 0) { in mlxbf_tmfifo_console_tx()
573 addr = cons->tx_buf.buf + cons->tx_buf.tail; in mlxbf_tmfifo_console_tx()
575 seg = CIRC_CNT_TO_END(cons->tx_buf.head, cons->tx_buf.tail, in mlxbf_tmfifo_console_tx()
581 memcpy((u8 *)&data + seg, cons->tx_buf.buf, in mlxbf_tmfifo_console_tx()
582 sizeof(u64) - seg); in mlxbf_tmfifo_console_tx()
584 writeq(data, fifo->tx.data); in mlxbf_tmfifo_console_tx()
586 if (size >= sizeof(u64)) { in mlxbf_tmfifo_console_tx()
587 cons->tx_buf.tail = (cons->tx_buf.tail + sizeof(u64)) % in mlxbf_tmfifo_console_tx()
589 size -= sizeof(u64); in mlxbf_tmfifo_console_tx()
591 cons->tx_buf.tail = (cons->tx_buf.tail + size) % in mlxbf_tmfifo_console_tx()
593 size = 0; in mlxbf_tmfifo_console_tx()
597 spin_unlock_irqrestore(&fifo->spin_lock[0], flags); in mlxbf_tmfifo_console_tx()
605 struct virtio_device *vdev = vring->vq->vdev; in mlxbf_tmfifo_rxtx_word()
606 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_word() local
611 addr = phys_to_virt(virtio64_to_cpu(vdev, desc->addr)); in mlxbf_tmfifo_rxtx_word()
613 /* Read a word from FIFO for Rx. */ in mlxbf_tmfifo_rxtx_word()
615 data = readq(fifo->rx.data); in mlxbf_tmfifo_rxtx_word()
617 if (vring->cur_len + sizeof(u64) <= len) { in mlxbf_tmfifo_rxtx_word()
621 memcpy(addr + vring->cur_len, &data, in mlxbf_tmfifo_rxtx_word()
624 memcpy(&data, addr + vring->cur_len, in mlxbf_tmfifo_rxtx_word()
627 vring->cur_len += sizeof(u64); in mlxbf_tmfifo_rxtx_word()
632 memcpy(addr + vring->cur_len, &data, in mlxbf_tmfifo_rxtx_word()
633 len - vring->cur_len); in mlxbf_tmfifo_rxtx_word()
636 memcpy(&data, addr + vring->cur_len, in mlxbf_tmfifo_rxtx_word()
637 len - vring->cur_len); in mlxbf_tmfifo_rxtx_word()
639 vring->cur_len = len; in mlxbf_tmfifo_rxtx_word()
642 /* Write the word into FIFO for Tx. */ in mlxbf_tmfifo_rxtx_word()
644 writeq(data, fifo->tx.data); in mlxbf_tmfifo_rxtx_word()
658 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_header() local
666 /* Drain one word from the FIFO. */ in mlxbf_tmfifo_rxtx_header()
667 *(u64 *)&hdr = readq(fifo->rx.data); in mlxbf_tmfifo_rxtx_header()
677 config = &fifo->vdev[vdev_id]->config.net; in mlxbf_tmfifo_rxtx_header()
678 /* A legacy-only interface for now. */ in mlxbf_tmfifo_rxtx_header()
681 config->mtu) + in mlxbf_tmfifo_rxtx_header()
693 if (vdev_id != vring->vdev_id) { in mlxbf_tmfifo_rxtx_header()
694 struct mlxbf_tmfifo_vdev *tm_dev2 = fifo->vdev[vdev_id]; in mlxbf_tmfifo_rxtx_header()
698 vring->desc = *desc; in mlxbf_tmfifo_rxtx_header()
699 vring = &tm_dev2->vrings[MLXBF_TMFIFO_VRING_RX]; in mlxbf_tmfifo_rxtx_header()
704 if (vring->desc_head) in mlxbf_tmfifo_rxtx_header()
706 *desc = &vring->drop_desc; in mlxbf_tmfifo_rxtx_header()
707 vring->desc_head = *desc; in mlxbf_tmfifo_rxtx_header()
708 vring->desc = *desc; in mlxbf_tmfifo_rxtx_header()
711 vring->pkt_len = ntohs(hdr.len) + hdr_len; in mlxbf_tmfifo_rxtx_header()
714 hdr_len = (vring->vdev_id == VIRTIO_ID_NET) ? in mlxbf_tmfifo_rxtx_header()
716 vring->pkt_len = mlxbf_tmfifo_get_pkt_len(vring, *desc); in mlxbf_tmfifo_rxtx_header()
717 hdr.type = (vring->vdev_id == VIRTIO_ID_NET) ? in mlxbf_tmfifo_rxtx_header()
719 hdr.len = htons(vring->pkt_len - hdr_len); in mlxbf_tmfifo_rxtx_header()
720 writeq(*(u64 *)&hdr, fifo->tx.data); in mlxbf_tmfifo_rxtx_header()
723 vring->cur_len = hdr_len; in mlxbf_tmfifo_rxtx_header()
724 vring->rem_len = vring->pkt_len; in mlxbf_tmfifo_rxtx_header()
725 fifo->vring[is_rx] = vring; in mlxbf_tmfifo_rxtx_header()
736 const struct vring *vr = virtqueue_get_vring(vring->vq); in mlxbf_tmfifo_rxtx_one_desc()
737 struct mlxbf_tmfifo *fifo = vring->fifo; in mlxbf_tmfifo_rxtx_one_desc() local
744 vdev = &fifo->vdev[vring->vdev_id]->vdev; in mlxbf_tmfifo_rxtx_one_desc()
747 if (!vring->desc) { in mlxbf_tmfifo_rxtx_one_desc()
752 desc = &vring->drop_desc; in mlxbf_tmfifo_rxtx_one_desc()
753 vring->desc_head = desc; in mlxbf_tmfifo_rxtx_one_desc()
754 vring->desc = desc; in mlxbf_tmfifo_rxtx_one_desc()
760 desc = vring->desc; in mlxbf_tmfifo_rxtx_one_desc()
764 if (vring->pkt_len == 0) { in mlxbf_tmfifo_rxtx_one_desc()
766 (*avail)--; in mlxbf_tmfifo_rxtx_one_desc()
775 len = virtio32_to_cpu(vdev, desc->len); in mlxbf_tmfifo_rxtx_one_desc()
776 if (len > vring->rem_len) in mlxbf_tmfifo_rxtx_one_desc()
777 len = vring->rem_len; in mlxbf_tmfifo_rxtx_one_desc()
780 if (vring->cur_len < len) { in mlxbf_tmfifo_rxtx_one_desc()
782 (*avail)--; in mlxbf_tmfifo_rxtx_one_desc()
786 if (vring->cur_len == len) { in mlxbf_tmfifo_rxtx_one_desc()
787 vring->cur_len = 0; in mlxbf_tmfifo_rxtx_one_desc()
788 vring->rem_len -= len; in mlxbf_tmfifo_rxtx_one_desc()
791 if (!IS_VRING_DROP(vring) && vring->rem_len > 0 && in mlxbf_tmfifo_rxtx_one_desc()
792 (virtio16_to_cpu(vdev, desc->flags) & VRING_DESC_F_NEXT)) { in mlxbf_tmfifo_rxtx_one_desc()
793 idx = virtio16_to_cpu(vdev, desc->next); in mlxbf_tmfifo_rxtx_one_desc()
794 desc = &vr->desc[idx]; in mlxbf_tmfifo_rxtx_one_desc()
800 fifo->vring[is_rx] = NULL; in mlxbf_tmfifo_rxtx_one_desc()
804 vring->pkt_len = 0; in mlxbf_tmfifo_rxtx_one_desc()
805 vring->desc_head = NULL; in mlxbf_tmfifo_rxtx_one_desc()
806 vring->desc = NULL; in mlxbf_tmfifo_rxtx_one_desc()
817 spin_lock_irqsave(&fifo->spin_lock[is_rx], flags); in mlxbf_tmfifo_rxtx_one_desc()
818 vring_interrupt(0, vring->vq); in mlxbf_tmfifo_rxtx_one_desc()
819 spin_unlock_irqrestore(&fifo->spin_lock[is_rx], flags); in mlxbf_tmfifo_rxtx_one_desc()
824 vring->desc = desc; in mlxbf_tmfifo_rxtx_one_desc()
834 if (vring->vdev_id != VIRTIO_ID_NET) in mlxbf_tmfifo_check_tx_timeout()
838 if (!vring->tx_timeout) { in mlxbf_tmfifo_check_tx_timeout()
840 vring->tx_timeout = jiffies + in mlxbf_tmfifo_check_tx_timeout()
843 } else if (time_before(jiffies, vring->tx_timeout)) { in mlxbf_tmfifo_check_tx_timeout()
855 vring->rem_padding = round_up(vring->rem_len, sizeof(u64)); in mlxbf_tmfifo_check_tx_timeout()
857 vring->cur_len = 0; in mlxbf_tmfifo_check_tx_timeout()
858 vring->rem_len = 0; in mlxbf_tmfifo_check_tx_timeout()
859 vring->fifo->vring[0] = NULL; in mlxbf_tmfifo_check_tx_timeout()
868 spin_lock_irqsave(&vring->fifo->spin_lock[0], flags); in mlxbf_tmfifo_check_tx_timeout()
869 vring_interrupt(0, vring->vq); in mlxbf_tmfifo_check_tx_timeout()
870 spin_unlock_irqrestore(&vring->fifo->spin_lock[0], flags); in mlxbf_tmfifo_check_tx_timeout()
876 int avail = 0, devid = vring->vdev_id; in mlxbf_tmfifo_rxtx()
877 struct mlxbf_tmfifo *fifo; in mlxbf_tmfifo_rxtx() local
880 fifo = vring->fifo; in mlxbf_tmfifo_rxtx()
883 if (!fifo || !fifo->vdev[devid]) in mlxbf_tmfifo_rxtx()
887 if (fifo->vring[is_rx] && fifo->vring[is_rx] != vring) in mlxbf_tmfifo_rxtx()
896 /* Get available FIFO space. */ in mlxbf_tmfifo_rxtx()
899 avail = mlxbf_tmfifo_get_rx_avail(fifo); in mlxbf_tmfifo_rxtx()
901 avail = mlxbf_tmfifo_get_tx_avail(fifo, devid); in mlxbf_tmfifo_rxtx()
908 vring->tx_timeout = 0; in mlxbf_tmfifo_rxtx()
909 while (vring->rem_padding >= sizeof(u64)) { in mlxbf_tmfifo_rxtx()
910 writeq(0, vring->fifo->tx.data); in mlxbf_tmfifo_rxtx()
911 vring->rem_padding -= sizeof(u64); in mlxbf_tmfifo_rxtx()
912 if (--avail == 0) in mlxbf_tmfifo_rxtx()
919 mlxbf_tmfifo_console_tx(fifo, avail); in mlxbf_tmfifo_rxtx()
933 static void mlxbf_tmfifo_work_rxtx(struct mlxbf_tmfifo *fifo, int queue_id, in mlxbf_tmfifo_work_rxtx() argument
940 if (!test_and_clear_bit(irq_id, &fifo->pend_events) || in mlxbf_tmfifo_work_rxtx()
941 !fifo->irq_info[irq_id].irq) in mlxbf_tmfifo_work_rxtx()
945 tm_vdev = fifo->vdev[i]; in mlxbf_tmfifo_work_rxtx()
947 vring = &tm_vdev->vrings[queue_id]; in mlxbf_tmfifo_work_rxtx()
948 if (vring->vq) in mlxbf_tmfifo_work_rxtx()
957 struct mlxbf_tmfifo *fifo; in mlxbf_tmfifo_work_handler() local
959 fifo = container_of(work, struct mlxbf_tmfifo, work); in mlxbf_tmfifo_work_handler()
960 if (!fifo->is_ready) in mlxbf_tmfifo_work_handler()
963 mutex_lock(&fifo->lock); in mlxbf_tmfifo_work_handler()
966 mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_TX, in mlxbf_tmfifo_work_handler()
970 mlxbf_tmfifo_work_rxtx(fifo, MLXBF_TMFIFO_VRING_RX, in mlxbf_tmfifo_work_handler()
973 mutex_unlock(&fifo->lock); in mlxbf_tmfifo_work_handler()
979 struct mlxbf_tmfifo_vring *vring = vq->priv; in mlxbf_tmfifo_virtio_notify()
981 struct mlxbf_tmfifo *fifo; in mlxbf_tmfifo_virtio_notify() local
984 fifo = vring->fifo; in mlxbf_tmfifo_virtio_notify()
990 if (vring->index & BIT(0)) { in mlxbf_tmfifo_virtio_notify()
997 if (vring->vdev_id == VIRTIO_ID_CONSOLE) { in mlxbf_tmfifo_virtio_notify()
998 spin_lock_irqsave(&fifo->spin_lock[0], flags); in mlxbf_tmfifo_virtio_notify()
999 tm_vdev = fifo->vdev[VIRTIO_ID_CONSOLE]; in mlxbf_tmfifo_virtio_notify()
1001 spin_unlock_irqrestore(&fifo->spin_lock[0], flags); in mlxbf_tmfifo_virtio_notify()
1002 set_bit(MLXBF_TM_TX_LWM_IRQ, &fifo->pend_events); in mlxbf_tmfifo_virtio_notify()
1004 &fifo->pend_events)) { in mlxbf_tmfifo_virtio_notify()
1008 if (test_and_set_bit(MLXBF_TM_RX_HWM_IRQ, &fifo->pend_events)) in mlxbf_tmfifo_virtio_notify()
1012 schedule_work(&fifo->work); in mlxbf_tmfifo_virtio_notify()
1022 return tm_vdev->features; in mlxbf_tmfifo_virtio_get_features()
1030 tm_vdev->features = vdev->features; in mlxbf_tmfifo_virtio_finalize_features()
1043 for (i = 0; i < ARRAY_SIZE(tm_vdev->vrings); i++) { in mlxbf_tmfifo_virtio_del_vqs()
1044 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_virtio_del_vqs()
1047 if (vring->desc) in mlxbf_tmfifo_virtio_del_vqs()
1049 vq = vring->vq; in mlxbf_tmfifo_virtio_del_vqs()
1051 vring->vq = NULL; in mlxbf_tmfifo_virtio_del_vqs()
1069 int i, ret, size; in mlxbf_tmfifo_virtio_find_vqs() local
1071 if (nvqs > ARRAY_SIZE(tm_vdev->vrings)) in mlxbf_tmfifo_virtio_find_vqs()
1072 return -EINVAL; in mlxbf_tmfifo_virtio_find_vqs()
1076 ret = -EINVAL; in mlxbf_tmfifo_virtio_find_vqs()
1079 vring = &tm_vdev->vrings[i]; in mlxbf_tmfifo_virtio_find_vqs()
1082 size = vring_size(vring->num, vring->align); in mlxbf_tmfifo_virtio_find_vqs()
1083 memset(vring->va, 0, size); in mlxbf_tmfifo_virtio_find_vqs()
1084 vq = vring_new_virtqueue(i, vring->num, vring->align, vdev, in mlxbf_tmfifo_virtio_find_vqs()
1085 false, false, vring->va, in mlxbf_tmfifo_virtio_find_vqs()
1089 dev_err(&vdev->dev, "vring_new_virtqueue failed\n"); in mlxbf_tmfifo_virtio_find_vqs()
1090 ret = -ENOMEM; in mlxbf_tmfifo_virtio_find_vqs()
1094 vq->num_max = vring->num; in mlxbf_tmfifo_virtio_find_vqs()
1096 vq->priv = vring; in mlxbf_tmfifo_virtio_find_vqs()
1102 vring->vq = vq; in mlxbf_tmfifo_virtio_find_vqs()
1117 return tm_vdev->status; in mlxbf_tmfifo_virtio_get_status()
1126 tm_vdev->status = status; in mlxbf_tmfifo_virtio_set_status()
1134 tm_vdev->status = 0; in mlxbf_tmfifo_virtio_reset()
1145 if ((u64)offset + len > sizeof(tm_vdev->config)) in mlxbf_tmfifo_virtio_get()
1148 memcpy(buf, (u8 *)&tm_vdev->config + offset, len); in mlxbf_tmfifo_virtio_get()
1159 if ((u64)offset + len > sizeof(tm_vdev->config)) in mlxbf_tmfifo_virtio_set()
1162 memcpy((u8 *)&tm_vdev->config + offset, buf, len); in mlxbf_tmfifo_virtio_set()
1187 /* Create vdev for the FIFO. */
1189 struct mlxbf_tmfifo *fifo, in mlxbf_tmfifo_create_vdev() argument
1191 void *config, u32 size) in mlxbf_tmfifo_create_vdev() argument
1196 mutex_lock(&fifo->lock); in mlxbf_tmfifo_create_vdev()
1198 tm_vdev = fifo->vdev[vdev_id]; in mlxbf_tmfifo_create_vdev()
1201 ret = -EEXIST; in mlxbf_tmfifo_create_vdev()
1207 ret = -ENOMEM; in mlxbf_tmfifo_create_vdev()
1211 tm_vdev->vdev.id.device = vdev_id; in mlxbf_tmfifo_create_vdev()
1212 tm_vdev->vdev.config = &mlxbf_tmfifo_virtio_config_ops; in mlxbf_tmfifo_create_vdev()
1213 tm_vdev->vdev.dev.parent = dev; in mlxbf_tmfifo_create_vdev()
1214 tm_vdev->vdev.dev.release = tmfifo_virtio_dev_release; in mlxbf_tmfifo_create_vdev()
1215 tm_vdev->features = features; in mlxbf_tmfifo_create_vdev()
1217 memcpy(&tm_vdev->config, config, size); in mlxbf_tmfifo_create_vdev()
1219 if (mlxbf_tmfifo_alloc_vrings(fifo, tm_vdev)) { in mlxbf_tmfifo_create_vdev()
1221 ret = -ENOMEM; in mlxbf_tmfifo_create_vdev()
1227 tm_vdev->tx_buf.buf = devm_kmalloc(dev, in mlxbf_tmfifo_create_vdev()
1230 fifo->vdev[vdev_id] = tm_vdev; in mlxbf_tmfifo_create_vdev()
1233 ret = register_virtio_device(&tm_vdev->vdev); in mlxbf_tmfifo_create_vdev()
1240 mutex_unlock(&fifo->lock); in mlxbf_tmfifo_create_vdev()
1244 mlxbf_tmfifo_free_vrings(fifo, tm_vdev); in mlxbf_tmfifo_create_vdev()
1245 fifo->vdev[vdev_id] = NULL; in mlxbf_tmfifo_create_vdev()
1247 put_device(&tm_vdev->vdev.dev); in mlxbf_tmfifo_create_vdev()
1251 mutex_unlock(&fifo->lock); in mlxbf_tmfifo_create_vdev()
1255 /* Delete vdev for the FIFO. */
1256 static int mlxbf_tmfifo_delete_vdev(struct mlxbf_tmfifo *fifo, int vdev_id) in mlxbf_tmfifo_delete_vdev() argument
1260 mutex_lock(&fifo->lock); in mlxbf_tmfifo_delete_vdev()
1263 tm_vdev = fifo->vdev[vdev_id]; in mlxbf_tmfifo_delete_vdev()
1265 unregister_virtio_device(&tm_vdev->vdev); in mlxbf_tmfifo_delete_vdev()
1266 mlxbf_tmfifo_free_vrings(fifo, tm_vdev); in mlxbf_tmfifo_delete_vdev()
1267 fifo->vdev[vdev_id] = NULL; in mlxbf_tmfifo_delete_vdev()
1270 mutex_unlock(&fifo->lock); in mlxbf_tmfifo_delete_vdev()
1279 unsigned long size = ETH_ALEN; in mlxbf_tmfifo_get_cfg_mac() local
1283 rc = efi.get_variable(mlxbf_tmfifo_efi_name, &guid, NULL, &size, buf); in mlxbf_tmfifo_get_cfg_mac()
1284 if (rc == EFI_SUCCESS && size == ETH_ALEN) in mlxbf_tmfifo_get_cfg_mac()
1291 static void mlxbf_tmfifo_set_threshold(struct mlxbf_tmfifo *fifo) in mlxbf_tmfifo_set_threshold() argument
1295 /* Get Tx FIFO size and set the low/high watermark. */ in mlxbf_tmfifo_set_threshold()
1296 ctl = readq(fifo->tx.ctl); in mlxbf_tmfifo_set_threshold()
1297 fifo->tx_fifo_size = in mlxbf_tmfifo_set_threshold()
1301 fifo->tx_fifo_size / 2); in mlxbf_tmfifo_set_threshold()
1304 fifo->tx_fifo_size - 1); in mlxbf_tmfifo_set_threshold()
1305 writeq(ctl, fifo->tx.ctl); in mlxbf_tmfifo_set_threshold()
1307 /* Get Rx FIFO size and set the low/high watermark. */ in mlxbf_tmfifo_set_threshold()
1308 ctl = readq(fifo->rx.ctl); in mlxbf_tmfifo_set_threshold()
1309 fifo->rx_fifo_size = in mlxbf_tmfifo_set_threshold()
1315 writeq(ctl, fifo->rx.ctl); in mlxbf_tmfifo_set_threshold()
1318 static void mlxbf_tmfifo_cleanup(struct mlxbf_tmfifo *fifo) in mlxbf_tmfifo_cleanup() argument
1322 fifo->is_ready = false; in mlxbf_tmfifo_cleanup()
1323 del_timer_sync(&fifo->timer); in mlxbf_tmfifo_cleanup()
1324 mlxbf_tmfifo_disable_irqs(fifo); in mlxbf_tmfifo_cleanup()
1325 cancel_work_sync(&fifo->work); in mlxbf_tmfifo_cleanup()
1327 mlxbf_tmfifo_delete_vdev(fifo, i); in mlxbf_tmfifo_cleanup()
1334 struct device *dev = &pdev->dev; in mlxbf_tmfifo_probe()
1335 struct mlxbf_tmfifo *fifo; in mlxbf_tmfifo_probe() local
1345 fifo = devm_kzalloc(dev, sizeof(*fifo), GFP_KERNEL); in mlxbf_tmfifo_probe()
1346 if (!fifo) in mlxbf_tmfifo_probe()
1347 return -ENOMEM; in mlxbf_tmfifo_probe()
1349 spin_lock_init(&fifo->spin_lock[0]); in mlxbf_tmfifo_probe()
1350 spin_lock_init(&fifo->spin_lock[1]); in mlxbf_tmfifo_probe()
1351 INIT_WORK(&fifo->work, mlxbf_tmfifo_work_handler); in mlxbf_tmfifo_probe()
1352 mutex_init(&fifo->lock); in mlxbf_tmfifo_probe()
1354 /* Get the resource of the Rx FIFO. */ in mlxbf_tmfifo_probe()
1355 fifo->res0 = devm_platform_ioremap_resource(pdev, 0); in mlxbf_tmfifo_probe()
1356 if (IS_ERR(fifo->res0)) in mlxbf_tmfifo_probe()
1357 return PTR_ERR(fifo->res0); in mlxbf_tmfifo_probe()
1359 /* Get the resource of the Tx FIFO. */ in mlxbf_tmfifo_probe()
1360 fifo->res1 = devm_platform_ioremap_resource(pdev, 1); in mlxbf_tmfifo_probe()
1361 if (IS_ERR(fifo->res1)) in mlxbf_tmfifo_probe()
1362 return PTR_ERR(fifo->res1); in mlxbf_tmfifo_probe()
1365 fifo->rx.ctl = fifo->res1 + MLXBF_TMFIFO_RX_CTL_BF3; in mlxbf_tmfifo_probe()
1366 fifo->rx.sts = fifo->res1 + MLXBF_TMFIFO_RX_STS_BF3; in mlxbf_tmfifo_probe()
1367 fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA_BF3; in mlxbf_tmfifo_probe()
1368 fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL_BF3; in mlxbf_tmfifo_probe()
1369 fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS_BF3; in mlxbf_tmfifo_probe()
1370 fifo->tx.data = fifo->res0 + MLXBF_TMFIFO_TX_DATA_BF3; in mlxbf_tmfifo_probe()
1372 fifo->rx.ctl = fifo->res0 + MLXBF_TMFIFO_RX_CTL; in mlxbf_tmfifo_probe()
1373 fifo->rx.sts = fifo->res0 + MLXBF_TMFIFO_RX_STS; in mlxbf_tmfifo_probe()
1374 fifo->rx.data = fifo->res0 + MLXBF_TMFIFO_RX_DATA; in mlxbf_tmfifo_probe()
1375 fifo->tx.ctl = fifo->res1 + MLXBF_TMFIFO_TX_CTL; in mlxbf_tmfifo_probe()
1376 fifo->tx.sts = fifo->res1 + MLXBF_TMFIFO_TX_STS; in mlxbf_tmfifo_probe()
1377 fifo->tx.data = fifo->res1 + MLXBF_TMFIFO_TX_DATA; in mlxbf_tmfifo_probe()
1380 platform_set_drvdata(pdev, fifo); in mlxbf_tmfifo_probe()
1382 timer_setup(&fifo->timer, mlxbf_tmfifo_timer, 0); in mlxbf_tmfifo_probe()
1385 fifo->irq_info[i].index = i; in mlxbf_tmfifo_probe()
1386 fifo->irq_info[i].fifo = fifo; in mlxbf_tmfifo_probe()
1387 fifo->irq_info[i].irq = platform_get_irq(pdev, i); in mlxbf_tmfifo_probe()
1388 rc = devm_request_irq(dev, fifo->irq_info[i].irq, in mlxbf_tmfifo_probe()
1390 "tmfifo", &fifo->irq_info[i]); in mlxbf_tmfifo_probe()
1393 fifo->irq_info[i].irq = 0; in mlxbf_tmfifo_probe()
1398 mlxbf_tmfifo_set_threshold(fifo); in mlxbf_tmfifo_probe()
1401 rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_CONSOLE, 0, NULL, 0); in mlxbf_tmfifo_probe()
1408 /* A legacy-only interface for now. */ in mlxbf_tmfifo_probe()
1414 rc = mlxbf_tmfifo_create_vdev(dev, fifo, VIRTIO_ID_NET, in mlxbf_tmfifo_probe()
1420 mod_timer(&fifo->timer, jiffies + MLXBF_TMFIFO_TIMER_INTERVAL); in mlxbf_tmfifo_probe()
1425 fifo->is_ready = true; in mlxbf_tmfifo_probe()
1429 mlxbf_tmfifo_cleanup(fifo); in mlxbf_tmfifo_probe()
1436 struct mlxbf_tmfifo *fifo = platform_get_drvdata(pdev); in mlxbf_tmfifo_remove() local
1438 mlxbf_tmfifo_cleanup(fifo); in mlxbf_tmfifo_remove()
1453 .name = "bf-tmfifo",