Lines Matching +full:port +full:-
1 // SPDX-License-Identifier: GPL-2.0
61 static void mvpp2_acpi_start(struct mvpp2_port *port);
76 writel(data, priv->swth_base[0] + offset); in mvpp2_write()
81 return readl(priv->swth_base[0] + offset); in mvpp2_read()
86 return readl_relaxed(priv->swth_base[0] + offset); in mvpp2_read_relaxed()
91 return cpu % priv->nthreads; in mvpp2_cpu_to_thread()
96 writel(data, priv->cm3_base + offset); in mvpp2_cm3_write()
101 return readl(priv->cm3_base + offset); in mvpp2_cm3_read()
124 * - per-thread registers, where each thread has its own copy of the
140 * - global registers that must be accessed through a specific thread
141 * window, because they are related to an access to a per-thread
161 writel(data, priv->swth_base[thread] + offset); in mvpp2_thread_write()
167 return readl(priv->swth_base[thread] + offset); in mvpp2_thread_read()
173 writel_relaxed(data, priv->swth_base[thread] + offset); in mvpp2_thread_write_relaxed()
179 return readl_relaxed(priv->swth_base[thread] + offset); in mvpp2_thread_read_relaxed()
182 static dma_addr_t mvpp2_txdesc_dma_addr_get(struct mvpp2_port *port, in mvpp2_txdesc_dma_addr_get() argument
185 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_dma_addr_get()
186 return le32_to_cpu(tx_desc->pp21.buf_dma_addr); in mvpp2_txdesc_dma_addr_get()
188 return le64_to_cpu(tx_desc->pp22.buf_dma_addr_ptp) & in mvpp2_txdesc_dma_addr_get()
192 static void mvpp2_txdesc_dma_addr_set(struct mvpp2_port *port, in mvpp2_txdesc_dma_addr_set() argument
201 if (port->priv->hw_version == MVPP21) { in mvpp2_txdesc_dma_addr_set()
202 tx_desc->pp21.buf_dma_addr = cpu_to_le32(addr); in mvpp2_txdesc_dma_addr_set()
203 tx_desc->pp21.packet_offset = offset; in mvpp2_txdesc_dma_addr_set()
207 tx_desc->pp22.buf_dma_addr_ptp &= ~cpu_to_le64(MVPP2_DESC_DMA_MASK); in mvpp2_txdesc_dma_addr_set()
208 tx_desc->pp22.buf_dma_addr_ptp |= val; in mvpp2_txdesc_dma_addr_set()
209 tx_desc->pp22.packet_offset = offset; in mvpp2_txdesc_dma_addr_set()
213 static size_t mvpp2_txdesc_size_get(struct mvpp2_port *port, in mvpp2_txdesc_size_get() argument
216 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_size_get()
217 return le16_to_cpu(tx_desc->pp21.data_size); in mvpp2_txdesc_size_get()
219 return le16_to_cpu(tx_desc->pp22.data_size); in mvpp2_txdesc_size_get()
222 static void mvpp2_txdesc_size_set(struct mvpp2_port *port, in mvpp2_txdesc_size_set() argument
226 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_size_set()
227 tx_desc->pp21.data_size = cpu_to_le16(size); in mvpp2_txdesc_size_set()
229 tx_desc->pp22.data_size = cpu_to_le16(size); in mvpp2_txdesc_size_set()
232 static void mvpp2_txdesc_txq_set(struct mvpp2_port *port, in mvpp2_txdesc_txq_set() argument
236 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_txq_set()
237 tx_desc->pp21.phys_txq = txq; in mvpp2_txdesc_txq_set()
239 tx_desc->pp22.phys_txq = txq; in mvpp2_txdesc_txq_set()
242 static void mvpp2_txdesc_cmd_set(struct mvpp2_port *port, in mvpp2_txdesc_cmd_set() argument
246 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_cmd_set()
247 tx_desc->pp21.command = cpu_to_le32(command); in mvpp2_txdesc_cmd_set()
249 tx_desc->pp22.command = cpu_to_le32(command); in mvpp2_txdesc_cmd_set()
252 static unsigned int mvpp2_txdesc_offset_get(struct mvpp2_port *port, in mvpp2_txdesc_offset_get() argument
255 if (port->priv->hw_version == MVPP21) in mvpp2_txdesc_offset_get()
256 return tx_desc->pp21.packet_offset; in mvpp2_txdesc_offset_get()
258 return tx_desc->pp22.packet_offset; in mvpp2_txdesc_offset_get()
261 static dma_addr_t mvpp2_rxdesc_dma_addr_get(struct mvpp2_port *port, in mvpp2_rxdesc_dma_addr_get() argument
264 if (port->priv->hw_version == MVPP21) in mvpp2_rxdesc_dma_addr_get()
265 return le32_to_cpu(rx_desc->pp21.buf_dma_addr); in mvpp2_rxdesc_dma_addr_get()
267 return le64_to_cpu(rx_desc->pp22.buf_dma_addr_key_hash) & in mvpp2_rxdesc_dma_addr_get()
271 static unsigned long mvpp2_rxdesc_cookie_get(struct mvpp2_port *port, in mvpp2_rxdesc_cookie_get() argument
274 if (port->priv->hw_version == MVPP21) in mvpp2_rxdesc_cookie_get()
275 return le32_to_cpu(rx_desc->pp21.buf_cookie); in mvpp2_rxdesc_cookie_get()
277 return le64_to_cpu(rx_desc->pp22.buf_cookie_misc) & in mvpp2_rxdesc_cookie_get()
281 static size_t mvpp2_rxdesc_size_get(struct mvpp2_port *port, in mvpp2_rxdesc_size_get() argument
284 if (port->priv->hw_version == MVPP21) in mvpp2_rxdesc_size_get()
285 return le16_to_cpu(rx_desc->pp21.data_size); in mvpp2_rxdesc_size_get()
287 return le16_to_cpu(rx_desc->pp22.data_size); in mvpp2_rxdesc_size_get()
290 static u32 mvpp2_rxdesc_status_get(struct mvpp2_port *port, in mvpp2_rxdesc_status_get() argument
293 if (port->priv->hw_version == MVPP21) in mvpp2_rxdesc_status_get()
294 return le32_to_cpu(rx_desc->pp21.status); in mvpp2_rxdesc_status_get()
296 return le32_to_cpu(rx_desc->pp22.status); in mvpp2_rxdesc_status_get()
301 txq_pcpu->txq_get_index++; in mvpp2_txq_inc_get()
302 if (txq_pcpu->txq_get_index == txq_pcpu->size) in mvpp2_txq_inc_get()
303 txq_pcpu->txq_get_index = 0; in mvpp2_txq_inc_get()
306 static void mvpp2_txq_inc_put(struct mvpp2_port *port, in mvpp2_txq_inc_put() argument
313 txq_pcpu->buffs + txq_pcpu->txq_put_index; in mvpp2_txq_inc_put()
314 tx_buf->type = buf_type; in mvpp2_txq_inc_put()
316 tx_buf->skb = data; in mvpp2_txq_inc_put()
318 tx_buf->xdpf = data; in mvpp2_txq_inc_put()
319 tx_buf->size = mvpp2_txdesc_size_get(port, tx_desc); in mvpp2_txq_inc_put()
320 tx_buf->dma = mvpp2_txdesc_dma_addr_get(port, tx_desc) + in mvpp2_txq_inc_put()
321 mvpp2_txdesc_offset_get(port, tx_desc); in mvpp2_txq_inc_put()
322 txq_pcpu->txq_put_index++; in mvpp2_txq_inc_put()
323 if (txq_pcpu->txq_put_index == txq_pcpu->size) in mvpp2_txq_inc_put()
324 txq_pcpu->txq_put_index = 0; in mvpp2_txq_inc_put()
332 if (priv->hw_version >= MVPP22 && queue_mode == MVPP2_QDIST_SINGLE_MODE) in mvpp2_get_nrxqs()
347 /* Get number of physical egress port */
348 static inline int mvpp2_egress_port(struct mvpp2_port *port) in mvpp2_egress_port() argument
350 return MVPP2_MAX_TCONT + port->id; in mvpp2_egress_port()
354 static inline int mvpp2_txq_phys(int port, int txq) in mvpp2_txq_phys() argument
356 return (MVPP2_MAX_TCONT + port) * MVPP2_MAX_TXQ + txq; in mvpp2_txq_phys()
366 if (likely(pool->frag_size <= PAGE_SIZE)) in mvpp2_frag_alloc()
367 return netdev_alloc_frag(pool->frag_size); in mvpp2_frag_alloc()
369 return kmalloc(pool->frag_size, GFP_ATOMIC); in mvpp2_frag_alloc()
377 else if (likely(pool->frag_size <= PAGE_SIZE)) in mvpp2_frag_free()
395 return -EINVAL; in mvpp2_bm_pool_create()
400 if (priv->hw_version == MVPP21) in mvpp2_bm_pool_create()
401 bm_pool->size_bytes = 2 * sizeof(u32) * size; in mvpp2_bm_pool_create()
403 bm_pool->size_bytes = 2 * sizeof(u64) * size; in mvpp2_bm_pool_create()
405 bm_pool->virt_addr = dma_alloc_coherent(dev, bm_pool->size_bytes, in mvpp2_bm_pool_create()
406 &bm_pool->dma_addr, in mvpp2_bm_pool_create()
408 if (!bm_pool->virt_addr) in mvpp2_bm_pool_create()
409 return -ENOMEM; in mvpp2_bm_pool_create()
411 if (!IS_ALIGNED((unsigned long)bm_pool->virt_addr, in mvpp2_bm_pool_create()
413 dma_free_coherent(dev, bm_pool->size_bytes, in mvpp2_bm_pool_create()
414 bm_pool->virt_addr, bm_pool->dma_addr); in mvpp2_bm_pool_create()
416 bm_pool->id, MVPP2_BM_POOL_PTR_ALIGN); in mvpp2_bm_pool_create()
417 return -ENOMEM; in mvpp2_bm_pool_create()
420 mvpp2_write(priv, MVPP2_BM_POOL_BASE_REG(bm_pool->id), in mvpp2_bm_pool_create()
421 lower_32_bits(bm_pool->dma_addr)); in mvpp2_bm_pool_create()
422 mvpp2_write(priv, MVPP2_BM_POOL_SIZE_REG(bm_pool->id), size); in mvpp2_bm_pool_create()
424 val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id)); in mvpp2_bm_pool_create()
431 if (priv->hw_version == MVPP23) { in mvpp2_bm_pool_create()
439 mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val); in mvpp2_bm_pool_create()
441 bm_pool->size = size; in mvpp2_bm_pool_create()
442 bm_pool->pkt_size = 0; in mvpp2_bm_pool_create()
443 bm_pool->buf_num = 0; in mvpp2_bm_pool_create()
455 bm_pool->buf_size = buf_size; in mvpp2_bm_pool_bufsize_set()
458 mvpp2_write(priv, MVPP2_POOL_BUF_SIZE_REG(bm_pool->id), val); in mvpp2_bm_pool_bufsize_set()
469 MVPP2_BM_PHY_ALLOC_REG(bm_pool->id)); in mvpp2_bm_bufs_get_addrs()
472 if (priv->hw_version >= MVPP22) { in mvpp2_bm_bufs_get_addrs()
498 if (buf_num > bm_pool->buf_num) { in mvpp2_bm_bufs_free()
500 bm_pool->id, buf_num); in mvpp2_bm_bufs_free()
501 buf_num = bm_pool->buf_num; in mvpp2_bm_bufs_free()
504 if (priv->percpu_pools) in mvpp2_bm_bufs_free()
505 pp = priv->page_pool[bm_pool->id]; in mvpp2_bm_bufs_free()
517 bm_pool->buf_size, DMA_FROM_DEVICE); in mvpp2_bm_bufs_free()
527 bm_pool->buf_num -= i; in mvpp2_bm_bufs_free()
535 buf_num += mvpp2_read(priv, MVPP2_BM_POOL_PTRS_NUM_REG(bm_pool->id)) & in mvpp2_check_hw_buf_num()
537 buf_num += mvpp2_read(priv, MVPP2_BM_BPPI_PTRS_NUM_REG(bm_pool->id)) & in mvpp2_check_hw_buf_num()
561 bm_pool->id, bm_pool->buf_num); in mvpp2_bm_pool_destroy()
565 val = mvpp2_read(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id)); in mvpp2_bm_pool_destroy()
567 mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val); in mvpp2_bm_pool_destroy()
569 if (priv->percpu_pools) { in mvpp2_bm_pool_destroy()
570 page_pool_destroy(priv->page_pool[bm_pool->id]); in mvpp2_bm_pool_destroy()
571 priv->page_pool[bm_pool->id] = NULL; in mvpp2_bm_pool_destroy()
574 dma_free_coherent(dev, bm_pool->size_bytes, in mvpp2_bm_pool_destroy()
575 bm_pool->virt_addr, in mvpp2_bm_pool_destroy()
576 bm_pool->dma_addr); in mvpp2_bm_pool_destroy()
585 if (priv->percpu_pools) in mvpp2_bm_pools_init()
591 bm_pool = &priv->bm_pools[i]; in mvpp2_bm_pools_init()
592 bm_pool->id = i; in mvpp2_bm_pools_init()
602 for (i = i - 1; i >= 0; i--) in mvpp2_bm_pools_init()
603 mvpp2_bm_pool_destroy(dev, priv, &priv->bm_pools[i]); in mvpp2_bm_pools_init()
640 struct mvpp2_port *port; in mvpp2_bm_init() local
642 if (priv->percpu_pools) in mvpp2_bm_init()
649 if (priv->percpu_pools) { in mvpp2_bm_init()
650 for (i = 0; i < priv->port_count; i++) { in mvpp2_bm_init()
651 port = priv->port_list[i]; in mvpp2_bm_init()
652 if (port->xdp_prog) { in mvpp2_bm_init()
662 priv->page_pool[i] = in mvpp2_bm_init()
667 if (IS_ERR(priv->page_pool[i])) { in mvpp2_bm_init()
671 page_pool_destroy(priv->page_pool[j]); in mvpp2_bm_init()
672 priv->page_pool[j] = NULL; in mvpp2_bm_init()
674 return PTR_ERR(priv->page_pool[i]); in mvpp2_bm_init()
680 priv->percpu_pools ? "per-cpu" : "shared"); in mvpp2_bm_init()
690 priv->bm_pools = devm_kcalloc(dev, poolnum, in mvpp2_bm_init()
691 sizeof(*priv->bm_pools), GFP_KERNEL); in mvpp2_bm_init()
692 if (!priv->bm_pools) in mvpp2_bm_init()
693 return -ENOMEM; in mvpp2_bm_init()
695 if (priv->hw_version == MVPP23) in mvpp2_bm_init()
720 static void mvpp2_rxq_long_pool_set(struct mvpp2_port *port, in mvpp2_rxq_long_pool_set() argument
727 prxq = port->rxqs[lrxq]->id; in mvpp2_rxq_long_pool_set()
729 if (port->priv->hw_version == MVPP21) in mvpp2_rxq_long_pool_set()
734 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq)); in mvpp2_rxq_long_pool_set()
737 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val); in mvpp2_rxq_long_pool_set()
741 static void mvpp2_rxq_short_pool_set(struct mvpp2_port *port, in mvpp2_rxq_short_pool_set() argument
748 prxq = port->rxqs[lrxq]->id; in mvpp2_rxq_short_pool_set()
750 if (port->priv->hw_version == MVPP21) in mvpp2_rxq_short_pool_set()
755 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq)); in mvpp2_rxq_short_pool_set()
758 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val); in mvpp2_rxq_short_pool_set()
761 static void *mvpp2_buf_alloc(struct mvpp2_port *port, in mvpp2_buf_alloc() argument
781 dma_addr = dma_map_single(port->dev->dev.parent, data, in mvpp2_buf_alloc()
782 MVPP2_RX_BUF_SIZE(bm_pool->pkt_size), in mvpp2_buf_alloc()
784 if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) { in mvpp2_buf_alloc()
796 static void mvpp2_rxq_enable_fc(struct mvpp2_port *port) in mvpp2_rxq_enable_fc() argument
799 int fq = port->first_rxq; in mvpp2_rxq_enable_fc()
802 spin_lock_irqsave(&port->priv->mss_spinlock, flags); in mvpp2_rxq_enable_fc()
805 * If Flow control was enabled, it would be re-enabled. in mvpp2_rxq_enable_fc()
807 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_rxq_enable_fc()
810 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_rxq_enable_fc()
813 for (q = 0; q < port->nrxqs; q++) { in mvpp2_rxq_enable_fc()
817 mvpp2_cm3_write(port->priv, MSS_RXQ_TRESH_REG(q, fq), val); in mvpp2_rxq_enable_fc()
819 val = mvpp2_cm3_read(port->priv, MSS_RXQ_ASS_REG(q, fq)); in mvpp2_rxq_enable_fc()
820 /* Set RXQ port ID */ in mvpp2_rxq_enable_fc()
822 val |= (port->id << MSS_RXQ_ASS_Q_BASE(q, fq)); in mvpp2_rxq_enable_fc()
834 host_id = port->nqvecs; in mvpp2_rxq_enable_fc()
844 mvpp2_cm3_write(port->priv, MSS_RXQ_ASS_REG(q, fq), val); in mvpp2_rxq_enable_fc()
848 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_rxq_enable_fc()
851 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_rxq_enable_fc()
853 spin_unlock_irqrestore(&port->priv->mss_spinlock, flags); in mvpp2_rxq_enable_fc()
857 static void mvpp2_rxq_disable_fc(struct mvpp2_port *port) in mvpp2_rxq_disable_fc() argument
861 int fq = port->first_rxq; in mvpp2_rxq_disable_fc()
863 spin_lock_irqsave(&port->priv->mss_spinlock, flags); in mvpp2_rxq_disable_fc()
866 * If Flow control was enabled, it would be re-enabled. in mvpp2_rxq_disable_fc()
868 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_rxq_disable_fc()
871 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_rxq_disable_fc()
874 for (q = 0; q < port->nrxqs; q++) { in mvpp2_rxq_disable_fc()
878 mvpp2_cm3_write(port->priv, MSS_RXQ_TRESH_REG(q, fq), val); in mvpp2_rxq_disable_fc()
880 val = mvpp2_cm3_read(port->priv, MSS_RXQ_ASS_REG(q, fq)); in mvpp2_rxq_disable_fc()
887 mvpp2_cm3_write(port->priv, MSS_RXQ_ASS_REG(q, fq), val); in mvpp2_rxq_disable_fc()
891 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_rxq_disable_fc()
894 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_rxq_disable_fc()
896 spin_unlock_irqrestore(&port->priv->mss_spinlock, flags); in mvpp2_rxq_disable_fc()
900 static void mvpp2_bm_pool_update_fc(struct mvpp2_port *port, in mvpp2_bm_pool_update_fc() argument
907 spin_lock_irqsave(&port->priv->mss_spinlock, flags); in mvpp2_bm_pool_update_fc()
910 * If Flow control were enabled, it would be re-enabled. in mvpp2_bm_pool_update_fc()
912 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_bm_pool_update_fc()
915 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_bm_pool_update_fc()
919 /* Set BM pool start and stop thresholds per port */ in mvpp2_bm_pool_update_fc()
920 val = mvpp2_cm3_read(port->priv, MSS_BUF_POOL_REG(pool->id)); in mvpp2_bm_pool_update_fc()
921 val |= MSS_BUF_POOL_PORT_OFFS(port->id); in mvpp2_bm_pool_update_fc()
926 mvpp2_cm3_write(port->priv, MSS_BUF_POOL_REG(pool->id), val); in mvpp2_bm_pool_update_fc()
928 /* Remove BM pool from the port */ in mvpp2_bm_pool_update_fc()
929 val = mvpp2_cm3_read(port->priv, MSS_BUF_POOL_REG(pool->id)); in mvpp2_bm_pool_update_fc()
930 val &= ~MSS_BUF_POOL_PORT_OFFS(port->id); in mvpp2_bm_pool_update_fc()
933 * flow control if pool empty (not used by any port) in mvpp2_bm_pool_update_fc()
935 if (!pool->buf_num) { in mvpp2_bm_pool_update_fc()
940 mvpp2_cm3_write(port->priv, MSS_BUF_POOL_REG(pool->id), val); in mvpp2_bm_pool_update_fc()
944 val = mvpp2_cm3_read(port->priv, MSS_FC_COM_REG); in mvpp2_bm_pool_update_fc()
947 mvpp2_cm3_write(port->priv, MSS_FC_COM_REG, val); in mvpp2_bm_pool_update_fc()
949 spin_unlock_irqrestore(&port->priv->mss_spinlock, flags); in mvpp2_bm_pool_update_fc()
955 struct mvpp2_port *port; in mvpp2_bm_pool_update_priv_fc() local
958 for (i = 0; i < priv->port_count; i++) { in mvpp2_bm_pool_update_priv_fc()
959 port = priv->port_list[i]; in mvpp2_bm_pool_update_priv_fc()
960 if (port->priv->percpu_pools) { in mvpp2_bm_pool_update_priv_fc()
961 for (j = 0; j < port->nrxqs; j++) in mvpp2_bm_pool_update_priv_fc()
962 mvpp2_bm_pool_update_fc(port, &port->priv->bm_pools[j], in mvpp2_bm_pool_update_priv_fc()
963 port->tx_fc & en); in mvpp2_bm_pool_update_priv_fc()
965 mvpp2_bm_pool_update_fc(port, port->pool_long, port->tx_fc & en); in mvpp2_bm_pool_update_priv_fc()
966 mvpp2_bm_pool_update_fc(port, port->pool_short, port->tx_fc & en); in mvpp2_bm_pool_update_priv_fc()
976 * flow control enabled, but still disabled per port. in mvpp2_enable_global_fc()
995 priv->global_tx_fc = false; in mvpp2_enable_global_fc()
996 return -EOPNOTSUPP; in mvpp2_enable_global_fc()
1000 static inline void mvpp2_bm_pool_put(struct mvpp2_port *port, int pool, in mvpp2_bm_pool_put() argument
1004 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_bm_pool_put()
1007 if (test_bit(thread, &port->priv->lock_map)) in mvpp2_bm_pool_put()
1008 spin_lock_irqsave(&port->bm_lock[thread], flags); in mvpp2_bm_pool_put()
1010 if (port->priv->hw_version >= MVPP22) { in mvpp2_bm_pool_put()
1022 mvpp2_thread_write_relaxed(port->priv, thread, in mvpp2_bm_pool_put()
1031 mvpp2_thread_write_relaxed(port->priv, thread, in mvpp2_bm_pool_put()
1033 mvpp2_thread_write_relaxed(port->priv, thread, in mvpp2_bm_pool_put()
1036 if (test_bit(thread, &port->priv->lock_map)) in mvpp2_bm_pool_put()
1037 spin_unlock_irqrestore(&port->bm_lock[thread], flags); in mvpp2_bm_pool_put()
1043 static int mvpp2_bm_bufs_add(struct mvpp2_port *port, in mvpp2_bm_bufs_add() argument
1052 if (port->priv->percpu_pools && in mvpp2_bm_bufs_add()
1053 bm_pool->pkt_size > MVPP2_BM_LONG_PKT_SIZE) { in mvpp2_bm_bufs_add()
1054 netdev_err(port->dev, in mvpp2_bm_bufs_add()
1055 "attempted to use jumbo frames with per-cpu pools"); in mvpp2_bm_bufs_add()
1059 buf_size = MVPP2_RX_BUF_SIZE(bm_pool->pkt_size); in mvpp2_bm_bufs_add()
1063 (buf_num + bm_pool->buf_num > bm_pool->size)) { in mvpp2_bm_bufs_add()
1064 netdev_err(port->dev, in mvpp2_bm_bufs_add()
1066 buf_num, bm_pool->id); in mvpp2_bm_bufs_add()
1070 if (port->priv->percpu_pools) in mvpp2_bm_bufs_add()
1071 pp = port->priv->page_pool[bm_pool->id]; in mvpp2_bm_bufs_add()
1073 buf = mvpp2_buf_alloc(port, bm_pool, pp, &dma_addr, in mvpp2_bm_bufs_add()
1078 mvpp2_bm_pool_put(port, bm_pool->id, dma_addr, in mvpp2_bm_bufs_add()
1083 bm_pool->buf_num += i; in mvpp2_bm_bufs_add()
1085 netdev_dbg(port->dev, in mvpp2_bm_bufs_add()
1087 bm_pool->id, bm_pool->pkt_size, buf_size, total_size); in mvpp2_bm_bufs_add()
1089 netdev_dbg(port->dev, in mvpp2_bm_bufs_add()
1091 bm_pool->id, i, buf_num); in mvpp2_bm_bufs_add()
1099 mvpp2_bm_pool_use(struct mvpp2_port *port, unsigned pool, int pkt_size) in mvpp2_bm_pool_use() argument
1101 struct mvpp2_bm_pool *new_pool = &port->priv->bm_pools[pool]; in mvpp2_bm_pool_use()
1104 if ((port->priv->percpu_pools && pool > mvpp2_get_nrxqs(port->priv) * 2) || in mvpp2_bm_pool_use()
1105 (!port->priv->percpu_pools && pool >= MVPP2_BM_POOLS_NUM)) { in mvpp2_bm_pool_use()
1106 netdev_err(port->dev, "Invalid pool %d\n", pool); in mvpp2_bm_pool_use()
1113 if (new_pool->pkt_size == 0) { in mvpp2_bm_pool_use()
1119 pkts_num = new_pool->buf_num; in mvpp2_bm_pool_use()
1121 if (port->priv->percpu_pools) { in mvpp2_bm_pool_use()
1122 if (pool < port->nrxqs) in mvpp2_bm_pool_use()
1130 mvpp2_bm_bufs_free(port->dev->dev.parent, in mvpp2_bm_pool_use()
1131 port->priv, new_pool, pkts_num); in mvpp2_bm_pool_use()
1134 new_pool->pkt_size = pkt_size; in mvpp2_bm_pool_use()
1135 new_pool->frag_size = in mvpp2_bm_pool_use()
1140 num = mvpp2_bm_bufs_add(port, new_pool, pkts_num); in mvpp2_bm_pool_use()
1143 new_pool->id, num, pkts_num); in mvpp2_bm_pool_use()
1148 mvpp2_bm_pool_bufsize_set(port->priv, new_pool, in mvpp2_bm_pool_use()
1149 MVPP2_RX_BUF_SIZE(new_pool->pkt_size)); in mvpp2_bm_pool_use()
1155 mvpp2_bm_pool_use_percpu(struct mvpp2_port *port, int type, in mvpp2_bm_pool_use_percpu() argument
1158 struct mvpp2_bm_pool *new_pool = &port->priv->bm_pools[pool]; in mvpp2_bm_pool_use_percpu()
1161 if (pool > port->nrxqs * 2) { in mvpp2_bm_pool_use_percpu()
1162 netdev_err(port->dev, "Invalid pool %d\n", pool); in mvpp2_bm_pool_use_percpu()
1169 if (new_pool->pkt_size == 0) { in mvpp2_bm_pool_use_percpu()
1175 pkts_num = new_pool->buf_num; in mvpp2_bm_pool_use_percpu()
1179 mvpp2_bm_bufs_free(port->dev->dev.parent, in mvpp2_bm_pool_use_percpu()
1180 port->priv, new_pool, pkts_num); in mvpp2_bm_pool_use_percpu()
1182 new_pool->pkt_size = pkt_size; in mvpp2_bm_pool_use_percpu()
1183 new_pool->frag_size = in mvpp2_bm_pool_use_percpu()
1188 num = mvpp2_bm_bufs_add(port, new_pool, pkts_num); in mvpp2_bm_pool_use_percpu()
1191 new_pool->id, num, pkts_num); in mvpp2_bm_pool_use_percpu()
1196 mvpp2_bm_pool_bufsize_set(port->priv, new_pool, in mvpp2_bm_pool_use_percpu()
1197 MVPP2_RX_BUF_SIZE(new_pool->pkt_size)); in mvpp2_bm_pool_use_percpu()
1203 static int mvpp2_swf_bm_pool_init_shared(struct mvpp2_port *port) in mvpp2_swf_bm_pool_init_shared() argument
1208 /* If port pkt_size is higher than 1518B: in mvpp2_swf_bm_pool_init_shared()
1209 * HW Long pool - SW Jumbo pool, HW Short pool - SW Long pool in mvpp2_swf_bm_pool_init_shared()
1210 * else: HW Long pool - SW Long pool, HW Short pool - SW Short pool in mvpp2_swf_bm_pool_init_shared()
1212 if (port->pkt_size > MVPP2_BM_LONG_PKT_SIZE) { in mvpp2_swf_bm_pool_init_shared()
1220 if (!port->pool_long) { in mvpp2_swf_bm_pool_init_shared()
1221 port->pool_long = in mvpp2_swf_bm_pool_init_shared()
1222 mvpp2_bm_pool_use(port, long_log_pool, in mvpp2_swf_bm_pool_init_shared()
1224 if (!port->pool_long) in mvpp2_swf_bm_pool_init_shared()
1225 return -ENOMEM; in mvpp2_swf_bm_pool_init_shared()
1227 port->pool_long->port_map |= BIT(port->id); in mvpp2_swf_bm_pool_init_shared()
1229 for (rxq = 0; rxq < port->nrxqs; rxq++) in mvpp2_swf_bm_pool_init_shared()
1230 mvpp2_rxq_long_pool_set(port, rxq, port->pool_long->id); in mvpp2_swf_bm_pool_init_shared()
1233 if (!port->pool_short) { in mvpp2_swf_bm_pool_init_shared()
1234 port->pool_short = in mvpp2_swf_bm_pool_init_shared()
1235 mvpp2_bm_pool_use(port, short_log_pool, in mvpp2_swf_bm_pool_init_shared()
1237 if (!port->pool_short) in mvpp2_swf_bm_pool_init_shared()
1238 return -ENOMEM; in mvpp2_swf_bm_pool_init_shared()
1240 port->pool_short->port_map |= BIT(port->id); in mvpp2_swf_bm_pool_init_shared()
1242 for (rxq = 0; rxq < port->nrxqs; rxq++) in mvpp2_swf_bm_pool_init_shared()
1243 mvpp2_rxq_short_pool_set(port, rxq, in mvpp2_swf_bm_pool_init_shared()
1244 port->pool_short->id); in mvpp2_swf_bm_pool_init_shared()
1251 static int mvpp2_swf_bm_pool_init_percpu(struct mvpp2_port *port) in mvpp2_swf_bm_pool_init_percpu() argument
1256 for (i = 0; i < port->nrxqs; i++) { in mvpp2_swf_bm_pool_init_percpu()
1257 bm_pool = mvpp2_bm_pool_use_percpu(port, MVPP2_BM_SHORT, i, in mvpp2_swf_bm_pool_init_percpu()
1260 return -ENOMEM; in mvpp2_swf_bm_pool_init_percpu()
1262 bm_pool->port_map |= BIT(port->id); in mvpp2_swf_bm_pool_init_percpu()
1263 mvpp2_rxq_short_pool_set(port, i, bm_pool->id); in mvpp2_swf_bm_pool_init_percpu()
1266 for (i = 0; i < port->nrxqs; i++) { in mvpp2_swf_bm_pool_init_percpu()
1267 bm_pool = mvpp2_bm_pool_use_percpu(port, MVPP2_BM_LONG, i + port->nrxqs, in mvpp2_swf_bm_pool_init_percpu()
1270 return -ENOMEM; in mvpp2_swf_bm_pool_init_percpu()
1272 bm_pool->port_map |= BIT(port->id); in mvpp2_swf_bm_pool_init_percpu()
1273 mvpp2_rxq_long_pool_set(port, i, bm_pool->id); in mvpp2_swf_bm_pool_init_percpu()
1276 port->pool_long = NULL; in mvpp2_swf_bm_pool_init_percpu()
1277 port->pool_short = NULL; in mvpp2_swf_bm_pool_init_percpu()
1282 static int mvpp2_swf_bm_pool_init(struct mvpp2_port *port) in mvpp2_swf_bm_pool_init() argument
1284 if (port->priv->percpu_pools) in mvpp2_swf_bm_pool_init()
1285 return mvpp2_swf_bm_pool_init_percpu(port); in mvpp2_swf_bm_pool_init()
1287 return mvpp2_swf_bm_pool_init_shared(port); in mvpp2_swf_bm_pool_init()
1290 static void mvpp2_set_hw_csum(struct mvpp2_port *port, in mvpp2_set_hw_csum() argument
1295 /* Update L4 checksum when jumbo enable/disable on port. in mvpp2_set_hw_csum()
1296 * Only port 0 supports hardware checksum offload due to in mvpp2_set_hw_csum()
1301 if (new_long_pool == MVPP2_BM_JUMBO && port->id != 0) { in mvpp2_set_hw_csum()
1302 port->dev->features &= ~csums; in mvpp2_set_hw_csum()
1303 port->dev->hw_features &= ~csums; in mvpp2_set_hw_csum()
1305 port->dev->features |= csums; in mvpp2_set_hw_csum()
1306 port->dev->hw_features |= csums; in mvpp2_set_hw_csum()
1312 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_bm_update_mtu() local
1316 if (port->priv->percpu_pools) in mvpp2_bm_update_mtu()
1319 /* If port MTU is higher than 1518B: in mvpp2_bm_update_mtu()
1320 * HW Long pool - SW Jumbo pool, HW Short pool - SW Long pool in mvpp2_bm_update_mtu()
1321 * else: HW Long pool - SW Long pool, HW Short pool - SW Short pool in mvpp2_bm_update_mtu()
1328 if (new_long_pool != port->pool_long->id) { in mvpp2_bm_update_mtu()
1329 if (port->tx_fc) { in mvpp2_bm_update_mtu()
1331 mvpp2_bm_pool_update_fc(port, in mvpp2_bm_update_mtu()
1332 port->pool_short, in mvpp2_bm_update_mtu()
1335 mvpp2_bm_pool_update_fc(port, port->pool_long, in mvpp2_bm_update_mtu()
1339 /* Remove port from old short & long pool */ in mvpp2_bm_update_mtu()
1340 port->pool_long = mvpp2_bm_pool_use(port, port->pool_long->id, in mvpp2_bm_update_mtu()
1341 port->pool_long->pkt_size); in mvpp2_bm_update_mtu()
1342 port->pool_long->port_map &= ~BIT(port->id); in mvpp2_bm_update_mtu()
1343 port->pool_long = NULL; in mvpp2_bm_update_mtu()
1345 port->pool_short = mvpp2_bm_pool_use(port, port->pool_short->id, in mvpp2_bm_update_mtu()
1346 port->pool_short->pkt_size); in mvpp2_bm_update_mtu()
1347 port->pool_short->port_map &= ~BIT(port->id); in mvpp2_bm_update_mtu()
1348 port->pool_short = NULL; in mvpp2_bm_update_mtu()
1350 port->pkt_size = pkt_size; in mvpp2_bm_update_mtu()
1352 /* Add port to new short & long pool */ in mvpp2_bm_update_mtu()
1353 mvpp2_swf_bm_pool_init(port); in mvpp2_bm_update_mtu()
1355 mvpp2_set_hw_csum(port, new_long_pool); in mvpp2_bm_update_mtu()
1357 if (port->tx_fc) { in mvpp2_bm_update_mtu()
1359 mvpp2_bm_pool_update_fc(port, port->pool_long, in mvpp2_bm_update_mtu()
1362 mvpp2_bm_pool_update_fc(port, port->pool_short, in mvpp2_bm_update_mtu()
1366 /* Update L4 checksum when jumbo enable/disable on port */ in mvpp2_bm_update_mtu()
1367 if (new_long_pool == MVPP2_BM_JUMBO && port->id != 0) { in mvpp2_bm_update_mtu()
1368 dev->features &= ~(NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM); in mvpp2_bm_update_mtu()
1369 dev->hw_features &= ~(NETIF_F_IP_CSUM | in mvpp2_bm_update_mtu()
1372 dev->features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; in mvpp2_bm_update_mtu()
1373 dev->hw_features |= NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; in mvpp2_bm_update_mtu()
1378 dev->mtu = mtu; in mvpp2_bm_update_mtu()
1379 dev->wanted_features = dev->features; in mvpp2_bm_update_mtu()
1385 static inline void mvpp2_interrupts_enable(struct mvpp2_port *port) in mvpp2_interrupts_enable() argument
1389 for (i = 0; i < port->nqvecs; i++) in mvpp2_interrupts_enable()
1390 sw_thread_mask |= port->qvecs[i].sw_thread_mask; in mvpp2_interrupts_enable()
1392 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id), in mvpp2_interrupts_enable()
1396 static inline void mvpp2_interrupts_disable(struct mvpp2_port *port) in mvpp2_interrupts_disable() argument
1400 for (i = 0; i < port->nqvecs; i++) in mvpp2_interrupts_disable()
1401 sw_thread_mask |= port->qvecs[i].sw_thread_mask; in mvpp2_interrupts_disable()
1403 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id), in mvpp2_interrupts_disable()
1409 struct mvpp2_port *port = qvec->port; in mvpp2_qvec_interrupt_enable() local
1411 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id), in mvpp2_qvec_interrupt_enable()
1412 MVPP2_ISR_ENABLE_INTERRUPT(qvec->sw_thread_mask)); in mvpp2_qvec_interrupt_enable()
1417 struct mvpp2_port *port = qvec->port; in mvpp2_qvec_interrupt_disable() local
1419 mvpp2_write(port->priv, MVPP2_ISR_ENABLE_REG(port->id), in mvpp2_qvec_interrupt_disable()
1420 MVPP2_ISR_DISABLE_INTERRUPT(qvec->sw_thread_mask)); in mvpp2_qvec_interrupt_disable()
1429 struct mvpp2_port *port = arg; in mvpp2_interrupts_mask() local
1434 if (cpu > port->priv->nthreads) in mvpp2_interrupts_mask()
1437 thread = mvpp2_cpu_to_thread(port->priv, cpu); in mvpp2_interrupts_mask()
1439 mvpp2_thread_write(port->priv, thread, in mvpp2_interrupts_mask()
1440 MVPP2_ISR_RX_TX_MASK_REG(port->id), 0); in mvpp2_interrupts_mask()
1441 mvpp2_thread_write(port->priv, thread, in mvpp2_interrupts_mask()
1442 MVPP2_ISR_RX_ERR_CAUSE_REG(port->id), 0); in mvpp2_interrupts_mask()
1451 struct mvpp2_port *port = arg; in mvpp2_interrupts_unmask() local
1456 if (cpu >= port->priv->nthreads) in mvpp2_interrupts_unmask()
1459 thread = mvpp2_cpu_to_thread(port->priv, cpu); in mvpp2_interrupts_unmask()
1462 MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(port->priv->hw_version); in mvpp2_interrupts_unmask()
1463 if (port->has_tx_irqs) in mvpp2_interrupts_unmask()
1466 mvpp2_thread_write(port->priv, thread, in mvpp2_interrupts_unmask()
1467 MVPP2_ISR_RX_TX_MASK_REG(port->id), val); in mvpp2_interrupts_unmask()
1468 mvpp2_thread_write(port->priv, thread, in mvpp2_interrupts_unmask()
1469 MVPP2_ISR_RX_ERR_CAUSE_REG(port->id), in mvpp2_interrupts_unmask()
1474 mvpp2_shared_interrupt_mask_unmask(struct mvpp2_port *port, bool mask) in mvpp2_shared_interrupt_mask_unmask() argument
1479 if (port->priv->hw_version == MVPP21) in mvpp2_shared_interrupt_mask_unmask()
1487 for (i = 0; i < port->nqvecs; i++) { in mvpp2_shared_interrupt_mask_unmask()
1488 struct mvpp2_queue_vector *v = port->qvecs + i; in mvpp2_shared_interrupt_mask_unmask()
1490 if (v->type != MVPP2_QUEUE_VECTOR_SHARED) in mvpp2_shared_interrupt_mask_unmask()
1493 mvpp2_thread_write(port->priv, v->sw_thread_id, in mvpp2_shared_interrupt_mask_unmask()
1494 MVPP2_ISR_RX_TX_MASK_REG(port->id), val); in mvpp2_shared_interrupt_mask_unmask()
1495 mvpp2_thread_write(port->priv, v->sw_thread_id, in mvpp2_shared_interrupt_mask_unmask()
1496 MVPP2_ISR_RX_ERR_CAUSE_REG(port->id), in mvpp2_shared_interrupt_mask_unmask()
1501 /* Only GOP port 0 has an XLG MAC */
1502 static bool mvpp2_port_supports_xlg(struct mvpp2_port *port) in mvpp2_port_supports_xlg() argument
1504 return port->gop_id == 0; in mvpp2_port_supports_xlg()
1507 static bool mvpp2_port_supports_rgmii(struct mvpp2_port *port) in mvpp2_port_supports_rgmii() argument
1509 return !(port->priv->hw_version >= MVPP22 && port->gop_id == 0); in mvpp2_port_supports_rgmii()
1512 /* Port configuration routines */
1531 static void mvpp22_gop_init_rgmii(struct mvpp2_port *port) in mvpp22_gop_init_rgmii() argument
1533 struct mvpp2 *priv = port->priv; in mvpp22_gop_init_rgmii()
1536 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val); in mvpp22_gop_init_rgmii()
1538 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val); in mvpp22_gop_init_rgmii()
1540 regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val); in mvpp22_gop_init_rgmii()
1541 if (port->gop_id == 2) in mvpp22_gop_init_rgmii()
1543 else if (port->gop_id == 3) in mvpp22_gop_init_rgmii()
1545 regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val); in mvpp22_gop_init_rgmii()
1548 static void mvpp22_gop_init_sgmii(struct mvpp2_port *port) in mvpp22_gop_init_sgmii() argument
1550 struct mvpp2 *priv = port->priv; in mvpp22_gop_init_sgmii()
1553 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val); in mvpp22_gop_init_sgmii()
1556 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val); in mvpp22_gop_init_sgmii()
1558 if (port->gop_id > 1) { in mvpp22_gop_init_sgmii()
1559 regmap_read(priv->sysctrl_base, GENCONF_CTRL0, &val); in mvpp22_gop_init_sgmii()
1560 if (port->gop_id == 2) in mvpp22_gop_init_sgmii()
1562 else if (port->gop_id == 3) in mvpp22_gop_init_sgmii()
1564 regmap_write(priv->sysctrl_base, GENCONF_CTRL0, val); in mvpp22_gop_init_sgmii()
1568 static void mvpp22_gop_init_10gkr(struct mvpp2_port *port) in mvpp22_gop_init_10gkr() argument
1570 struct mvpp2 *priv = port->priv; in mvpp22_gop_init_10gkr()
1571 void __iomem *mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id); in mvpp22_gop_init_10gkr()
1572 void __iomem *xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id); in mvpp22_gop_init_10gkr()
1591 static void mvpp22_gop_fca_enable_periodic(struct mvpp2_port *port, bool en) in mvpp22_gop_fca_enable_periodic() argument
1593 struct mvpp2 *priv = port->priv; in mvpp22_gop_fca_enable_periodic()
1594 void __iomem *fca = priv->iface_base + MVPP22_FCA_BASE(port->gop_id); in mvpp22_gop_fca_enable_periodic()
1604 static void mvpp22_gop_fca_set_timer(struct mvpp2_port *port, u32 timer) in mvpp22_gop_fca_set_timer() argument
1606 struct mvpp2 *priv = port->priv; in mvpp22_gop_fca_set_timer()
1607 void __iomem *fca = priv->iface_base + MVPP22_FCA_BASE(port->gop_id); in mvpp22_gop_fca_set_timer()
1618 * partner won't send traffic if port is in XOFF mode.
1620 static void mvpp22_gop_fca_set_periodic_timer(struct mvpp2_port *port) in mvpp22_gop_fca_set_periodic_timer() argument
1624 timer = (port->priv->tclk / (USEC_PER_SEC * FC_CLK_DIVIDER)) in mvpp22_gop_fca_set_periodic_timer()
1627 mvpp22_gop_fca_enable_periodic(port, false); in mvpp22_gop_fca_set_periodic_timer()
1629 mvpp22_gop_fca_set_timer(port, timer); in mvpp22_gop_fca_set_periodic_timer()
1631 mvpp22_gop_fca_enable_periodic(port, true); in mvpp22_gop_fca_set_periodic_timer()
1634 static int mvpp22_gop_init(struct mvpp2_port *port, phy_interface_t interface) in mvpp22_gop_init() argument
1636 struct mvpp2 *priv = port->priv; in mvpp22_gop_init()
1639 if (!priv->sysctrl_base) in mvpp22_gop_init()
1647 if (!mvpp2_port_supports_rgmii(port)) in mvpp22_gop_init()
1649 mvpp22_gop_init_rgmii(port); in mvpp22_gop_init()
1654 mvpp22_gop_init_sgmii(port); in mvpp22_gop_init()
1658 if (!mvpp2_port_supports_xlg(port)) in mvpp22_gop_init()
1660 mvpp22_gop_init_10gkr(port); in mvpp22_gop_init()
1666 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL1, &val); in mvpp22_gop_init()
1667 val |= GENCONF_PORT_CTRL1_RESET(port->gop_id) | in mvpp22_gop_init()
1668 GENCONF_PORT_CTRL1_EN(port->gop_id); in mvpp22_gop_init()
1669 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL1, val); in mvpp22_gop_init()
1671 regmap_read(priv->sysctrl_base, GENCONF_PORT_CTRL0, &val); in mvpp22_gop_init()
1673 regmap_write(priv->sysctrl_base, GENCONF_PORT_CTRL0, val); in mvpp22_gop_init()
1675 regmap_read(priv->sysctrl_base, GENCONF_SOFT_RESET1, &val); in mvpp22_gop_init()
1677 regmap_write(priv->sysctrl_base, GENCONF_SOFT_RESET1, val); in mvpp22_gop_init()
1679 mvpp22_gop_fca_set_periodic_timer(port); in mvpp22_gop_init()
1685 netdev_err(port->dev, "Invalid port configuration\n"); in mvpp22_gop_init()
1686 return -EINVAL; in mvpp22_gop_init()
1689 static void mvpp22_gop_unmask_irq(struct mvpp2_port *port) in mvpp22_gop_unmask_irq() argument
1693 if (phy_interface_mode_is_rgmii(port->phy_interface) || in mvpp22_gop_unmask_irq()
1694 phy_interface_mode_is_8023z(port->phy_interface) || in mvpp22_gop_unmask_irq()
1695 port->phy_interface == PHY_INTERFACE_MODE_SGMII) { in mvpp22_gop_unmask_irq()
1696 /* Enable the GMAC link status irq for this port */ in mvpp22_gop_unmask_irq()
1697 val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK); in mvpp22_gop_unmask_irq()
1699 writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK); in mvpp22_gop_unmask_irq()
1702 if (mvpp2_port_supports_xlg(port)) { in mvpp22_gop_unmask_irq()
1703 /* Enable the XLG/GIG irqs for this port */ in mvpp22_gop_unmask_irq()
1704 val = readl(port->base + MVPP22_XLG_EXT_INT_MASK); in mvpp22_gop_unmask_irq()
1705 if (mvpp2_is_xlg(port->phy_interface)) in mvpp22_gop_unmask_irq()
1709 writel(val, port->base + MVPP22_XLG_EXT_INT_MASK); in mvpp22_gop_unmask_irq()
1713 static void mvpp22_gop_mask_irq(struct mvpp2_port *port) in mvpp22_gop_mask_irq() argument
1717 if (mvpp2_port_supports_xlg(port)) { in mvpp22_gop_mask_irq()
1718 val = readl(port->base + MVPP22_XLG_EXT_INT_MASK); in mvpp22_gop_mask_irq()
1721 writel(val, port->base + MVPP22_XLG_EXT_INT_MASK); in mvpp22_gop_mask_irq()
1724 if (phy_interface_mode_is_rgmii(port->phy_interface) || in mvpp22_gop_mask_irq()
1725 phy_interface_mode_is_8023z(port->phy_interface) || in mvpp22_gop_mask_irq()
1726 port->phy_interface == PHY_INTERFACE_MODE_SGMII) { in mvpp22_gop_mask_irq()
1727 val = readl(port->base + MVPP22_GMAC_INT_SUM_MASK); in mvpp22_gop_mask_irq()
1729 writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK); in mvpp22_gop_mask_irq()
1733 static void mvpp22_gop_setup_irq(struct mvpp2_port *port) in mvpp22_gop_setup_irq() argument
1737 mvpp2_modify(port->base + MVPP22_GMAC_INT_SUM_MASK, in mvpp22_gop_setup_irq()
1741 if (port->phylink || in mvpp22_gop_setup_irq()
1742 phy_interface_mode_is_rgmii(port->phy_interface) || in mvpp22_gop_setup_irq()
1743 phy_interface_mode_is_8023z(port->phy_interface) || in mvpp22_gop_setup_irq()
1744 port->phy_interface == PHY_INTERFACE_MODE_SGMII) { in mvpp22_gop_setup_irq()
1745 val = readl(port->base + MVPP22_GMAC_INT_MASK); in mvpp22_gop_setup_irq()
1747 writel(val, port->base + MVPP22_GMAC_INT_MASK); in mvpp22_gop_setup_irq()
1750 if (mvpp2_port_supports_xlg(port)) { in mvpp22_gop_setup_irq()
1751 val = readl(port->base + MVPP22_XLG_INT_MASK); in mvpp22_gop_setup_irq()
1753 writel(val, port->base + MVPP22_XLG_INT_MASK); in mvpp22_gop_setup_irq()
1755 mvpp2_modify(port->base + MVPP22_XLG_EXT_INT_MASK, in mvpp22_gop_setup_irq()
1760 mvpp22_gop_unmask_irq(port); in mvpp22_gop_setup_irq()
1771 * "PPv2 (2500BaseX) - COMPHY (2500SGMII)" are valid.
1773 static int mvpp22_comphy_init(struct mvpp2_port *port, in mvpp22_comphy_init() argument
1778 if (!port->comphy) in mvpp22_comphy_init()
1781 ret = phy_set_mode_ext(port->comphy, PHY_MODE_ETHERNET, interface); in mvpp22_comphy_init()
1785 return phy_power_on(port->comphy); in mvpp22_comphy_init()
1788 static void mvpp2_port_enable(struct mvpp2_port *port) in mvpp2_port_enable() argument
1792 if (mvpp2_port_supports_xlg(port) && in mvpp2_port_enable()
1793 mvpp2_is_xlg(port->phy_interface)) { in mvpp2_port_enable()
1794 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_enable()
1797 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_enable()
1799 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_port_enable()
1802 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_port_enable()
1806 static void mvpp2_port_disable(struct mvpp2_port *port) in mvpp2_port_disable() argument
1810 if (mvpp2_port_supports_xlg(port) && in mvpp2_port_disable()
1811 mvpp2_is_xlg(port->phy_interface)) { in mvpp2_port_disable()
1812 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_disable()
1814 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_disable()
1817 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_port_disable()
1819 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_port_disable()
1823 static void mvpp2_port_periodic_xon_disable(struct mvpp2_port *port) in mvpp2_port_periodic_xon_disable() argument
1827 val = readl(port->base + MVPP2_GMAC_CTRL_1_REG) & in mvpp2_port_periodic_xon_disable()
1829 writel(val, port->base + MVPP2_GMAC_CTRL_1_REG); in mvpp2_port_periodic_xon_disable()
1832 /* Configure loopback port */
1833 static void mvpp2_port_loopback_set(struct mvpp2_port *port, in mvpp2_port_loopback_set() argument
1838 val = readl(port->base + MVPP2_GMAC_CTRL_1_REG); in mvpp2_port_loopback_set()
1840 if (state->speed == 1000) in mvpp2_port_loopback_set()
1845 if (phy_interface_mode_is_8023z(state->interface) || in mvpp2_port_loopback_set()
1846 state->interface == PHY_INTERFACE_MODE_SGMII) in mvpp2_port_loopback_set()
1851 writel(val, port->base + MVPP2_GMAC_CTRL_1_REG); in mvpp2_port_loopback_set()
1870 static u64 mvpp2_read_count(struct mvpp2_port *port, in mvpp2_read_count() argument
1875 val = readl(port->stats_base + counter->offset); in mvpp2_read_count()
1876 if (counter->reg_is_64b) in mvpp2_read_count()
1877 val += (u64)readl(port->stats_base + counter->offset + 4) << 32; in mvpp2_read_count()
1975 struct mvpp2_port *port = netdev_priv(netdev); in mvpp2_ethtool_get_strings() local
1993 for (q = 0; q < port->ntxqs; q++) { in mvpp2_ethtool_get_strings()
2001 for (q = 0; q < port->nrxqs; q++) { in mvpp2_ethtool_get_strings()
2018 mvpp2_get_xdp_stats(struct mvpp2_port *port, struct mvpp2_pcpu_stats *xdp_stats) in mvpp2_get_xdp_stats() argument
2034 cpu_stats = per_cpu_ptr(port->stats, cpu); in mvpp2_get_xdp_stats()
2036 start = u64_stats_fetch_begin(&cpu_stats->syncp); in mvpp2_get_xdp_stats()
2037 xdp_redirect = cpu_stats->xdp_redirect; in mvpp2_get_xdp_stats()
2038 xdp_pass = cpu_stats->xdp_pass; in mvpp2_get_xdp_stats()
2039 xdp_drop = cpu_stats->xdp_drop; in mvpp2_get_xdp_stats()
2040 xdp_xmit = cpu_stats->xdp_xmit; in mvpp2_get_xdp_stats()
2041 xdp_xmit_err = cpu_stats->xdp_xmit_err; in mvpp2_get_xdp_stats()
2042 xdp_tx = cpu_stats->xdp_tx; in mvpp2_get_xdp_stats()
2043 xdp_tx_err = cpu_stats->xdp_tx_err; in mvpp2_get_xdp_stats()
2044 } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); in mvpp2_get_xdp_stats()
2046 xdp_stats->xdp_redirect += xdp_redirect; in mvpp2_get_xdp_stats()
2047 xdp_stats->xdp_pass += xdp_pass; in mvpp2_get_xdp_stats()
2048 xdp_stats->xdp_drop += xdp_drop; in mvpp2_get_xdp_stats()
2049 xdp_stats->xdp_xmit += xdp_xmit; in mvpp2_get_xdp_stats()
2050 xdp_stats->xdp_xmit_err += xdp_xmit_err; in mvpp2_get_xdp_stats()
2051 xdp_stats->xdp_tx += xdp_tx; in mvpp2_get_xdp_stats()
2052 xdp_stats->xdp_tx_err += xdp_tx_err; in mvpp2_get_xdp_stats()
2056 static void mvpp2_read_stats(struct mvpp2_port *port) in mvpp2_read_stats() argument
2063 pstats = port->ethtool_stats; in mvpp2_read_stats()
2066 *pstats++ += mvpp2_read_count(port, &mvpp2_ethtool_mib_regs[i]); in mvpp2_read_stats()
2069 *pstats++ += mvpp2_read(port->priv, in mvpp2_read_stats()
2071 4 * port->id); in mvpp2_read_stats()
2073 for (q = 0; q < port->ntxqs; q++) in mvpp2_read_stats()
2075 *pstats++ += mvpp2_read_index(port->priv, in mvpp2_read_stats()
2076 MVPP22_CTRS_TX_CTR(port->id, q), in mvpp2_read_stats()
2080 * driver's. We need to add the port->first_rxq offset. in mvpp2_read_stats()
2082 for (q = 0; q < port->nrxqs; q++) in mvpp2_read_stats()
2084 *pstats++ += mvpp2_read_index(port->priv, in mvpp2_read_stats()
2085 port->first_rxq + q, in mvpp2_read_stats()
2089 mvpp2_get_xdp_stats(port, &xdp_stats); in mvpp2_read_stats()
2094 switch (s->offset) { in mvpp2_read_stats()
2123 struct mvpp2_port *port = container_of(del_work, struct mvpp2_port, in mvpp2_gather_hw_statistics() local
2126 mutex_lock(&port->gather_stats_lock); in mvpp2_gather_hw_statistics()
2128 mvpp2_read_stats(port); in mvpp2_gather_hw_statistics()
2133 cancel_delayed_work(&port->stats_work); in mvpp2_gather_hw_statistics()
2134 queue_delayed_work(port->priv->stats_queue, &port->stats_work, in mvpp2_gather_hw_statistics()
2137 mutex_unlock(&port->gather_stats_lock); in mvpp2_gather_hw_statistics()
2143 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_stats() local
2145 /* Update statistics for the given port, then take the lock to avoid in mvpp2_ethtool_get_stats()
2148 mvpp2_gather_hw_statistics(&port->stats_work.work); in mvpp2_ethtool_get_stats()
2150 mutex_lock(&port->gather_stats_lock); in mvpp2_ethtool_get_stats()
2151 memcpy(data, port->ethtool_stats, in mvpp2_ethtool_get_stats()
2152 sizeof(u64) * MVPP2_N_ETHTOOL_STATS(port->ntxqs, port->nrxqs)); in mvpp2_ethtool_get_stats()
2153 mutex_unlock(&port->gather_stats_lock); in mvpp2_ethtool_get_stats()
2158 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_sset_count() local
2161 return MVPP2_N_ETHTOOL_STATS(port->ntxqs, port->nrxqs); in mvpp2_ethtool_get_sset_count()
2163 return -EOPNOTSUPP; in mvpp2_ethtool_get_sset_count()
2166 static void mvpp2_mac_reset_assert(struct mvpp2_port *port) in mvpp2_mac_reset_assert() argument
2170 val = readl(port->base + MVPP2_GMAC_CTRL_2_REG) | in mvpp2_mac_reset_assert()
2172 writel(val, port->base + MVPP2_GMAC_CTRL_2_REG); in mvpp2_mac_reset_assert()
2174 if (port->priv->hw_version >= MVPP22 && port->gop_id == 0) { in mvpp2_mac_reset_assert()
2175 val = readl(port->base + MVPP22_XLG_CTRL0_REG) & in mvpp2_mac_reset_assert()
2177 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_mac_reset_assert()
2181 static void mvpp22_pcs_reset_assert(struct mvpp2_port *port) in mvpp22_pcs_reset_assert() argument
2183 struct mvpp2 *priv = port->priv; in mvpp22_pcs_reset_assert()
2187 if (port->priv->hw_version == MVPP21 || port->gop_id != 0) in mvpp22_pcs_reset_assert()
2190 mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id); in mvpp22_pcs_reset_assert()
2191 xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id); in mvpp22_pcs_reset_assert()
2202 static void mvpp22_pcs_reset_deassert(struct mvpp2_port *port, in mvpp22_pcs_reset_deassert() argument
2205 struct mvpp2 *priv = port->priv; in mvpp22_pcs_reset_deassert()
2209 if (port->priv->hw_version == MVPP21 || port->gop_id != 0) in mvpp22_pcs_reset_deassert()
2212 mpcs = priv->iface_base + MVPP22_MPCS_BASE(port->gop_id); in mvpp22_pcs_reset_deassert()
2213 xpcs = priv->iface_base + MVPP22_XPCS_BASE(port->gop_id); in mvpp22_pcs_reset_deassert()
2234 /* Change maximum receive size of the port */
2235 static inline void mvpp2_gmac_max_rx_size_set(struct mvpp2_port *port) in mvpp2_gmac_max_rx_size_set() argument
2239 val = readl(port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_gmac_max_rx_size_set()
2241 val |= (((port->pkt_size - MVPP2_MH_SIZE) / 2) << in mvpp2_gmac_max_rx_size_set()
2243 writel(val, port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_gmac_max_rx_size_set()
2246 /* Change maximum receive size of the port */
2247 static inline void mvpp2_xlg_max_rx_size_set(struct mvpp2_port *port) in mvpp2_xlg_max_rx_size_set() argument
2251 val = readl(port->base + MVPP22_XLG_CTRL1_REG); in mvpp2_xlg_max_rx_size_set()
2253 val |= ((port->pkt_size - MVPP2_MH_SIZE) / 2) << in mvpp2_xlg_max_rx_size_set()
2255 writel(val, port->base + MVPP22_XLG_CTRL1_REG); in mvpp2_xlg_max_rx_size_set()
2258 /* Set defaults to the MVPP2 port */
2259 static void mvpp2_defaults_set(struct mvpp2_port *port) in mvpp2_defaults_set() argument
2263 if (port->priv->hw_version == MVPP21) { in mvpp2_defaults_set()
2265 val = readl(port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG); in mvpp2_defaults_set()
2268 val |= MVPP2_GMAC_TX_FIFO_MIN_TH_MASK(64 - 4 - 2); in mvpp2_defaults_set()
2269 writel(val, port->base + MVPP2_GMAC_PORT_FIFO_CFG_1_REG); in mvpp2_defaults_set()
2273 tx_port_num = mvpp2_egress_port(port); in mvpp2_defaults_set()
2274 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, in mvpp2_defaults_set()
2276 mvpp2_write(port->priv, MVPP2_TXP_SCHED_CMD_1_REG, 0); in mvpp2_defaults_set()
2278 /* Set TXQ scheduling to Round-Robin */ in mvpp2_defaults_set()
2279 mvpp2_write(port->priv, MVPP2_TXP_SCHED_FIXED_PRIO_REG, 0); in mvpp2_defaults_set()
2283 mvpp2_write(port->priv, in mvpp2_defaults_set()
2289 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PERIOD_REG, in mvpp2_defaults_set()
2290 port->priv->tclk / USEC_PER_SEC); in mvpp2_defaults_set()
2291 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_REFILL_REG); in mvpp2_defaults_set()
2295 mvpp2_write(port->priv, MVPP2_TXP_SCHED_REFILL_REG, val); in mvpp2_defaults_set()
2297 mvpp2_write(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG, val); in mvpp2_defaults_set()
2300 mvpp2_write(port->priv, MVPP2_RX_CTRL_REG(port->id), in mvpp2_defaults_set()
2305 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) { in mvpp2_defaults_set()
2306 queue = port->rxqs[lrxq]->id; in mvpp2_defaults_set()
2307 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue)); in mvpp2_defaults_set()
2310 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val); in mvpp2_defaults_set()
2314 mvpp2_interrupts_disable(port); in mvpp2_defaults_set()
2318 static void mvpp2_ingress_enable(struct mvpp2_port *port) in mvpp2_ingress_enable() argument
2323 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) { in mvpp2_ingress_enable()
2324 queue = port->rxqs[lrxq]->id; in mvpp2_ingress_enable()
2325 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue)); in mvpp2_ingress_enable()
2327 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val); in mvpp2_ingress_enable()
2331 static void mvpp2_ingress_disable(struct mvpp2_port *port) in mvpp2_ingress_disable() argument
2336 for (lrxq = 0; lrxq < port->nrxqs; lrxq++) { in mvpp2_ingress_disable()
2337 queue = port->rxqs[lrxq]->id; in mvpp2_ingress_disable()
2338 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(queue)); in mvpp2_ingress_disable()
2340 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(queue), val); in mvpp2_ingress_disable()
2345 * - HW starts take descriptors from DRAM
2347 static void mvpp2_egress_enable(struct mvpp2_port *port) in mvpp2_egress_enable() argument
2351 int tx_port_num = mvpp2_egress_port(port); in mvpp2_egress_enable()
2355 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_egress_enable()
2356 struct mvpp2_tx_queue *txq = port->txqs[queue]; in mvpp2_egress_enable()
2358 if (txq->descs) in mvpp2_egress_enable()
2362 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num); in mvpp2_egress_enable()
2363 mvpp2_write(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG, qmap); in mvpp2_egress_enable()
2367 * - HW doesn't take descriptors from DRAM
2369 static void mvpp2_egress_disable(struct mvpp2_port *port) in mvpp2_egress_disable() argument
2373 int tx_port_num = mvpp2_egress_port(port); in mvpp2_egress_disable()
2376 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num); in mvpp2_egress_disable()
2377 reg_data = (mvpp2_read(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG)) & in mvpp2_egress_disable()
2380 mvpp2_write(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG, in mvpp2_egress_disable()
2387 netdev_warn(port->dev, in mvpp2_egress_disable()
2395 /* Check port TX Command register that all in mvpp2_egress_disable()
2398 reg_data = mvpp2_read(port->priv, MVPP2_TXP_SCHED_Q_CMD_REG); in mvpp2_egress_disable()
2406 mvpp2_rxq_received(struct mvpp2_port *port, int rxq_id) in mvpp2_rxq_received() argument
2408 u32 val = mvpp2_read(port->priv, MVPP2_RXQ_STATUS_REG(rxq_id)); in mvpp2_rxq_received()
2417 mvpp2_rxq_status_update(struct mvpp2_port *port, int rxq_id, in mvpp2_rxq_status_update() argument
2425 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_UPDATE_REG(rxq_id), val); in mvpp2_rxq_status_update()
2432 int rx_desc = rxq->next_desc_to_proc; in mvpp2_rxq_next_desc_get()
2434 rxq->next_desc_to_proc = MVPP2_QUEUE_NEXT_DESC(rxq, rx_desc); in mvpp2_rxq_next_desc_get()
2435 prefetch(rxq->descs + rxq->next_desc_to_proc); in mvpp2_rxq_next_desc_get()
2436 return rxq->descs + rx_desc; in mvpp2_rxq_next_desc_get()
2440 static void mvpp2_rxq_offset_set(struct mvpp2_port *port, in mvpp2_rxq_offset_set() argument
2448 val = mvpp2_read(port->priv, MVPP2_RXQ_CONFIG_REG(prxq)); in mvpp2_rxq_offset_set()
2455 mvpp2_write(port->priv, MVPP2_RXQ_CONFIG_REG(prxq), val); in mvpp2_rxq_offset_set()
2464 int tx_desc = txq->next_desc_to_proc; in mvpp2_txq_next_desc_get()
2466 txq->next_desc_to_proc = MVPP2_QUEUE_NEXT_DESC(txq, tx_desc); in mvpp2_txq_next_desc_get()
2467 return txq->descs + tx_desc; in mvpp2_txq_next_desc_get()
2475 static void mvpp2_aggr_txq_pend_desc_add(struct mvpp2_port *port, int pending) in mvpp2_aggr_txq_pend_desc_add() argument
2477 /* aggregated access - relevant TXQ number is written in TX desc */ in mvpp2_aggr_txq_pend_desc_add()
2478 mvpp2_thread_write(port->priv, in mvpp2_aggr_txq_pend_desc_add()
2479 mvpp2_cpu_to_thread(port->priv, smp_processor_id()), in mvpp2_aggr_txq_pend_desc_add()
2489 static int mvpp2_aggr_desc_num_check(struct mvpp2_port *port, in mvpp2_aggr_desc_num_check() argument
2492 if ((aggr_txq->count + num) > MVPP2_AGGR_TXQ_SIZE) { in mvpp2_aggr_desc_num_check()
2495 mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_aggr_desc_num_check()
2496 u32 val = mvpp2_read_relaxed(port->priv, in mvpp2_aggr_desc_num_check()
2499 aggr_txq->count = val & MVPP2_AGGR_TXQ_PENDING_MASK; in mvpp2_aggr_desc_num_check()
2501 if ((aggr_txq->count + num) > MVPP2_AGGR_TXQ_SIZE) in mvpp2_aggr_desc_num_check()
2502 return -ENOMEM; in mvpp2_aggr_desc_num_check()
2513 static int mvpp2_txq_alloc_reserved_desc(struct mvpp2_port *port, in mvpp2_txq_alloc_reserved_desc() argument
2516 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_txq_alloc_reserved_desc()
2517 struct mvpp2 *priv = port->priv; in mvpp2_txq_alloc_reserved_desc()
2520 val = (txq->id << MVPP2_TXQ_RSVD_REQ_Q_OFFSET) | num; in mvpp2_txq_alloc_reserved_desc()
2531 static int mvpp2_txq_reserved_desc_num_proc(struct mvpp2_port *port, in mvpp2_txq_reserved_desc_num_proc() argument
2539 if (txq_pcpu->reserved_num >= num) in mvpp2_txq_reserved_desc_num_proc()
2548 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_txq_reserved_desc_num_proc()
2551 txq_pcpu_aux = per_cpu_ptr(txq->pcpu, thread); in mvpp2_txq_reserved_desc_num_proc()
2552 desc_count += txq_pcpu_aux->count; in mvpp2_txq_reserved_desc_num_proc()
2553 desc_count += txq_pcpu_aux->reserved_num; in mvpp2_txq_reserved_desc_num_proc()
2556 req = max(MVPP2_CPU_DESC_CHUNK, num - txq_pcpu->reserved_num); in mvpp2_txq_reserved_desc_num_proc()
2560 (txq->size - (MVPP2_MAX_THREADS * MVPP2_CPU_DESC_CHUNK))) in mvpp2_txq_reserved_desc_num_proc()
2561 return -ENOMEM; in mvpp2_txq_reserved_desc_num_proc()
2563 txq_pcpu->reserved_num += mvpp2_txq_alloc_reserved_desc(port, txq, req); in mvpp2_txq_reserved_desc_num_proc()
2566 if (txq_pcpu->reserved_num < num) in mvpp2_txq_reserved_desc_num_proc()
2567 return -ENOMEM; in mvpp2_txq_reserved_desc_num_proc()
2576 if (txq->next_desc_to_proc == 0) in mvpp2_txq_desc_put()
2577 txq->next_desc_to_proc = txq->last_desc - 1; in mvpp2_txq_desc_put()
2579 txq->next_desc_to_proc--; in mvpp2_txq_desc_put()
2617 * Per-thread access
2623 static inline int mvpp2_txq_sent_desc_proc(struct mvpp2_port *port, in mvpp2_txq_sent_desc_proc() argument
2629 val = mvpp2_thread_read_relaxed(port->priv, in mvpp2_txq_sent_desc_proc()
2630 mvpp2_cpu_to_thread(port->priv, smp_processor_id()), in mvpp2_txq_sent_desc_proc()
2631 MVPP2_TXQ_SENT_REG(txq->id)); in mvpp2_txq_sent_desc_proc()
2642 struct mvpp2_port *port = arg; in mvpp2_txq_sent_counter_clear() local
2646 if (smp_processor_id() >= port->priv->nthreads) in mvpp2_txq_sent_counter_clear()
2649 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_txq_sent_counter_clear()
2650 int id = port->txqs[queue]->id; in mvpp2_txq_sent_counter_clear()
2652 mvpp2_thread_read(port->priv, in mvpp2_txq_sent_counter_clear()
2653 mvpp2_cpu_to_thread(port->priv, smp_processor_id()), in mvpp2_txq_sent_counter_clear()
2659 static void mvpp2_txp_max_tx_size_set(struct mvpp2_port *port) in mvpp2_txp_max_tx_size_set() argument
2664 mtu = port->pkt_size * 8; in mvpp2_txp_max_tx_size_set()
2672 tx_port_num = mvpp2_egress_port(port); in mvpp2_txp_max_tx_size_set()
2673 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num); in mvpp2_txp_max_tx_size_set()
2676 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_MTU_REG); in mvpp2_txp_max_tx_size_set()
2679 mvpp2_write(port->priv, MVPP2_TXP_SCHED_MTU_REG, val); in mvpp2_txp_max_tx_size_set()
2682 val = mvpp2_read(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG); in mvpp2_txp_max_tx_size_set()
2688 mvpp2_write(port->priv, MVPP2_TXP_SCHED_TOKEN_SIZE_REG, val); in mvpp2_txp_max_tx_size_set()
2691 for (txq = 0; txq < port->ntxqs; txq++) { in mvpp2_txp_max_tx_size_set()
2692 val = mvpp2_read(port->priv, in mvpp2_txp_max_tx_size_set()
2700 mvpp2_write(port->priv, in mvpp2_txp_max_tx_size_set()
2707 /* Set the number of non-occupied descriptors threshold */
2708 static void mvpp2_set_rxq_free_tresh(struct mvpp2_port *port, in mvpp2_set_rxq_free_tresh() argument
2713 mvpp2_write(port->priv, MVPP2_RXQ_NUM_REG, rxq->id); in mvpp2_set_rxq_free_tresh()
2715 val = mvpp2_read(port->priv, MVPP2_RXQ_THRESH_REG); in mvpp2_set_rxq_free_tresh()
2718 mvpp2_write(port->priv, MVPP2_RXQ_THRESH_REG, val); in mvpp2_set_rxq_free_tresh()
2724 static void mvpp2_rx_pkts_coal_set(struct mvpp2_port *port, in mvpp2_rx_pkts_coal_set() argument
2727 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_rx_pkts_coal_set()
2729 if (rxq->pkts_coal > MVPP2_OCCUPIED_THRESH_MASK) in mvpp2_rx_pkts_coal_set()
2730 rxq->pkts_coal = MVPP2_OCCUPIED_THRESH_MASK; in mvpp2_rx_pkts_coal_set()
2732 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id); in mvpp2_rx_pkts_coal_set()
2733 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_THRESH_REG, in mvpp2_rx_pkts_coal_set()
2734 rxq->pkts_coal); in mvpp2_rx_pkts_coal_set()
2740 static void mvpp2_tx_pkts_coal_set(struct mvpp2_port *port, in mvpp2_tx_pkts_coal_set() argument
2746 if (txq->done_pkts_coal > MVPP2_TXQ_THRESH_MASK) in mvpp2_tx_pkts_coal_set()
2747 txq->done_pkts_coal = MVPP2_TXQ_THRESH_MASK; in mvpp2_tx_pkts_coal_set()
2749 val = (txq->done_pkts_coal << MVPP2_TXQ_THRESH_OFFSET); in mvpp2_tx_pkts_coal_set()
2750 /* PKT-coalescing registers are per-queue + per-thread */ in mvpp2_tx_pkts_coal_set()
2752 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id); in mvpp2_tx_pkts_coal_set()
2753 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_THRESH_REG, val); in mvpp2_tx_pkts_coal_set()
2776 static void mvpp2_rx_time_coal_set(struct mvpp2_port *port, in mvpp2_rx_time_coal_set() argument
2779 unsigned long freq = port->priv->tclk; in mvpp2_rx_time_coal_set()
2780 u32 val = mvpp2_usec_to_cycles(rxq->time_coal, freq); in mvpp2_rx_time_coal_set()
2783 rxq->time_coal = in mvpp2_rx_time_coal_set()
2786 /* re-evaluate to get actual register value */ in mvpp2_rx_time_coal_set()
2787 val = mvpp2_usec_to_cycles(rxq->time_coal, freq); in mvpp2_rx_time_coal_set()
2790 mvpp2_write(port->priv, MVPP2_ISR_RX_THRESHOLD_REG(rxq->id), val); in mvpp2_rx_time_coal_set()
2793 static void mvpp2_tx_time_coal_set(struct mvpp2_port *port) in mvpp2_tx_time_coal_set() argument
2795 unsigned long freq = port->priv->tclk; in mvpp2_tx_time_coal_set()
2796 u32 val = mvpp2_usec_to_cycles(port->tx_time_coal, freq); in mvpp2_tx_time_coal_set()
2799 port->tx_time_coal = in mvpp2_tx_time_coal_set()
2802 /* re-evaluate to get actual register value */ in mvpp2_tx_time_coal_set()
2803 val = mvpp2_usec_to_cycles(port->tx_time_coal, freq); in mvpp2_tx_time_coal_set()
2806 mvpp2_write(port->priv, MVPP2_ISR_TX_THRESHOLD_REG(port->id), val); in mvpp2_tx_time_coal_set()
2810 static void mvpp2_txq_bufs_free(struct mvpp2_port *port, in mvpp2_txq_bufs_free() argument
2823 txq_pcpu->buffs + txq_pcpu->txq_get_index; in mvpp2_txq_bufs_free()
2825 if (!IS_TSO_HEADER(txq_pcpu, tx_buf->dma) && in mvpp2_txq_bufs_free()
2826 tx_buf->type != MVPP2_TYPE_XDP_TX) in mvpp2_txq_bufs_free()
2827 dma_unmap_single(port->dev->dev.parent, tx_buf->dma, in mvpp2_txq_bufs_free()
2828 tx_buf->size, DMA_TO_DEVICE); in mvpp2_txq_bufs_free()
2829 if (tx_buf->type == MVPP2_TYPE_SKB && tx_buf->skb) in mvpp2_txq_bufs_free()
2830 dev_kfree_skb_any(tx_buf->skb); in mvpp2_txq_bufs_free()
2831 else if (tx_buf->type == MVPP2_TYPE_XDP_TX || in mvpp2_txq_bufs_free()
2832 tx_buf->type == MVPP2_TYPE_XDP_NDO) in mvpp2_txq_bufs_free()
2833 xdp_return_frame_bulk(tx_buf->xdpf, &bq); in mvpp2_txq_bufs_free()
2842 static inline struct mvpp2_rx_queue *mvpp2_get_rx_queue(struct mvpp2_port *port, in mvpp2_get_rx_queue() argument
2845 int queue = fls(cause) - 1; in mvpp2_get_rx_queue()
2847 return port->rxqs[queue]; in mvpp2_get_rx_queue()
2850 static inline struct mvpp2_tx_queue *mvpp2_get_tx_queue(struct mvpp2_port *port, in mvpp2_get_tx_queue() argument
2853 int queue = fls(cause) - 1; in mvpp2_get_tx_queue()
2855 return port->txqs[queue]; in mvpp2_get_tx_queue()
2859 static void mvpp2_txq_done(struct mvpp2_port *port, struct mvpp2_tx_queue *txq, in mvpp2_txq_done() argument
2862 struct netdev_queue *nq = netdev_get_tx_queue(port->dev, txq->log_id); in mvpp2_txq_done()
2865 if (txq_pcpu->thread != mvpp2_cpu_to_thread(port->priv, smp_processor_id())) in mvpp2_txq_done()
2866 netdev_err(port->dev, "wrong cpu on the end of Tx processing\n"); in mvpp2_txq_done()
2868 tx_done = mvpp2_txq_sent_desc_proc(port, txq); in mvpp2_txq_done()
2871 mvpp2_txq_bufs_free(port, txq, txq_pcpu, tx_done); in mvpp2_txq_done()
2873 txq_pcpu->count -= tx_done; in mvpp2_txq_done()
2876 if (txq_pcpu->count <= txq_pcpu->wake_threshold) in mvpp2_txq_done()
2880 static unsigned int mvpp2_tx_done(struct mvpp2_port *port, u32 cause, in mvpp2_tx_done() argument
2888 txq = mvpp2_get_tx_queue(port, cause); in mvpp2_tx_done()
2892 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_tx_done()
2894 if (txq_pcpu->count) { in mvpp2_tx_done()
2895 mvpp2_txq_done(port, txq, txq_pcpu); in mvpp2_tx_done()
2896 tx_todo += txq_pcpu->count; in mvpp2_tx_done()
2899 cause &= ~(1 << txq->log_id); in mvpp2_tx_done()
2914 aggr_txq->descs = dma_alloc_coherent(&pdev->dev, in mvpp2_aggr_txq_init()
2916 &aggr_txq->descs_dma, GFP_KERNEL); in mvpp2_aggr_txq_init()
2917 if (!aggr_txq->descs) in mvpp2_aggr_txq_init()
2918 return -ENOMEM; in mvpp2_aggr_txq_init()
2920 aggr_txq->last_desc = MVPP2_AGGR_TXQ_SIZE - 1; in mvpp2_aggr_txq_init()
2923 aggr_txq->next_desc_to_proc = mvpp2_read(priv, in mvpp2_aggr_txq_init()
2929 if (priv->hw_version == MVPP21) in mvpp2_aggr_txq_init()
2930 txq_dma = aggr_txq->descs_dma; in mvpp2_aggr_txq_init()
2932 txq_dma = aggr_txq->descs_dma >> in mvpp2_aggr_txq_init()
2943 static int mvpp2_rxq_init(struct mvpp2_port *port, in mvpp2_rxq_init() argument
2946 struct mvpp2 *priv = port->priv; in mvpp2_rxq_init()
2951 rxq->size = port->rx_ring_size; in mvpp2_rxq_init()
2954 rxq->descs = dma_alloc_coherent(port->dev->dev.parent, in mvpp2_rxq_init()
2955 rxq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_rxq_init()
2956 &rxq->descs_dma, GFP_KERNEL); in mvpp2_rxq_init()
2957 if (!rxq->descs) in mvpp2_rxq_init()
2958 return -ENOMEM; in mvpp2_rxq_init()
2960 rxq->last_desc = rxq->size - 1; in mvpp2_rxq_init()
2962 /* Zero occupied and non-occupied counters - direct access */ in mvpp2_rxq_init()
2963 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0); in mvpp2_rxq_init()
2965 /* Set Rx descriptors queue starting address - indirect access */ in mvpp2_rxq_init()
2966 thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_rxq_init()
2967 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id); in mvpp2_rxq_init()
2968 if (port->priv->hw_version == MVPP21) in mvpp2_rxq_init()
2969 rxq_dma = rxq->descs_dma; in mvpp2_rxq_init()
2971 rxq_dma = rxq->descs_dma >> MVPP22_DESC_ADDR_OFFS; in mvpp2_rxq_init()
2972 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_ADDR_REG, rxq_dma); in mvpp2_rxq_init()
2973 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_SIZE_REG, rxq->size); in mvpp2_rxq_init()
2974 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_INDEX_REG, 0); in mvpp2_rxq_init()
2978 mvpp2_rxq_offset_set(port, rxq->id, MVPP2_SKB_HEADROOM); in mvpp2_rxq_init()
2981 mvpp2_rx_pkts_coal_set(port, rxq); in mvpp2_rxq_init()
2982 mvpp2_rx_time_coal_set(port, rxq); in mvpp2_rxq_init()
2985 mvpp2_set_rxq_free_tresh(port, rxq); in mvpp2_rxq_init()
2988 mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size); in mvpp2_rxq_init()
2990 if (priv->percpu_pools) { in mvpp2_rxq_init()
2991 err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->logic_rxq, 0); in mvpp2_rxq_init()
2995 err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->logic_rxq, 0); in mvpp2_rxq_init()
3000 err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq_short, in mvpp2_rxq_init()
3002 priv->page_pool[rxq->logic_rxq]); in mvpp2_rxq_init()
3006 err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq_long, in mvpp2_rxq_init()
3008 priv->page_pool[rxq->logic_rxq + in mvpp2_rxq_init()
3009 port->nrxqs]); in mvpp2_rxq_init()
3017 xdp_rxq_info_unreg_mem_model(&rxq->xdp_rxq_short); in mvpp2_rxq_init()
3019 xdp_rxq_info_unreg(&rxq->xdp_rxq_long); in mvpp2_rxq_init()
3021 xdp_rxq_info_unreg(&rxq->xdp_rxq_short); in mvpp2_rxq_init()
3023 dma_free_coherent(port->dev->dev.parent, in mvpp2_rxq_init()
3024 rxq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_rxq_init()
3025 rxq->descs, rxq->descs_dma); in mvpp2_rxq_init()
3030 static void mvpp2_rxq_drop_pkts(struct mvpp2_port *port, in mvpp2_rxq_drop_pkts() argument
3035 rx_received = mvpp2_rxq_received(port, rxq->id); in mvpp2_rxq_drop_pkts()
3041 u32 status = mvpp2_rxdesc_status_get(port, rx_desc); in mvpp2_rxq_drop_pkts()
3047 mvpp2_bm_pool_put(port, pool, in mvpp2_rxq_drop_pkts()
3048 mvpp2_rxdesc_dma_addr_get(port, rx_desc), in mvpp2_rxq_drop_pkts()
3049 mvpp2_rxdesc_cookie_get(port, rx_desc)); in mvpp2_rxq_drop_pkts()
3051 mvpp2_rxq_status_update(port, rxq->id, rx_received, rx_received); in mvpp2_rxq_drop_pkts()
3055 static void mvpp2_rxq_deinit(struct mvpp2_port *port, in mvpp2_rxq_deinit() argument
3060 if (xdp_rxq_info_is_reg(&rxq->xdp_rxq_short)) in mvpp2_rxq_deinit()
3061 xdp_rxq_info_unreg(&rxq->xdp_rxq_short); in mvpp2_rxq_deinit()
3063 if (xdp_rxq_info_is_reg(&rxq->xdp_rxq_long)) in mvpp2_rxq_deinit()
3064 xdp_rxq_info_unreg(&rxq->xdp_rxq_long); in mvpp2_rxq_deinit()
3066 mvpp2_rxq_drop_pkts(port, rxq); in mvpp2_rxq_deinit()
3068 if (rxq->descs) in mvpp2_rxq_deinit()
3069 dma_free_coherent(port->dev->dev.parent, in mvpp2_rxq_deinit()
3070 rxq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_rxq_deinit()
3071 rxq->descs, in mvpp2_rxq_deinit()
3072 rxq->descs_dma); in mvpp2_rxq_deinit()
3074 rxq->descs = NULL; in mvpp2_rxq_deinit()
3075 rxq->last_desc = 0; in mvpp2_rxq_deinit()
3076 rxq->next_desc_to_proc = 0; in mvpp2_rxq_deinit()
3077 rxq->descs_dma = 0; in mvpp2_rxq_deinit()
3082 mvpp2_write(port->priv, MVPP2_RXQ_STATUS_REG(rxq->id), 0); in mvpp2_rxq_deinit()
3083 thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_rxq_deinit()
3084 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_NUM_REG, rxq->id); in mvpp2_rxq_deinit()
3085 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_ADDR_REG, 0); in mvpp2_rxq_deinit()
3086 mvpp2_thread_write(port->priv, thread, MVPP2_RXQ_DESC_SIZE_REG, 0); in mvpp2_rxq_deinit()
3091 static int mvpp2_txq_init(struct mvpp2_port *port, in mvpp2_txq_init() argument
3099 txq->size = port->tx_ring_size; in mvpp2_txq_init()
3102 txq->descs = dma_alloc_coherent(port->dev->dev.parent, in mvpp2_txq_init()
3103 txq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_txq_init()
3104 &txq->descs_dma, GFP_KERNEL); in mvpp2_txq_init()
3105 if (!txq->descs) in mvpp2_txq_init()
3106 return -ENOMEM; in mvpp2_txq_init()
3108 txq->last_desc = txq->size - 1; in mvpp2_txq_init()
3110 /* Set Tx descriptors queue starting address - indirect access */ in mvpp2_txq_init()
3111 thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_txq_init()
3112 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id); in mvpp2_txq_init()
3113 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_ADDR_REG, in mvpp2_txq_init()
3114 txq->descs_dma); in mvpp2_txq_init()
3115 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_SIZE_REG, in mvpp2_txq_init()
3116 txq->size & MVPP2_TXQ_DESC_SIZE_MASK); in mvpp2_txq_init()
3117 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_INDEX_REG, 0); in mvpp2_txq_init()
3118 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_RSVD_CLR_REG, in mvpp2_txq_init()
3119 txq->id << MVPP2_TXQ_RSVD_CLR_OFFSET); in mvpp2_txq_init()
3120 val = mvpp2_thread_read(port->priv, thread, MVPP2_TXQ_PENDING_REG); in mvpp2_txq_init()
3122 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PENDING_REG, val); in mvpp2_txq_init()
3126 * TCONTS for PON port must be continuous from 0 to MVPP2_MAX_TCONT in mvpp2_txq_init()
3130 desc = (port->id * MVPP2_MAX_TXQ * desc_per_txq) + in mvpp2_txq_init()
3131 (txq->log_id * desc_per_txq); in mvpp2_txq_init()
3133 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, in mvpp2_txq_init()
3138 /* WRR / EJP configuration - indirect access */ in mvpp2_txq_init()
3139 tx_port_num = mvpp2_egress_port(port); in mvpp2_txq_init()
3140 mvpp2_write(port->priv, MVPP2_TXP_SCHED_PORT_INDEX_REG, tx_port_num); in mvpp2_txq_init()
3142 val = mvpp2_read(port->priv, MVPP2_TXQ_SCHED_REFILL_REG(txq->log_id)); in mvpp2_txq_init()
3146 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_REFILL_REG(txq->log_id), val); in mvpp2_txq_init()
3149 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_SIZE_REG(txq->log_id), in mvpp2_txq_init()
3152 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_txq_init()
3153 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_txq_init()
3154 txq_pcpu->size = txq->size; in mvpp2_txq_init()
3155 txq_pcpu->buffs = kmalloc_array(txq_pcpu->size, in mvpp2_txq_init()
3156 sizeof(*txq_pcpu->buffs), in mvpp2_txq_init()
3158 if (!txq_pcpu->buffs) in mvpp2_txq_init()
3159 return -ENOMEM; in mvpp2_txq_init()
3161 txq_pcpu->count = 0; in mvpp2_txq_init()
3162 txq_pcpu->reserved_num = 0; in mvpp2_txq_init()
3163 txq_pcpu->txq_put_index = 0; in mvpp2_txq_init()
3164 txq_pcpu->txq_get_index = 0; in mvpp2_txq_init()
3165 txq_pcpu->tso_headers = NULL; in mvpp2_txq_init()
3167 txq_pcpu->stop_threshold = txq->size - MVPP2_MAX_SKB_DESCS; in mvpp2_txq_init()
3168 txq_pcpu->wake_threshold = txq_pcpu->stop_threshold / 2; in mvpp2_txq_init()
3170 txq_pcpu->tso_headers = in mvpp2_txq_init()
3171 dma_alloc_coherent(port->dev->dev.parent, in mvpp2_txq_init()
3172 txq_pcpu->size * TSO_HEADER_SIZE, in mvpp2_txq_init()
3173 &txq_pcpu->tso_headers_dma, in mvpp2_txq_init()
3175 if (!txq_pcpu->tso_headers) in mvpp2_txq_init()
3176 return -ENOMEM; in mvpp2_txq_init()
3183 static void mvpp2_txq_deinit(struct mvpp2_port *port, in mvpp2_txq_deinit() argument
3189 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_txq_deinit()
3190 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_txq_deinit()
3191 kfree(txq_pcpu->buffs); in mvpp2_txq_deinit()
3193 if (txq_pcpu->tso_headers) in mvpp2_txq_deinit()
3194 dma_free_coherent(port->dev->dev.parent, in mvpp2_txq_deinit()
3195 txq_pcpu->size * TSO_HEADER_SIZE, in mvpp2_txq_deinit()
3196 txq_pcpu->tso_headers, in mvpp2_txq_deinit()
3197 txq_pcpu->tso_headers_dma); in mvpp2_txq_deinit()
3199 txq_pcpu->tso_headers = NULL; in mvpp2_txq_deinit()
3202 if (txq->descs) in mvpp2_txq_deinit()
3203 dma_free_coherent(port->dev->dev.parent, in mvpp2_txq_deinit()
3204 txq->size * MVPP2_DESC_ALIGNED_SIZE, in mvpp2_txq_deinit()
3205 txq->descs, txq->descs_dma); in mvpp2_txq_deinit()
3207 txq->descs = NULL; in mvpp2_txq_deinit()
3208 txq->last_desc = 0; in mvpp2_txq_deinit()
3209 txq->next_desc_to_proc = 0; in mvpp2_txq_deinit()
3210 txq->descs_dma = 0; in mvpp2_txq_deinit()
3213 mvpp2_write(port->priv, MVPP2_TXQ_SCHED_TOKEN_CNTR_REG(txq->log_id), 0); in mvpp2_txq_deinit()
3216 thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_txq_deinit()
3217 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id); in mvpp2_txq_deinit()
3218 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_ADDR_REG, 0); in mvpp2_txq_deinit()
3219 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_DESC_SIZE_REG, 0); in mvpp2_txq_deinit()
3224 static void mvpp2_txq_clean(struct mvpp2_port *port, struct mvpp2_tx_queue *txq) in mvpp2_txq_clean() argument
3228 unsigned int thread = mvpp2_cpu_to_thread(port->priv, get_cpu()); in mvpp2_txq_clean()
3231 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_NUM_REG, txq->id); in mvpp2_txq_clean()
3232 val = mvpp2_thread_read(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG); in mvpp2_txq_clean()
3234 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, val); in mvpp2_txq_clean()
3242 netdev_warn(port->dev, in mvpp2_txq_clean()
3243 "port %d: cleaning queue %d timed out\n", in mvpp2_txq_clean()
3244 port->id, txq->log_id); in mvpp2_txq_clean()
3250 pending = mvpp2_thread_read(port->priv, thread, in mvpp2_txq_clean()
3256 mvpp2_thread_write(port->priv, thread, MVPP2_TXQ_PREF_BUF_REG, val); in mvpp2_txq_clean()
3259 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_txq_clean()
3260 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_txq_clean()
3263 mvpp2_txq_bufs_free(port, txq, txq_pcpu, txq_pcpu->count); in mvpp2_txq_clean()
3266 txq_pcpu->count = 0; in mvpp2_txq_clean()
3267 txq_pcpu->txq_put_index = 0; in mvpp2_txq_clean()
3268 txq_pcpu->txq_get_index = 0; in mvpp2_txq_clean()
3273 static void mvpp2_cleanup_txqs(struct mvpp2_port *port) in mvpp2_cleanup_txqs() argument
3279 val = mvpp2_read(port->priv, MVPP2_TX_PORT_FLUSH_REG); in mvpp2_cleanup_txqs()
3282 val |= MVPP2_TX_PORT_FLUSH_MASK(port->id); in mvpp2_cleanup_txqs()
3283 mvpp2_write(port->priv, MVPP2_TX_PORT_FLUSH_REG, val); in mvpp2_cleanup_txqs()
3285 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_cleanup_txqs()
3286 txq = port->txqs[queue]; in mvpp2_cleanup_txqs()
3287 mvpp2_txq_clean(port, txq); in mvpp2_cleanup_txqs()
3288 mvpp2_txq_deinit(port, txq); in mvpp2_cleanup_txqs()
3291 on_each_cpu(mvpp2_txq_sent_counter_clear, port, 1); in mvpp2_cleanup_txqs()
3293 val &= ~MVPP2_TX_PORT_FLUSH_MASK(port->id); in mvpp2_cleanup_txqs()
3294 mvpp2_write(port->priv, MVPP2_TX_PORT_FLUSH_REG, val); in mvpp2_cleanup_txqs()
3298 static void mvpp2_cleanup_rxqs(struct mvpp2_port *port) in mvpp2_cleanup_rxqs() argument
3302 for (queue = 0; queue < port->nrxqs; queue++) in mvpp2_cleanup_rxqs()
3303 mvpp2_rxq_deinit(port, port->rxqs[queue]); in mvpp2_cleanup_rxqs()
3305 if (port->tx_fc) in mvpp2_cleanup_rxqs()
3306 mvpp2_rxq_disable_fc(port); in mvpp2_cleanup_rxqs()
3309 /* Init all Rx queues for port */
3310 static int mvpp2_setup_rxqs(struct mvpp2_port *port) in mvpp2_setup_rxqs() argument
3314 for (queue = 0; queue < port->nrxqs; queue++) { in mvpp2_setup_rxqs()
3315 err = mvpp2_rxq_init(port, port->rxqs[queue]); in mvpp2_setup_rxqs()
3320 if (port->tx_fc) in mvpp2_setup_rxqs()
3321 mvpp2_rxq_enable_fc(port); in mvpp2_setup_rxqs()
3326 mvpp2_cleanup_rxqs(port); in mvpp2_setup_rxqs()
3330 /* Init all tx queues for port */
3331 static int mvpp2_setup_txqs(struct mvpp2_port *port) in mvpp2_setup_txqs() argument
3336 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_setup_txqs()
3337 txq = port->txqs[queue]; in mvpp2_setup_txqs()
3338 err = mvpp2_txq_init(port, txq); in mvpp2_setup_txqs()
3344 netif_set_xps_queue(port->dev, cpumask_of(queue), queue); in mvpp2_setup_txqs()
3347 if (port->has_tx_irqs) { in mvpp2_setup_txqs()
3348 mvpp2_tx_time_coal_set(port); in mvpp2_setup_txqs()
3349 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_setup_txqs()
3350 txq = port->txqs[queue]; in mvpp2_setup_txqs()
3351 mvpp2_tx_pkts_coal_set(port, txq); in mvpp2_setup_txqs()
3355 on_each_cpu(mvpp2_txq_sent_counter_clear, port, 1); in mvpp2_setup_txqs()
3359 mvpp2_cleanup_txqs(port); in mvpp2_setup_txqs()
3363 /* The callback for per-port interrupt */
3370 napi_schedule(&qv->napi); in mvpp2_isr()
3375 static void mvpp2_isr_handle_ptp_queue(struct mvpp2_port *port, int nq) in mvpp2_isr_handle_ptp_queue() argument
3384 ptp_q = port->priv->iface_base + MVPP22_PTP_BASE(port->gop_id); in mvpp2_isr_handle_ptp_queue()
3386 ptp_q += MVPP22_PTP_TX_Q1_R0 - MVPP22_PTP_TX_Q0_R0; in mvpp2_isr_handle_ptp_queue()
3388 queue = &port->tx_hwtstamp_queue[nq]; in mvpp2_isr_handle_ptp_queue()
3400 skb = queue->skb[id]; in mvpp2_isr_handle_ptp_queue()
3401 queue->skb[id] = NULL; in mvpp2_isr_handle_ptp_queue()
3405 mvpp22_tai_tstamp(port->priv->tai, ts, &shhwtstamps); in mvpp2_isr_handle_ptp_queue()
3412 static void mvpp2_isr_handle_ptp(struct mvpp2_port *port) in mvpp2_isr_handle_ptp() argument
3417 ptp = port->priv->iface_base + MVPP22_PTP_BASE(port->gop_id); in mvpp2_isr_handle_ptp()
3420 mvpp2_isr_handle_ptp_queue(port, 0); in mvpp2_isr_handle_ptp()
3422 mvpp2_isr_handle_ptp_queue(port, 1); in mvpp2_isr_handle_ptp()
3425 static void mvpp2_isr_handle_link(struct mvpp2_port *port, bool link) in mvpp2_isr_handle_link() argument
3427 struct net_device *dev = port->dev; in mvpp2_isr_handle_link()
3429 if (port->phylink) { in mvpp2_isr_handle_link()
3430 phylink_mac_change(port->phylink, link); in mvpp2_isr_handle_link()
3438 mvpp2_interrupts_enable(port); in mvpp2_isr_handle_link()
3440 mvpp2_egress_enable(port); in mvpp2_isr_handle_link()
3441 mvpp2_ingress_enable(port); in mvpp2_isr_handle_link()
3447 mvpp2_ingress_disable(port); in mvpp2_isr_handle_link()
3448 mvpp2_egress_disable(port); in mvpp2_isr_handle_link()
3450 mvpp2_interrupts_disable(port); in mvpp2_isr_handle_link()
3454 static void mvpp2_isr_handle_xlg(struct mvpp2_port *port) in mvpp2_isr_handle_xlg() argument
3459 val = readl(port->base + MVPP22_XLG_INT_STAT); in mvpp2_isr_handle_xlg()
3461 val = readl(port->base + MVPP22_XLG_STATUS); in mvpp2_isr_handle_xlg()
3463 mvpp2_isr_handle_link(port, link); in mvpp2_isr_handle_xlg()
3467 static void mvpp2_isr_handle_gmac_internal(struct mvpp2_port *port) in mvpp2_isr_handle_gmac_internal() argument
3472 if (phy_interface_mode_is_rgmii(port->phy_interface) || in mvpp2_isr_handle_gmac_internal()
3473 phy_interface_mode_is_8023z(port->phy_interface) || in mvpp2_isr_handle_gmac_internal()
3474 port->phy_interface == PHY_INTERFACE_MODE_SGMII) { in mvpp2_isr_handle_gmac_internal()
3475 val = readl(port->base + MVPP22_GMAC_INT_STAT); in mvpp2_isr_handle_gmac_internal()
3477 val = readl(port->base + MVPP2_GMAC_STATUS0); in mvpp2_isr_handle_gmac_internal()
3479 mvpp2_isr_handle_link(port, link); in mvpp2_isr_handle_gmac_internal()
3484 /* Per-port interrupt for link status changes */
3487 struct mvpp2_port *port = (struct mvpp2_port *)dev_id; in mvpp2_port_isr() local
3490 mvpp22_gop_mask_irq(port); in mvpp2_port_isr()
3492 if (mvpp2_port_supports_xlg(port) && in mvpp2_port_isr()
3493 mvpp2_is_xlg(port->phy_interface)) { in mvpp2_port_isr()
3495 val = readl(port->base + MVPP22_XLG_EXT_INT_STAT); in mvpp2_port_isr()
3497 mvpp2_isr_handle_xlg(port); in mvpp2_port_isr()
3499 mvpp2_isr_handle_ptp(port); in mvpp2_port_isr()
3504 val = readl(port->base + MVPP22_GMAC_INT_SUM_STAT); in mvpp2_port_isr()
3506 mvpp2_isr_handle_gmac_internal(port); in mvpp2_port_isr()
3508 mvpp2_isr_handle_ptp(port); in mvpp2_port_isr()
3511 mvpp22_gop_unmask_irq(port); in mvpp2_port_isr()
3518 struct mvpp2_port *port; in mvpp2_hr_timer_cb() local
3523 dev = port_pcpu->dev; in mvpp2_hr_timer_cb()
3528 port_pcpu->timer_scheduled = false; in mvpp2_hr_timer_cb()
3529 port = netdev_priv(dev); in mvpp2_hr_timer_cb()
3532 cause = (1 << port->ntxqs) - 1; in mvpp2_hr_timer_cb()
3533 tx_todo = mvpp2_tx_done(port, cause, in mvpp2_hr_timer_cb()
3534 mvpp2_cpu_to_thread(port->priv, smp_processor_id())); in mvpp2_hr_timer_cb()
3537 if (tx_todo && !port_pcpu->timer_scheduled) { in mvpp2_hr_timer_cb()
3538 port_pcpu->timer_scheduled = true; in mvpp2_hr_timer_cb()
3539 hrtimer_forward_now(&port_pcpu->tx_done_timer, in mvpp2_hr_timer_cb()
3550 static void mvpp2_rx_error(struct mvpp2_port *port, in mvpp2_rx_error() argument
3553 u32 status = mvpp2_rxdesc_status_get(port, rx_desc); in mvpp2_rx_error()
3554 size_t sz = mvpp2_rxdesc_size_get(port, rx_desc); in mvpp2_rx_error()
3569 netdev_err(port->dev, in mvpp2_rx_error()
3575 static int mvpp2_rx_csum(struct mvpp2_port *port, u32 status) in mvpp2_rx_csum() argument
3589 static int mvpp2_rx_refill(struct mvpp2_port *port, in mvpp2_rx_refill() argument
3597 buf = mvpp2_buf_alloc(port, bm_pool, page_pool, in mvpp2_rx_refill()
3600 return -ENOMEM; in mvpp2_rx_refill()
3602 mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr); in mvpp2_rx_refill()
3608 static u32 mvpp2_skb_tx_csum(struct mvpp2_port *port, struct sk_buff *skb) in mvpp2_skb_tx_csum() argument
3610 if (skb->ip_summed == CHECKSUM_PARTIAL) { in mvpp2_skb_tx_csum()
3619 ip_hdr_len = ip4h->ihl; in mvpp2_skb_tx_csum()
3620 l4_proto = ip4h->protocol; in mvpp2_skb_tx_csum()
3627 l4_proto = ip6h->nexthdr; in mvpp2_skb_tx_csum()
3639 static void mvpp2_xdp_finish_tx(struct mvpp2_port *port, u16 txq_id, int nxmit, int nxmit_byte) in mvpp2_xdp_finish_tx() argument
3641 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_xdp_finish_tx()
3647 txq = port->txqs[txq_id]; in mvpp2_xdp_finish_tx()
3648 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_xdp_finish_tx()
3649 nq = netdev_get_tx_queue(port->dev, txq_id); in mvpp2_xdp_finish_tx()
3650 aggr_txq = &port->priv->aggr_txqs[thread]; in mvpp2_xdp_finish_tx()
3652 txq_pcpu->reserved_num -= nxmit; in mvpp2_xdp_finish_tx()
3653 txq_pcpu->count += nxmit; in mvpp2_xdp_finish_tx()
3654 aggr_txq->count += nxmit; in mvpp2_xdp_finish_tx()
3658 mvpp2_aggr_txq_pend_desc_add(port, nxmit); in mvpp2_xdp_finish_tx()
3660 if (txq_pcpu->count >= txq_pcpu->stop_threshold) in mvpp2_xdp_finish_tx()
3664 if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal) in mvpp2_xdp_finish_tx()
3665 mvpp2_txq_done(port, txq, txq_pcpu); in mvpp2_xdp_finish_tx()
3669 mvpp2_xdp_submit_frame(struct mvpp2_port *port, u16 txq_id, in mvpp2_xdp_submit_frame() argument
3672 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_xdp_submit_frame()
3683 txq = port->txqs[txq_id]; in mvpp2_xdp_submit_frame()
3684 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_xdp_submit_frame()
3685 aggr_txq = &port->priv->aggr_txqs[thread]; in mvpp2_xdp_submit_frame()
3688 if (mvpp2_aggr_desc_num_check(port, aggr_txq, 1) || in mvpp2_xdp_submit_frame()
3689 mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, 1)) { in mvpp2_xdp_submit_frame()
3696 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_xdp_submit_frame()
3697 mvpp2_txdesc_size_set(port, tx_desc, xdpf->len); in mvpp2_xdp_submit_frame()
3701 dma_addr = dma_map_single(port->dev->dev.parent, xdpf->data, in mvpp2_xdp_submit_frame()
3702 xdpf->len, DMA_TO_DEVICE); in mvpp2_xdp_submit_frame()
3704 if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) { in mvpp2_xdp_submit_frame()
3713 struct page *page = virt_to_page(xdpf->data); in mvpp2_xdp_submit_frame()
3716 sizeof(*xdpf) + xdpf->headroom; in mvpp2_xdp_submit_frame()
3717 dma_sync_single_for_device(port->dev->dev.parent, dma_addr, in mvpp2_xdp_submit_frame()
3718 xdpf->len, DMA_BIDIRECTIONAL); in mvpp2_xdp_submit_frame()
3723 mvpp2_txdesc_dma_addr_set(port, tx_desc, dma_addr); in mvpp2_xdp_submit_frame()
3725 mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd); in mvpp2_xdp_submit_frame()
3726 mvpp2_txq_inc_put(port, txq_pcpu, xdpf, tx_desc, buf_type); in mvpp2_xdp_submit_frame()
3733 mvpp2_xdp_xmit_back(struct mvpp2_port *port, struct xdp_buff *xdp) in mvpp2_xdp_xmit_back() argument
3735 struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats); in mvpp2_xdp_xmit_back()
3747 txq_id = mvpp2_cpu_to_thread(port->priv, smp_processor_id()) + (port->ntxqs / 2); in mvpp2_xdp_xmit_back()
3749 ret = mvpp2_xdp_submit_frame(port, txq_id, xdpf, false); in mvpp2_xdp_xmit_back()
3751 u64_stats_update_begin(&stats->syncp); in mvpp2_xdp_xmit_back()
3752 stats->tx_bytes += xdpf->len; in mvpp2_xdp_xmit_back()
3753 stats->tx_packets++; in mvpp2_xdp_xmit_back()
3754 stats->xdp_tx++; in mvpp2_xdp_xmit_back()
3755 u64_stats_update_end(&stats->syncp); in mvpp2_xdp_xmit_back()
3757 mvpp2_xdp_finish_tx(port, txq_id, 1, xdpf->len); in mvpp2_xdp_xmit_back()
3759 u64_stats_update_begin(&stats->syncp); in mvpp2_xdp_xmit_back()
3760 stats->xdp_tx_err++; in mvpp2_xdp_xmit_back()
3761 u64_stats_update_end(&stats->syncp); in mvpp2_xdp_xmit_back()
3771 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_xdp_xmit() local
3777 if (unlikely(test_bit(0, &port->state))) in mvpp2_xdp_xmit()
3778 return -ENETDOWN; in mvpp2_xdp_xmit()
3781 return -EINVAL; in mvpp2_xdp_xmit()
3786 txq_id = mvpp2_cpu_to_thread(port->priv, smp_processor_id()) + (port->ntxqs / 2); in mvpp2_xdp_xmit()
3789 ret = mvpp2_xdp_submit_frame(port, txq_id, frames[i], true); in mvpp2_xdp_xmit()
3793 nxmit_byte += frames[i]->len; in mvpp2_xdp_xmit()
3798 mvpp2_xdp_finish_tx(port, txq_id, nxmit, nxmit_byte); in mvpp2_xdp_xmit()
3800 stats = this_cpu_ptr(port->stats); in mvpp2_xdp_xmit()
3801 u64_stats_update_begin(&stats->syncp); in mvpp2_xdp_xmit()
3802 stats->tx_bytes += nxmit_byte; in mvpp2_xdp_xmit()
3803 stats->tx_packets += nxmit; in mvpp2_xdp_xmit()
3804 stats->xdp_xmit += nxmit; in mvpp2_xdp_xmit()
3805 stats->xdp_xmit_err += num_frame - nxmit; in mvpp2_xdp_xmit()
3806 u64_stats_update_end(&stats->syncp); in mvpp2_xdp_xmit()
3812 mvpp2_run_xdp(struct mvpp2_port *port, struct bpf_prog *prog, in mvpp2_run_xdp() argument
3820 len = xdp->data_end - xdp->data_hard_start - MVPP2_SKB_HEADROOM; in mvpp2_run_xdp()
3824 sync = xdp->data_end - xdp->data_hard_start - MVPP2_SKB_HEADROOM; in mvpp2_run_xdp()
3829 stats->xdp_pass++; in mvpp2_run_xdp()
3833 err = xdp_do_redirect(port->dev, xdp, prog); in mvpp2_run_xdp()
3836 page = virt_to_head_page(xdp->data); in mvpp2_run_xdp()
3840 stats->xdp_redirect++; in mvpp2_run_xdp()
3844 ret = mvpp2_xdp_xmit_back(port, xdp); in mvpp2_run_xdp()
3846 page = virt_to_head_page(xdp->data); in mvpp2_run_xdp()
3851 bpf_warn_invalid_xdp_action(port->dev, prog, act); in mvpp2_run_xdp()
3854 trace_xdp_exception(port->dev, prog, act); in mvpp2_run_xdp()
3857 page = virt_to_head_page(xdp->data); in mvpp2_run_xdp()
3860 stats->xdp_drop++; in mvpp2_run_xdp()
3867 static void mvpp2_buff_hdr_pool_put(struct mvpp2_port *port, struct mvpp2_rx_desc *rx_desc, in mvpp2_buff_hdr_pool_put() argument
3874 phys_addr = mvpp2_rxdesc_dma_addr_get(port, rx_desc); in mvpp2_buff_hdr_pool_put()
3875 dma_addr = mvpp2_rxdesc_cookie_get(port, rx_desc); in mvpp2_buff_hdr_pool_put()
3880 phys_addr_next = le32_to_cpu(buff_hdr->next_phys_addr); in mvpp2_buff_hdr_pool_put()
3881 dma_addr_next = le32_to_cpu(buff_hdr->next_dma_addr); in mvpp2_buff_hdr_pool_put()
3883 if (port->priv->hw_version >= MVPP22) { in mvpp2_buff_hdr_pool_put()
3884 phys_addr_next |= ((u64)buff_hdr->next_phys_addr_high << 32); in mvpp2_buff_hdr_pool_put()
3885 dma_addr_next |= ((u64)buff_hdr->next_dma_addr_high << 32); in mvpp2_buff_hdr_pool_put()
3888 mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr); in mvpp2_buff_hdr_pool_put()
3893 } while (!MVPP2_B_HDR_INFO_IS_LAST(le16_to_cpu(buff_hdr->info))); in mvpp2_buff_hdr_pool_put()
3897 static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi, in mvpp2_rx() argument
3900 struct net_device *dev = port->dev; in mvpp2_rx()
3909 xdp_prog = READ_ONCE(port->xdp_prog); in mvpp2_rx()
3911 /* Get number of received packets and clamp the to-do */ in mvpp2_rx()
3912 rx_received = mvpp2_rxq_received(port, rxq->id); in mvpp2_rx()
3929 phys_addr = mvpp2_rxdesc_cookie_get(port, rx_desc); in mvpp2_rx()
3935 rx_status = mvpp2_rxdesc_status_get(port, rx_desc); in mvpp2_rx()
3936 rx_bytes = mvpp2_rxdesc_size_get(port, rx_desc); in mvpp2_rx()
3937 rx_bytes -= MVPP2_MH_SIZE; in mvpp2_rx()
3938 dma_addr = mvpp2_rxdesc_dma_addr_get(port, rx_desc); in mvpp2_rx()
3942 bm_pool = &port->priv->bm_pools[pool]; in mvpp2_rx()
3944 if (port->priv->percpu_pools) { in mvpp2_rx()
3945 pp = port->priv->page_pool[pool]; in mvpp2_rx()
3951 dma_sync_single_for_cpu(dev->dev.parent, dma_addr, in mvpp2_rx()
3970 if (bm_pool->frag_size > PAGE_SIZE) in mvpp2_rx()
3973 frag_size = bm_pool->frag_size; in mvpp2_rx()
3978 if (bm_pool->pkt_size == MVPP2_BM_SHORT_PKT_SIZE) in mvpp2_rx()
3979 xdp_rxq = &rxq->xdp_rxq_short; in mvpp2_rx()
3981 xdp_rxq = &rxq->xdp_rxq_long; in mvpp2_rx()
3988 ret = mvpp2_run_xdp(port, xdp_prog, &xdp, pp, &ps); in mvpp2_rx()
3992 err = mvpp2_rx_refill(port, bm_pool, pp, pool); in mvpp2_rx()
3994 netdev_err(port->dev, "failed to refill BM pools\n"); in mvpp2_rx()
4009 netdev_warn(port->dev, "skb build failed\n"); in mvpp2_rx()
4016 if (mvpp22_rx_hwtstamping(port)) { in mvpp2_rx()
4017 timestamp = le32_to_cpu(rx_desc->pp22.timestamp); in mvpp2_rx()
4018 mvpp22_tai_tstamp(port->priv->tai, timestamp, in mvpp2_rx()
4022 err = mvpp2_rx_refill(port, bm_pool, pp, pool); in mvpp2_rx()
4024 netdev_err(port->dev, "failed to refill BM pools\n"); in mvpp2_rx()
4032 dma_unmap_single_attrs(dev->dev.parent, dma_addr, in mvpp2_rx()
4033 bm_pool->buf_size, DMA_FROM_DEVICE, in mvpp2_rx()
4041 skb->ip_summed = mvpp2_rx_csum(port, rx_status); in mvpp2_rx()
4042 skb->protocol = eth_type_trans(skb, dev); in mvpp2_rx()
4048 dev->stats.rx_errors++; in mvpp2_rx()
4049 mvpp2_rx_error(port, rx_desc); in mvpp2_rx()
4052 mvpp2_buff_hdr_pool_put(port, rx_desc, pool, rx_status); in mvpp2_rx()
4054 mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr); in mvpp2_rx()
4061 struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats); in mvpp2_rx()
4063 u64_stats_update_begin(&stats->syncp); in mvpp2_rx()
4064 stats->rx_packets += ps.rx_packets; in mvpp2_rx()
4065 stats->rx_bytes += ps.rx_bytes; in mvpp2_rx()
4067 stats->xdp_redirect += ps.xdp_redirect; in mvpp2_rx()
4068 stats->xdp_pass += ps.xdp_pass; in mvpp2_rx()
4069 stats->xdp_drop += ps.xdp_drop; in mvpp2_rx()
4070 u64_stats_update_end(&stats->syncp); in mvpp2_rx()
4075 mvpp2_rxq_status_update(port, rxq->id, rx_done, rx_done); in mvpp2_rx()
4081 tx_desc_unmap_put(struct mvpp2_port *port, struct mvpp2_tx_queue *txq, in tx_desc_unmap_put() argument
4084 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in tx_desc_unmap_put()
4085 struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in tx_desc_unmap_put()
4088 mvpp2_txdesc_dma_addr_get(port, desc); in tx_desc_unmap_put()
4090 mvpp2_txdesc_size_get(port, desc); in tx_desc_unmap_put()
4092 dma_unmap_single(port->dev->dev.parent, buf_dma_addr, in tx_desc_unmap_put()
4097 static void mvpp2_txdesc_clear_ptp(struct mvpp2_port *port, in mvpp2_txdesc_clear_ptp() argument
4101 if (port->priv->hw_version >= MVPP22) in mvpp2_txdesc_clear_ptp()
4102 desc->pp22.ptp_descriptor &= in mvpp2_txdesc_clear_ptp()
4106 static bool mvpp2_tx_hw_tstamp(struct mvpp2_port *port, in mvpp2_tx_hw_tstamp() argument
4115 if (port->priv->hw_version == MVPP21 || in mvpp2_tx_hw_tstamp()
4116 port->tx_hwtstamp_type == HWTSTAMP_TX_OFF) in mvpp2_tx_hw_tstamp()
4127 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; in mvpp2_tx_hw_tstamp()
4131 queue = &port->tx_hwtstamp_queue[0]; in mvpp2_tx_hw_tstamp()
4140 mtype = hdr->tsmt & 15; in mvpp2_tx_hw_tstamp()
4144 queue = &port->tx_hwtstamp_queue[1]; in mvpp2_tx_hw_tstamp()
4150 i = queue->next; in mvpp2_tx_hw_tstamp()
4151 queue->next = (i + 1) & 31; in mvpp2_tx_hw_tstamp()
4152 if (queue->skb[i]) in mvpp2_tx_hw_tstamp()
4153 dev_kfree_skb_any(queue->skb[i]); in mvpp2_tx_hw_tstamp()
4154 queue->skb[i] = skb_get(skb); in mvpp2_tx_hw_tstamp()
4159 * 3:0 - PTPAction in mvpp2_tx_hw_tstamp()
4160 * 6:4 - PTPPacketFormat in mvpp2_tx_hw_tstamp()
4161 * 7 - PTP_CF_WraparoundCheckEn in mvpp2_tx_hw_tstamp()
4162 * 9:8 - IngressTimestampSeconds[1:0] in mvpp2_tx_hw_tstamp()
4163 * 10 - Reserved in mvpp2_tx_hw_tstamp()
4164 * 11 - MACTimestampingEn in mvpp2_tx_hw_tstamp()
4165 * 17:12 - PTP_TimestampQueueEntryID[5:0] in mvpp2_tx_hw_tstamp()
4166 * 18 - PTPTimestampQueueSelect in mvpp2_tx_hw_tstamp()
4167 * 19 - UDPChecksumUpdateEn in mvpp2_tx_hw_tstamp()
4168 * 27:20 - TimestampOffset in mvpp2_tx_hw_tstamp()
4169 * PTP, NTPTransmit, OWAMP/TWAMP - L3 to PTP header in mvpp2_tx_hw_tstamp()
4170 * NTPTs, Y.1731 - L3 to timestamp entry in mvpp2_tx_hw_tstamp()
4171 * 35:28 - UDP Checksum Offset in mvpp2_tx_hw_tstamp()
4175 tx_desc->pp22.ptp_descriptor &= in mvpp2_tx_hw_tstamp()
4177 tx_desc->pp22.ptp_descriptor |= in mvpp2_tx_hw_tstamp()
4179 tx_desc->pp22.buf_dma_addr_ptp &= cpu_to_le64(~0xffffff0000000000ULL); in mvpp2_tx_hw_tstamp()
4180 tx_desc->pp22.buf_dma_addr_ptp |= cpu_to_le64((ptpdesc >> 12) << 40); in mvpp2_tx_hw_tstamp()
4186 static int mvpp2_tx_frag_process(struct mvpp2_port *port, struct sk_buff *skb, in mvpp2_tx_frag_process() argument
4190 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_tx_frag_process()
4191 struct mvpp2_txq_pcpu *txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_tx_frag_process()
4196 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { in mvpp2_tx_frag_process()
4197 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; in mvpp2_tx_frag_process()
4201 mvpp2_txdesc_clear_ptp(port, tx_desc); in mvpp2_tx_frag_process()
4202 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_tx_frag_process()
4203 mvpp2_txdesc_size_set(port, tx_desc, skb_frag_size(frag)); in mvpp2_tx_frag_process()
4205 buf_dma_addr = dma_map_single(port->dev->dev.parent, addr, in mvpp2_tx_frag_process()
4208 if (dma_mapping_error(port->dev->dev.parent, buf_dma_addr)) { in mvpp2_tx_frag_process()
4213 mvpp2_txdesc_dma_addr_set(port, tx_desc, buf_dma_addr); in mvpp2_tx_frag_process()
4215 if (i == (skb_shinfo(skb)->nr_frags - 1)) { in mvpp2_tx_frag_process()
4217 mvpp2_txdesc_cmd_set(port, tx_desc, in mvpp2_tx_frag_process()
4219 mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB); in mvpp2_tx_frag_process()
4222 mvpp2_txdesc_cmd_set(port, tx_desc, 0); in mvpp2_tx_frag_process()
4223 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB); in mvpp2_tx_frag_process()
4232 for (i = i - 1; i >= 0; i--) { in mvpp2_tx_frag_process()
4233 tx_desc = txq->descs + i; in mvpp2_tx_frag_process()
4234 tx_desc_unmap_put(port, txq, tx_desc); in mvpp2_tx_frag_process()
4237 return -ENOMEM; in mvpp2_tx_frag_process()
4247 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_tso_put_hdr() local
4251 mvpp2_txdesc_clear_ptp(port, tx_desc); in mvpp2_tso_put_hdr()
4252 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_tso_put_hdr()
4253 mvpp2_txdesc_size_set(port, tx_desc, hdr_sz); in mvpp2_tso_put_hdr()
4255 addr = txq_pcpu->tso_headers_dma + in mvpp2_tso_put_hdr()
4256 txq_pcpu->txq_put_index * TSO_HEADER_SIZE; in mvpp2_tso_put_hdr()
4257 mvpp2_txdesc_dma_addr_set(port, tx_desc, addr); in mvpp2_tso_put_hdr()
4259 mvpp2_txdesc_cmd_set(port, tx_desc, mvpp2_skb_tx_csum(port, skb) | in mvpp2_tso_put_hdr()
4262 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB); in mvpp2_tso_put_hdr()
4272 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_tso_put_data() local
4276 mvpp2_txdesc_clear_ptp(port, tx_desc); in mvpp2_tso_put_data()
4277 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_tso_put_data()
4278 mvpp2_txdesc_size_set(port, tx_desc, sz); in mvpp2_tso_put_data()
4280 buf_dma_addr = dma_map_single(dev->dev.parent, tso->data, sz, in mvpp2_tso_put_data()
4282 if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) { in mvpp2_tso_put_data()
4284 return -ENOMEM; in mvpp2_tso_put_data()
4287 mvpp2_txdesc_dma_addr_set(port, tx_desc, buf_dma_addr); in mvpp2_tso_put_data()
4290 mvpp2_txdesc_cmd_set(port, tx_desc, MVPP2_TXD_L_DESC); in mvpp2_tso_put_data()
4292 mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB); in mvpp2_tso_put_data()
4296 mvpp2_txdesc_cmd_set(port, tx_desc, 0); in mvpp2_tso_put_data()
4299 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB); in mvpp2_tso_put_data()
4308 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_tx_tso() local
4313 if (mvpp2_aggr_desc_num_check(port, aggr_txq, tso_count_descs(skb)) || in mvpp2_tx_tso()
4314 mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, in mvpp2_tx_tso()
4320 len = skb->len - hdr_sz; in mvpp2_tx_tso()
4322 int left = min_t(int, skb_shinfo(skb)->gso_size, len); in mvpp2_tx_tso()
4323 char *hdr = txq_pcpu->tso_headers + in mvpp2_tx_tso()
4324 txq_pcpu->txq_put_index * TSO_HEADER_SIZE; in mvpp2_tx_tso()
4326 len -= left; in mvpp2_tx_tso()
4334 left -= sz; in mvpp2_tx_tso()
4347 for (i = descs - 1; i >= 0; i--) { in mvpp2_tx_tso()
4348 struct mvpp2_tx_desc *tx_desc = txq->descs + i; in mvpp2_tx_tso()
4349 tx_desc_unmap_put(port, txq, tx_desc); in mvpp2_tx_tso()
4357 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_tx() local
4368 thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_tx()
4371 txq = port->txqs[txq_id]; in mvpp2_tx()
4372 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_tx()
4373 aggr_txq = &port->priv->aggr_txqs[thread]; in mvpp2_tx()
4375 if (test_bit(thread, &port->priv->lock_map)) in mvpp2_tx()
4376 spin_lock_irqsave(&port->tx_lock[thread], flags); in mvpp2_tx()
4382 frags = skb_shinfo(skb)->nr_frags + 1; in mvpp2_tx()
4385 if (mvpp2_aggr_desc_num_check(port, aggr_txq, frags) || in mvpp2_tx()
4386 mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, frags)) { in mvpp2_tx()
4393 if (!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) || in mvpp2_tx()
4394 !mvpp2_tx_hw_tstamp(port, tx_desc, skb)) in mvpp2_tx()
4395 mvpp2_txdesc_clear_ptp(port, tx_desc); in mvpp2_tx()
4396 mvpp2_txdesc_txq_set(port, tx_desc, txq->id); in mvpp2_tx()
4397 mvpp2_txdesc_size_set(port, tx_desc, skb_headlen(skb)); in mvpp2_tx()
4399 buf_dma_addr = dma_map_single(dev->dev.parent, skb->data, in mvpp2_tx()
4401 if (unlikely(dma_mapping_error(dev->dev.parent, buf_dma_addr))) { in mvpp2_tx()
4407 mvpp2_txdesc_dma_addr_set(port, tx_desc, buf_dma_addr); in mvpp2_tx()
4409 tx_cmd = mvpp2_skb_tx_csum(port, skb); in mvpp2_tx()
4414 mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd); in mvpp2_tx()
4415 mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB); in mvpp2_tx()
4419 mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd); in mvpp2_tx()
4420 mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB); in mvpp2_tx()
4423 if (mvpp2_tx_frag_process(port, skb, aggr_txq, txq)) { in mvpp2_tx()
4424 tx_desc_unmap_put(port, txq, tx_desc); in mvpp2_tx()
4431 struct mvpp2_pcpu_stats *stats = per_cpu_ptr(port->stats, thread); in mvpp2_tx()
4434 txq_pcpu->reserved_num -= frags; in mvpp2_tx()
4435 txq_pcpu->count += frags; in mvpp2_tx()
4436 aggr_txq->count += frags; in mvpp2_tx()
4440 mvpp2_aggr_txq_pend_desc_add(port, frags); in mvpp2_tx()
4442 if (txq_pcpu->count >= txq_pcpu->stop_threshold) in mvpp2_tx()
4445 u64_stats_update_begin(&stats->syncp); in mvpp2_tx()
4446 stats->tx_packets++; in mvpp2_tx()
4447 stats->tx_bytes += skb->len; in mvpp2_tx()
4448 u64_stats_update_end(&stats->syncp); in mvpp2_tx()
4450 dev->stats.tx_dropped++; in mvpp2_tx()
4455 if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal) in mvpp2_tx()
4456 mvpp2_txq_done(port, txq, txq_pcpu); in mvpp2_tx()
4459 if (!port->has_tx_irqs && txq_pcpu->count <= frags && in mvpp2_tx()
4460 txq_pcpu->count > 0) { in mvpp2_tx()
4461 struct mvpp2_port_pcpu *port_pcpu = per_cpu_ptr(port->pcpu, thread); in mvpp2_tx()
4463 if (!port_pcpu->timer_scheduled) { in mvpp2_tx()
4464 port_pcpu->timer_scheduled = true; in mvpp2_tx()
4465 hrtimer_start(&port_pcpu->tx_done_timer, in mvpp2_tx()
4471 if (test_bit(thread, &port->priv->lock_map)) in mvpp2_tx()
4472 spin_unlock_irqrestore(&port->tx_lock[thread], flags); in mvpp2_tx()
4491 struct mvpp2_port *port = netdev_priv(napi->dev); in mvpp2_poll() local
4493 unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id()); in mvpp2_poll()
4499 * Bits 0-15: each bit indicates received packets on the Rx queue in mvpp2_poll()
4502 * Bits 16-23: each bit indicates transmitted packets on the Tx queue in mvpp2_poll()
4507 cause_rx_tx = mvpp2_thread_read_relaxed(port->priv, qv->sw_thread_id, in mvpp2_poll()
4508 MVPP2_ISR_RX_TX_CAUSE_REG(port->id)); in mvpp2_poll()
4512 mvpp2_cause_error(port->dev, cause_misc); in mvpp2_poll()
4515 mvpp2_write(port->priv, MVPP2_ISR_MISC_CAUSE_REG, 0); in mvpp2_poll()
4516 mvpp2_thread_write(port->priv, thread, in mvpp2_poll()
4517 MVPP2_ISR_RX_TX_CAUSE_REG(port->id), in mvpp2_poll()
4521 if (port->has_tx_irqs) { in mvpp2_poll()
4525 mvpp2_tx_done(port, cause_tx, qv->sw_thread_id); in mvpp2_poll()
4531 MVPP2_CAUSE_RXQ_OCCUP_DESC_ALL_MASK(port->priv->hw_version); in mvpp2_poll()
4532 cause_rx <<= qv->first_rxq; in mvpp2_poll()
4533 cause_rx |= qv->pending_cause_rx; in mvpp2_poll()
4538 rxq = mvpp2_get_rx_queue(port, cause_rx); in mvpp2_poll()
4542 count = mvpp2_rx(port, napi, budget, rxq); in mvpp2_poll()
4544 budget -= count; in mvpp2_poll()
4550 cause_rx &= ~(1 << rxq->logic_rxq); in mvpp2_poll()
4560 qv->pending_cause_rx = cause_rx; in mvpp2_poll()
4564 static void mvpp22_mode_reconfigure(struct mvpp2_port *port, in mvpp22_mode_reconfigure() argument
4570 mvpp2_mac_reset_assert(port); in mvpp22_mode_reconfigure()
4573 mvpp22_pcs_reset_assert(port); in mvpp22_mode_reconfigure()
4576 mvpp22_comphy_init(port, interface); in mvpp22_mode_reconfigure()
4579 mvpp22_gop_init(port, interface); in mvpp22_mode_reconfigure()
4581 mvpp22_pcs_reset_deassert(port, interface); in mvpp22_mode_reconfigure()
4583 if (mvpp2_port_supports_xlg(port)) { in mvpp22_mode_reconfigure()
4584 ctrl3 = readl(port->base + MVPP22_XLG_CTRL3_REG); in mvpp22_mode_reconfigure()
4592 writel(ctrl3, port->base + MVPP22_XLG_CTRL3_REG); in mvpp22_mode_reconfigure()
4595 if (mvpp2_port_supports_xlg(port) && mvpp2_is_xlg(interface)) in mvpp22_mode_reconfigure()
4596 mvpp2_xlg_max_rx_size_set(port); in mvpp22_mode_reconfigure()
4598 mvpp2_gmac_max_rx_size_set(port); in mvpp22_mode_reconfigure()
4601 /* Set hw internals when starting port */
4602 static void mvpp2_start_dev(struct mvpp2_port *port) in mvpp2_start_dev() argument
4606 mvpp2_txp_max_tx_size_set(port); in mvpp2_start_dev()
4608 for (i = 0; i < port->nqvecs; i++) in mvpp2_start_dev()
4609 napi_enable(&port->qvecs[i].napi); in mvpp2_start_dev()
4612 mvpp2_interrupts_enable(port); in mvpp2_start_dev()
4614 if (port->priv->hw_version >= MVPP22) in mvpp2_start_dev()
4615 mvpp22_mode_reconfigure(port, port->phy_interface); in mvpp2_start_dev()
4617 if (port->phylink) { in mvpp2_start_dev()
4618 phylink_start(port->phylink); in mvpp2_start_dev()
4620 mvpp2_acpi_start(port); in mvpp2_start_dev()
4623 netif_tx_start_all_queues(port->dev); in mvpp2_start_dev()
4625 clear_bit(0, &port->state); in mvpp2_start_dev()
4628 /* Set hw internals when stopping port */
4629 static void mvpp2_stop_dev(struct mvpp2_port *port) in mvpp2_stop_dev() argument
4633 set_bit(0, &port->state); in mvpp2_stop_dev()
4636 mvpp2_interrupts_disable(port); in mvpp2_stop_dev()
4638 for (i = 0; i < port->nqvecs; i++) in mvpp2_stop_dev()
4639 napi_disable(&port->qvecs[i].napi); in mvpp2_stop_dev()
4641 if (port->phylink) in mvpp2_stop_dev()
4642 phylink_stop(port->phylink); in mvpp2_stop_dev()
4643 phy_power_off(port->comphy); in mvpp2_stop_dev()
4649 u16 new_rx_pending = ring->rx_pending; in mvpp2_check_ringparam_valid()
4650 u16 new_tx_pending = ring->tx_pending; in mvpp2_check_ringparam_valid()
4652 if (ring->rx_pending == 0 || ring->tx_pending == 0) in mvpp2_check_ringparam_valid()
4653 return -EINVAL; in mvpp2_check_ringparam_valid()
4655 if (ring->rx_pending > MVPP2_MAX_RXD_MAX) in mvpp2_check_ringparam_valid()
4657 else if (ring->rx_pending < MSS_THRESHOLD_START) in mvpp2_check_ringparam_valid()
4659 else if (!IS_ALIGNED(ring->rx_pending, 16)) in mvpp2_check_ringparam_valid()
4660 new_rx_pending = ALIGN(ring->rx_pending, 16); in mvpp2_check_ringparam_valid()
4662 if (ring->tx_pending > MVPP2_MAX_TXD_MAX) in mvpp2_check_ringparam_valid()
4664 else if (!IS_ALIGNED(ring->tx_pending, 32)) in mvpp2_check_ringparam_valid()
4665 new_tx_pending = ALIGN(ring->tx_pending, 32); in mvpp2_check_ringparam_valid()
4673 if (ring->rx_pending != new_rx_pending) { in mvpp2_check_ringparam_valid()
4675 ring->rx_pending, new_rx_pending); in mvpp2_check_ringparam_valid()
4676 ring->rx_pending = new_rx_pending; in mvpp2_check_ringparam_valid()
4679 if (ring->tx_pending != new_tx_pending) { in mvpp2_check_ringparam_valid()
4681 ring->tx_pending, new_tx_pending); in mvpp2_check_ringparam_valid()
4682 ring->tx_pending = new_tx_pending; in mvpp2_check_ringparam_valid()
4688 static void mvpp21_get_mac_address(struct mvpp2_port *port, unsigned char *addr) in mvpp21_get_mac_address() argument
4692 mac_addr_l = readl(port->base + MVPP2_GMAC_CTRL_1_REG); in mvpp21_get_mac_address()
4693 mac_addr_m = readl(port->priv->lms_base + MVPP2_SRC_ADDR_MIDDLE); in mvpp21_get_mac_address()
4694 mac_addr_h = readl(port->priv->lms_base + MVPP2_SRC_ADDR_HIGH); in mvpp21_get_mac_address()
4703 static int mvpp2_irqs_init(struct mvpp2_port *port) in mvpp2_irqs_init() argument
4707 for (i = 0; i < port->nqvecs; i++) { in mvpp2_irqs_init()
4708 struct mvpp2_queue_vector *qv = port->qvecs + i; in mvpp2_irqs_init()
4710 if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE) { in mvpp2_irqs_init()
4711 qv->mask = kzalloc(cpumask_size(), GFP_KERNEL); in mvpp2_irqs_init()
4712 if (!qv->mask) { in mvpp2_irqs_init()
4713 err = -ENOMEM; in mvpp2_irqs_init()
4717 irq_set_status_flags(qv->irq, IRQ_NO_BALANCING); in mvpp2_irqs_init()
4720 err = request_irq(qv->irq, mvpp2_isr, 0, port->dev->name, qv); in mvpp2_irqs_init()
4724 if (qv->type == MVPP2_QUEUE_VECTOR_PRIVATE) { in mvpp2_irqs_init()
4728 if (mvpp2_cpu_to_thread(port->priv, cpu) == in mvpp2_irqs_init()
4729 qv->sw_thread_id) in mvpp2_irqs_init()
4730 cpumask_set_cpu(cpu, qv->mask); in mvpp2_irqs_init()
4733 irq_set_affinity_hint(qv->irq, qv->mask); in mvpp2_irqs_init()
4739 for (i = 0; i < port->nqvecs; i++) { in mvpp2_irqs_init()
4740 struct mvpp2_queue_vector *qv = port->qvecs + i; in mvpp2_irqs_init()
4742 irq_set_affinity_hint(qv->irq, NULL); in mvpp2_irqs_init()
4743 kfree(qv->mask); in mvpp2_irqs_init()
4744 qv->mask = NULL; in mvpp2_irqs_init()
4745 free_irq(qv->irq, qv); in mvpp2_irqs_init()
4751 static void mvpp2_irqs_deinit(struct mvpp2_port *port) in mvpp2_irqs_deinit() argument
4755 for (i = 0; i < port->nqvecs; i++) { in mvpp2_irqs_deinit()
4756 struct mvpp2_queue_vector *qv = port->qvecs + i; in mvpp2_irqs_deinit()
4758 irq_set_affinity_hint(qv->irq, NULL); in mvpp2_irqs_deinit()
4759 kfree(qv->mask); in mvpp2_irqs_deinit()
4760 qv->mask = NULL; in mvpp2_irqs_deinit()
4761 irq_clear_status_flags(qv->irq, IRQ_NO_BALANCING); in mvpp2_irqs_deinit()
4762 free_irq(qv->irq, qv); in mvpp2_irqs_deinit()
4766 static bool mvpp22_rss_is_supported(struct mvpp2_port *port) in mvpp22_rss_is_supported() argument
4769 !(port->flags & MVPP2_F_LOOPBACK); in mvpp22_rss_is_supported()
4774 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_open() local
4775 struct mvpp2 *priv = port->priv; in mvpp2_open()
4781 err = mvpp2_prs_mac_da_accept(port, mac_bcast, true); in mvpp2_open()
4786 err = mvpp2_prs_mac_da_accept(port, dev->dev_addr, true); in mvpp2_open()
4791 err = mvpp2_prs_tag_mode_set(port->priv, port->id, MVPP2_TAG_TYPE_MH); in mvpp2_open()
4796 err = mvpp2_prs_def_flow(port); in mvpp2_open()
4803 err = mvpp2_setup_rxqs(port); in mvpp2_open()
4805 netdev_err(port->dev, "cannot allocate Rx queues\n"); in mvpp2_open()
4809 err = mvpp2_setup_txqs(port); in mvpp2_open()
4811 netdev_err(port->dev, "cannot allocate Tx queues\n"); in mvpp2_open()
4815 err = mvpp2_irqs_init(port); in mvpp2_open()
4817 netdev_err(port->dev, "cannot init IRQs\n"); in mvpp2_open()
4821 if (port->phylink) { in mvpp2_open()
4822 err = phylink_fwnode_phy_connect(port->phylink, port->fwnode, 0); in mvpp2_open()
4824 netdev_err(port->dev, "could not attach PHY (%d)\n", in mvpp2_open()
4832 if (priv->hw_version >= MVPP22 && port->port_irq) { in mvpp2_open()
4833 err = request_irq(port->port_irq, mvpp2_port_isr, 0, in mvpp2_open()
4834 dev->name, port); in mvpp2_open()
4836 netdev_err(port->dev, in mvpp2_open()
4837 "cannot request port link/ptp IRQ %d\n", in mvpp2_open()
4838 port->port_irq); in mvpp2_open()
4842 mvpp22_gop_setup_irq(port); in mvpp2_open()
4845 netif_carrier_off(port->dev); in mvpp2_open()
4849 port->port_irq = 0; in mvpp2_open()
4853 netdev_err(port->dev, in mvpp2_open()
4855 err = -ENOENT; in mvpp2_open()
4860 on_each_cpu(mvpp2_interrupts_unmask, port, 1); in mvpp2_open()
4861 mvpp2_shared_interrupt_mask_unmask(port, false); in mvpp2_open()
4863 mvpp2_start_dev(port); in mvpp2_open()
4866 queue_delayed_work(priv->stats_queue, &port->stats_work, in mvpp2_open()
4872 mvpp2_irqs_deinit(port); in mvpp2_open()
4874 mvpp2_cleanup_txqs(port); in mvpp2_open()
4876 mvpp2_cleanup_rxqs(port); in mvpp2_open()
4882 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_stop() local
4886 mvpp2_stop_dev(port); in mvpp2_stop()
4889 on_each_cpu(mvpp2_interrupts_mask, port, 1); in mvpp2_stop()
4890 mvpp2_shared_interrupt_mask_unmask(port, true); in mvpp2_stop()
4892 if (port->phylink) in mvpp2_stop()
4893 phylink_disconnect_phy(port->phylink); in mvpp2_stop()
4894 if (port->port_irq) in mvpp2_stop()
4895 free_irq(port->port_irq, port); in mvpp2_stop()
4897 mvpp2_irqs_deinit(port); in mvpp2_stop()
4898 if (!port->has_tx_irqs) { in mvpp2_stop()
4899 for (thread = 0; thread < port->priv->nthreads; thread++) { in mvpp2_stop()
4900 port_pcpu = per_cpu_ptr(port->pcpu, thread); in mvpp2_stop()
4902 hrtimer_cancel(&port_pcpu->tx_done_timer); in mvpp2_stop()
4903 port_pcpu->timer_scheduled = false; in mvpp2_stop()
4906 mvpp2_cleanup_rxqs(port); in mvpp2_stop()
4907 mvpp2_cleanup_txqs(port); in mvpp2_stop()
4909 cancel_delayed_work_sync(&port->stats_work); in mvpp2_stop()
4911 mvpp2_mac_reset_assert(port); in mvpp2_stop()
4912 mvpp22_pcs_reset_assert(port); in mvpp2_stop()
4917 static int mvpp2_prs_mac_da_accept_list(struct mvpp2_port *port, in mvpp2_prs_mac_da_accept_list() argument
4924 ret = mvpp2_prs_mac_da_accept(port, ha->addr, true); in mvpp2_prs_mac_da_accept_list()
4932 static void mvpp2_set_rx_promisc(struct mvpp2_port *port, bool enable) in mvpp2_set_rx_promisc() argument
4934 if (!enable && (port->dev->features & NETIF_F_HW_VLAN_CTAG_FILTER)) in mvpp2_set_rx_promisc()
4935 mvpp2_prs_vid_enable_filtering(port); in mvpp2_set_rx_promisc()
4937 mvpp2_prs_vid_disable_filtering(port); in mvpp2_set_rx_promisc()
4939 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_promisc()
4942 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_promisc()
4948 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_set_rx_mode() local
4951 mvpp2_prs_mac_del_all(port); in mvpp2_set_rx_mode()
4953 if (dev->flags & IFF_PROMISC) { in mvpp2_set_rx_mode()
4954 mvpp2_set_rx_promisc(port, true); in mvpp2_set_rx_mode()
4958 mvpp2_set_rx_promisc(port, false); in mvpp2_set_rx_mode()
4961 mvpp2_prs_mac_da_accept_list(port, &dev->uc)) in mvpp2_set_rx_mode()
4962 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_mode()
4965 if (dev->flags & IFF_ALLMULTI) { in mvpp2_set_rx_mode()
4966 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_mode()
4972 mvpp2_prs_mac_da_accept_list(port, &dev->mc)) in mvpp2_set_rx_mode()
4973 mvpp2_prs_mac_promisc_set(port->priv, port->id, in mvpp2_set_rx_mode()
4982 if (!is_valid_ether_addr(addr->sa_data)) in mvpp2_set_mac_address()
4983 return -EADDRNOTAVAIL; in mvpp2_set_mac_address()
4985 err = mvpp2_prs_update_mac_da(dev, addr->sa_data); in mvpp2_set_mac_address()
4988 mvpp2_prs_update_mac_da(dev, dev->dev_addr); in mvpp2_set_mac_address()
4999 bool change_percpu = (percpu != priv->percpu_pools); in mvpp2_bm_switch_buffers()
5001 struct mvpp2_port *port = NULL; in mvpp2_bm_switch_buffers() local
5004 for (i = 0; i < priv->port_count; i++) { in mvpp2_bm_switch_buffers()
5005 port = priv->port_list[i]; in mvpp2_bm_switch_buffers()
5006 status[i] = netif_running(port->dev); in mvpp2_bm_switch_buffers()
5008 mvpp2_stop(port->dev); in mvpp2_bm_switch_buffers()
5012 if (priv->percpu_pools) in mvpp2_bm_switch_buffers()
5013 numbufs = port->nrxqs * 2; in mvpp2_bm_switch_buffers()
5019 mvpp2_bm_pool_destroy(port->dev->dev.parent, priv, &priv->bm_pools[i]); in mvpp2_bm_switch_buffers()
5021 devm_kfree(port->dev->dev.parent, priv->bm_pools); in mvpp2_bm_switch_buffers()
5022 priv->percpu_pools = percpu; in mvpp2_bm_switch_buffers()
5023 mvpp2_bm_init(port->dev->dev.parent, priv); in mvpp2_bm_switch_buffers()
5025 for (i = 0; i < priv->port_count; i++) { in mvpp2_bm_switch_buffers()
5026 port = priv->port_list[i]; in mvpp2_bm_switch_buffers()
5027 if (percpu && port->ntxqs >= num_possible_cpus() * 2) in mvpp2_bm_switch_buffers()
5028 xdp_set_features_flag(port->dev, in mvpp2_bm_switch_buffers()
5033 xdp_clear_features_flag(port->dev); in mvpp2_bm_switch_buffers()
5035 mvpp2_swf_bm_pool_init(port); in mvpp2_bm_switch_buffers()
5037 mvpp2_open(port->dev); in mvpp2_bm_switch_buffers()
5048 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_change_mtu() local
5050 struct mvpp2 *priv = port->priv; in mvpp2_change_mtu()
5059 if (port->xdp_prog && mtu > MVPP2_MAX_RX_BUF_SIZE) { in mvpp2_change_mtu()
5062 return -EINVAL; in mvpp2_change_mtu()
5066 if (priv->percpu_pools) { in mvpp2_change_mtu()
5074 for (i = 0; i < priv->port_count; i++) in mvpp2_change_mtu()
5075 if (priv->port_list[i] != port && in mvpp2_change_mtu()
5076 MVPP2_RX_PKT_SIZE(priv->port_list[i]->dev->mtu) > in mvpp2_change_mtu()
5082 /* No port is using jumbo frames */ in mvpp2_change_mtu()
5084 dev_info(port->dev->dev.parent, in mvpp2_change_mtu()
5085 "all ports have a low MTU, switching to per-cpu buffers"); in mvpp2_change_mtu()
5091 mvpp2_stop_dev(port); in mvpp2_change_mtu()
5097 mvpp2_bm_update_mtu(dev, dev->mtu); in mvpp2_change_mtu()
5099 port->pkt_size = MVPP2_RX_PKT_SIZE(mtu); in mvpp2_change_mtu()
5103 mvpp2_start_dev(port); in mvpp2_change_mtu()
5104 mvpp2_egress_enable(port); in mvpp2_change_mtu()
5105 mvpp2_ingress_enable(port); in mvpp2_change_mtu()
5111 static int mvpp2_check_pagepool_dma(struct mvpp2_port *port) in mvpp2_check_pagepool_dma() argument
5114 struct mvpp2 *priv = port->priv; in mvpp2_check_pagepool_dma()
5115 int err = -1, i; in mvpp2_check_pagepool_dma()
5117 if (!priv->percpu_pools) in mvpp2_check_pagepool_dma()
5120 if (!priv->page_pool[0]) in mvpp2_check_pagepool_dma()
5121 return -ENOMEM; in mvpp2_check_pagepool_dma()
5123 for (i = 0; i < priv->port_count; i++) { in mvpp2_check_pagepool_dma()
5124 port = priv->port_list[i]; in mvpp2_check_pagepool_dma()
5125 if (port->xdp_prog) { in mvpp2_check_pagepool_dma()
5132 if (priv->page_pool[0]->p.dma_dir != dma_dir) in mvpp2_check_pagepool_dma()
5141 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_get_stats64() local
5152 cpu_stats = per_cpu_ptr(port->stats, cpu); in mvpp2_get_stats64()
5154 start = u64_stats_fetch_begin(&cpu_stats->syncp); in mvpp2_get_stats64()
5155 rx_packets = cpu_stats->rx_packets; in mvpp2_get_stats64()
5156 rx_bytes = cpu_stats->rx_bytes; in mvpp2_get_stats64()
5157 tx_packets = cpu_stats->tx_packets; in mvpp2_get_stats64()
5158 tx_bytes = cpu_stats->tx_bytes; in mvpp2_get_stats64()
5159 } while (u64_stats_fetch_retry(&cpu_stats->syncp, start)); in mvpp2_get_stats64()
5161 stats->rx_packets += rx_packets; in mvpp2_get_stats64()
5162 stats->rx_bytes += rx_bytes; in mvpp2_get_stats64()
5163 stats->tx_packets += tx_packets; in mvpp2_get_stats64()
5164 stats->tx_bytes += tx_bytes; in mvpp2_get_stats64()
5167 stats->rx_errors = dev->stats.rx_errors; in mvpp2_get_stats64()
5168 stats->rx_dropped = dev->stats.rx_dropped; in mvpp2_get_stats64()
5169 stats->tx_dropped = dev->stats.tx_dropped; in mvpp2_get_stats64()
5172 static int mvpp2_set_ts_config(struct mvpp2_port *port, struct ifreq *ifr) in mvpp2_set_ts_config() argument
5178 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) in mvpp2_set_ts_config()
5179 return -EFAULT; in mvpp2_set_ts_config()
5183 return -ERANGE; in mvpp2_set_ts_config()
5185 ptp = port->priv->iface_base + MVPP22_PTP_BASE(port->gop_id); in mvpp2_set_ts_config()
5200 mvpp22_tai_start(port->priv->tai); in mvpp2_set_ts_config()
5208 port->rx_hwtstamp = true; in mvpp2_set_ts_config()
5210 port->rx_hwtstamp = false; in mvpp2_set_ts_config()
5222 mvpp22_tai_stop(port->priv->tai); in mvpp2_set_ts_config()
5224 port->tx_hwtstamp_type = config.tx_type; in mvpp2_set_ts_config()
5226 if (copy_to_user(ifr->ifr_data, &config, sizeof(config))) in mvpp2_set_ts_config()
5227 return -EFAULT; in mvpp2_set_ts_config()
5232 static int mvpp2_get_ts_config(struct mvpp2_port *port, struct ifreq *ifr) in mvpp2_get_ts_config() argument
5238 config.tx_type = port->tx_hwtstamp_type; in mvpp2_get_ts_config()
5239 config.rx_filter = port->rx_hwtstamp ? in mvpp2_get_ts_config()
5242 if (copy_to_user(ifr->ifr_data, &config, sizeof(config))) in mvpp2_get_ts_config()
5243 return -EFAULT; in mvpp2_get_ts_config()
5251 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_ts_info() local
5253 if (!port->hwtstamp) in mvpp2_ethtool_get_ts_info()
5254 return -EOPNOTSUPP; in mvpp2_ethtool_get_ts_info()
5256 info->phc_index = mvpp22_tai_ptp_clock_index(port->priv->tai); in mvpp2_ethtool_get_ts_info()
5257 info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | in mvpp2_ethtool_get_ts_info()
5263 info->tx_types = BIT(HWTSTAMP_TX_OFF) | in mvpp2_ethtool_get_ts_info()
5265 info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) | in mvpp2_ethtool_get_ts_info()
5273 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ioctl() local
5277 if (port->hwtstamp) in mvpp2_ioctl()
5278 return mvpp2_set_ts_config(port, ifr); in mvpp2_ioctl()
5282 if (port->hwtstamp) in mvpp2_ioctl()
5283 return mvpp2_get_ts_config(port, ifr); in mvpp2_ioctl()
5287 if (!port->phylink) in mvpp2_ioctl()
5288 return -ENOTSUPP; in mvpp2_ioctl()
5290 return phylink_mii_ioctl(port->phylink, ifr, cmd); in mvpp2_ioctl()
5295 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_vlan_rx_add_vid() local
5298 ret = mvpp2_prs_vid_entry_add(port, vid); in mvpp2_vlan_rx_add_vid()
5300 netdev_err(dev, "rx-vlan-filter offloading cannot accept more than %d VIDs per port\n", in mvpp2_vlan_rx_add_vid()
5301 MVPP2_PRS_VLAN_FILT_MAX - 1); in mvpp2_vlan_rx_add_vid()
5307 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_vlan_rx_kill_vid() local
5309 mvpp2_prs_vid_entry_remove(port, vid); in mvpp2_vlan_rx_kill_vid()
5316 netdev_features_t changed = dev->features ^ features; in mvpp2_set_features()
5317 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_set_features() local
5321 mvpp2_prs_vid_enable_filtering(port); in mvpp2_set_features()
5324 * port in mvpp2_set_features()
5326 mvpp2_prs_vid_remove_all(port); in mvpp2_set_features()
5328 mvpp2_prs_vid_disable_filtering(port); in mvpp2_set_features()
5334 mvpp22_port_rss_enable(port); in mvpp2_set_features()
5336 mvpp22_port_rss_disable(port); in mvpp2_set_features()
5342 static int mvpp2_xdp_setup(struct mvpp2_port *port, struct netdev_bpf *bpf) in mvpp2_xdp_setup() argument
5344 struct bpf_prog *prog = bpf->prog, *old_prog; in mvpp2_xdp_setup()
5345 bool running = netif_running(port->dev); in mvpp2_xdp_setup()
5346 bool reset = !prog != !port->xdp_prog; in mvpp2_xdp_setup()
5348 if (port->dev->mtu > MVPP2_MAX_RX_BUF_SIZE) { in mvpp2_xdp_setup()
5349 NL_SET_ERR_MSG_MOD(bpf->extack, "MTU too large for XDP"); in mvpp2_xdp_setup()
5350 return -EOPNOTSUPP; in mvpp2_xdp_setup()
5353 if (!port->priv->percpu_pools) { in mvpp2_xdp_setup()
5354 NL_SET_ERR_MSG_MOD(bpf->extack, "Per CPU Pools required for XDP"); in mvpp2_xdp_setup()
5355 return -EOPNOTSUPP; in mvpp2_xdp_setup()
5358 if (port->ntxqs < num_possible_cpus() * 2) { in mvpp2_xdp_setup()
5359 NL_SET_ERR_MSG_MOD(bpf->extack, "XDP_TX needs two TX queues per CPU"); in mvpp2_xdp_setup()
5360 return -EOPNOTSUPP; in mvpp2_xdp_setup()
5365 mvpp2_stop(port->dev); in mvpp2_xdp_setup()
5367 old_prog = xchg(&port->xdp_prog, prog); in mvpp2_xdp_setup()
5377 mvpp2_open(port->dev); in mvpp2_xdp_setup()
5380 mvpp2_check_pagepool_dma(port); in mvpp2_xdp_setup()
5387 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_xdp() local
5389 switch (xdp->command) { in mvpp2_xdp()
5391 return mvpp2_xdp_setup(port, xdp); in mvpp2_xdp()
5393 return -EINVAL; in mvpp2_xdp()
5401 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_nway_reset() local
5403 if (!port->phylink) in mvpp2_ethtool_nway_reset()
5404 return -ENOTSUPP; in mvpp2_ethtool_nway_reset()
5406 return phylink_ethtool_nway_reset(port->phylink); in mvpp2_ethtool_nway_reset()
5416 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_set_coalesce() local
5419 for (queue = 0; queue < port->nrxqs; queue++) { in mvpp2_ethtool_set_coalesce()
5420 struct mvpp2_rx_queue *rxq = port->rxqs[queue]; in mvpp2_ethtool_set_coalesce()
5422 rxq->time_coal = c->rx_coalesce_usecs; in mvpp2_ethtool_set_coalesce()
5423 rxq->pkts_coal = c->rx_max_coalesced_frames; in mvpp2_ethtool_set_coalesce()
5424 mvpp2_rx_pkts_coal_set(port, rxq); in mvpp2_ethtool_set_coalesce()
5425 mvpp2_rx_time_coal_set(port, rxq); in mvpp2_ethtool_set_coalesce()
5428 if (port->has_tx_irqs) { in mvpp2_ethtool_set_coalesce()
5429 port->tx_time_coal = c->tx_coalesce_usecs; in mvpp2_ethtool_set_coalesce()
5430 mvpp2_tx_time_coal_set(port); in mvpp2_ethtool_set_coalesce()
5433 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_ethtool_set_coalesce()
5434 struct mvpp2_tx_queue *txq = port->txqs[queue]; in mvpp2_ethtool_set_coalesce()
5436 txq->done_pkts_coal = c->tx_max_coalesced_frames; in mvpp2_ethtool_set_coalesce()
5438 if (port->has_tx_irqs) in mvpp2_ethtool_set_coalesce()
5439 mvpp2_tx_pkts_coal_set(port, txq); in mvpp2_ethtool_set_coalesce()
5452 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_coalesce() local
5454 c->rx_coalesce_usecs = port->rxqs[0]->time_coal; in mvpp2_ethtool_get_coalesce()
5455 c->rx_max_coalesced_frames = port->rxqs[0]->pkts_coal; in mvpp2_ethtool_get_coalesce()
5456 c->tx_max_coalesced_frames = port->txqs[0]->done_pkts_coal; in mvpp2_ethtool_get_coalesce()
5457 c->tx_coalesce_usecs = port->tx_time_coal; in mvpp2_ethtool_get_coalesce()
5464 strscpy(drvinfo->driver, MVPP2_DRIVER_NAME, in mvpp2_ethtool_get_drvinfo()
5465 sizeof(drvinfo->driver)); in mvpp2_ethtool_get_drvinfo()
5466 strscpy(drvinfo->version, MVPP2_DRIVER_VERSION, in mvpp2_ethtool_get_drvinfo()
5467 sizeof(drvinfo->version)); in mvpp2_ethtool_get_drvinfo()
5468 strscpy(drvinfo->bus_info, dev_name(&dev->dev), in mvpp2_ethtool_get_drvinfo()
5469 sizeof(drvinfo->bus_info)); in mvpp2_ethtool_get_drvinfo()
5478 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_ringparam() local
5480 ring->rx_max_pending = MVPP2_MAX_RXD_MAX; in mvpp2_ethtool_get_ringparam()
5481 ring->tx_max_pending = MVPP2_MAX_TXD_MAX; in mvpp2_ethtool_get_ringparam()
5482 ring->rx_pending = port->rx_ring_size; in mvpp2_ethtool_get_ringparam()
5483 ring->tx_pending = port->tx_ring_size; in mvpp2_ethtool_get_ringparam()
5492 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_set_ringparam() local
5493 u16 prev_rx_ring_size = port->rx_ring_size; in mvpp2_ethtool_set_ringparam()
5494 u16 prev_tx_ring_size = port->tx_ring_size; in mvpp2_ethtool_set_ringparam()
5502 port->rx_ring_size = ring->rx_pending; in mvpp2_ethtool_set_ringparam()
5503 port->tx_ring_size = ring->tx_pending; in mvpp2_ethtool_set_ringparam()
5510 mvpp2_stop_dev(port); in mvpp2_ethtool_set_ringparam()
5511 mvpp2_cleanup_rxqs(port); in mvpp2_ethtool_set_ringparam()
5512 mvpp2_cleanup_txqs(port); in mvpp2_ethtool_set_ringparam()
5514 port->rx_ring_size = ring->rx_pending; in mvpp2_ethtool_set_ringparam()
5515 port->tx_ring_size = ring->tx_pending; in mvpp2_ethtool_set_ringparam()
5517 err = mvpp2_setup_rxqs(port); in mvpp2_ethtool_set_ringparam()
5520 port->rx_ring_size = prev_rx_ring_size; in mvpp2_ethtool_set_ringparam()
5521 ring->rx_pending = prev_rx_ring_size; in mvpp2_ethtool_set_ringparam()
5522 err = mvpp2_setup_rxqs(port); in mvpp2_ethtool_set_ringparam()
5526 err = mvpp2_setup_txqs(port); in mvpp2_ethtool_set_ringparam()
5529 port->tx_ring_size = prev_tx_ring_size; in mvpp2_ethtool_set_ringparam()
5530 ring->tx_pending = prev_tx_ring_size; in mvpp2_ethtool_set_ringparam()
5531 err = mvpp2_setup_txqs(port); in mvpp2_ethtool_set_ringparam()
5536 mvpp2_start_dev(port); in mvpp2_ethtool_set_ringparam()
5537 mvpp2_egress_enable(port); in mvpp2_ethtool_set_ringparam()
5538 mvpp2_ingress_enable(port); in mvpp2_ethtool_set_ringparam()
5543 mvpp2_cleanup_rxqs(port); in mvpp2_ethtool_set_ringparam()
5552 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_pause_param() local
5554 if (!port->phylink) in mvpp2_ethtool_get_pause_param()
5557 phylink_ethtool_get_pauseparam(port->phylink, pause); in mvpp2_ethtool_get_pause_param()
5563 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_set_pause_param() local
5565 if (!port->phylink) in mvpp2_ethtool_set_pause_param()
5566 return -ENOTSUPP; in mvpp2_ethtool_set_pause_param()
5568 return phylink_ethtool_set_pauseparam(port->phylink, pause); in mvpp2_ethtool_set_pause_param()
5574 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_link_ksettings() local
5576 if (!port->phylink) in mvpp2_ethtool_get_link_ksettings()
5577 return -ENOTSUPP; in mvpp2_ethtool_get_link_ksettings()
5579 return phylink_ethtool_ksettings_get(port->phylink, cmd); in mvpp2_ethtool_get_link_ksettings()
5585 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_set_link_ksettings() local
5587 if (!port->phylink) in mvpp2_ethtool_set_link_ksettings()
5588 return -ENOTSUPP; in mvpp2_ethtool_set_link_ksettings()
5590 return phylink_ethtool_ksettings_set(port->phylink, cmd); in mvpp2_ethtool_set_link_ksettings()
5596 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_rxnfc() local
5599 if (!mvpp22_rss_is_supported(port)) in mvpp2_ethtool_get_rxnfc()
5600 return -EOPNOTSUPP; in mvpp2_ethtool_get_rxnfc()
5602 switch (info->cmd) { in mvpp2_ethtool_get_rxnfc()
5604 ret = mvpp2_ethtool_rxfh_get(port, info); in mvpp2_ethtool_get_rxnfc()
5607 info->data = port->nrxqs; in mvpp2_ethtool_get_rxnfc()
5610 info->rule_cnt = port->n_rfs_rules; in mvpp2_ethtool_get_rxnfc()
5613 ret = mvpp2_ethtool_cls_rule_get(port, info); in mvpp2_ethtool_get_rxnfc()
5617 if (loc == info->rule_cnt) { in mvpp2_ethtool_get_rxnfc()
5618 ret = -EMSGSIZE; in mvpp2_ethtool_get_rxnfc()
5622 if (port->rfs_rules[i]) in mvpp2_ethtool_get_rxnfc()
5627 return -ENOTSUPP; in mvpp2_ethtool_get_rxnfc()
5636 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_set_rxnfc() local
5639 if (!mvpp22_rss_is_supported(port)) in mvpp2_ethtool_set_rxnfc()
5640 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxnfc()
5642 switch (info->cmd) { in mvpp2_ethtool_set_rxnfc()
5644 ret = mvpp2_ethtool_rxfh_set(port, info); in mvpp2_ethtool_set_rxnfc()
5647 ret = mvpp2_ethtool_cls_rule_ins(port, info); in mvpp2_ethtool_set_rxnfc()
5650 ret = mvpp2_ethtool_cls_rule_del(port, info); in mvpp2_ethtool_set_rxnfc()
5653 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxnfc()
5660 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_rxfh_indir_size() local
5662 return mvpp22_rss_is_supported(port) ? MVPP22_RSS_TABLE_ENTRIES : 0; in mvpp2_ethtool_get_rxfh_indir_size()
5668 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_rxfh() local
5671 if (!mvpp22_rss_is_supported(port)) in mvpp2_ethtool_get_rxfh()
5672 return -EOPNOTSUPP; in mvpp2_ethtool_get_rxfh()
5675 ret = mvpp22_port_rss_ctx_indir_get(port, 0, indir); in mvpp2_ethtool_get_rxfh()
5686 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_set_rxfh() local
5689 if (!mvpp22_rss_is_supported(port)) in mvpp2_ethtool_set_rxfh()
5690 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh()
5693 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh()
5696 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh()
5699 ret = mvpp22_port_rss_ctx_indir_set(port, 0, indir); in mvpp2_ethtool_set_rxfh()
5707 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_get_rxfh_context() local
5710 if (!mvpp22_rss_is_supported(port)) in mvpp2_ethtool_get_rxfh_context()
5711 return -EOPNOTSUPP; in mvpp2_ethtool_get_rxfh_context()
5713 return -EINVAL; in mvpp2_ethtool_get_rxfh_context()
5719 ret = mvpp22_port_rss_ctx_indir_get(port, rss_context, indir); in mvpp2_ethtool_get_rxfh_context()
5729 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_ethtool_set_rxfh_context() local
5732 if (!mvpp22_rss_is_supported(port)) in mvpp2_ethtool_set_rxfh_context()
5733 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh_context()
5736 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh_context()
5739 return -EOPNOTSUPP; in mvpp2_ethtool_set_rxfh_context()
5742 return mvpp22_port_rss_ctx_delete(port, *rss_context); in mvpp2_ethtool_set_rxfh_context()
5745 ret = mvpp22_port_rss_ctx_create(port, rss_context); in mvpp2_ethtool_set_rxfh_context()
5750 return mvpp22_port_rss_ctx_indir_set(port, *rss_context, indir); in mvpp2_ethtool_set_rxfh_context()
5798 * had a single IRQ defined per-port.
5800 static int mvpp2_simple_queue_vectors_init(struct mvpp2_port *port, in mvpp2_simple_queue_vectors_init() argument
5803 struct mvpp2_queue_vector *v = &port->qvecs[0]; in mvpp2_simple_queue_vectors_init()
5805 v->first_rxq = 0; in mvpp2_simple_queue_vectors_init()
5806 v->nrxqs = port->nrxqs; in mvpp2_simple_queue_vectors_init()
5807 v->type = MVPP2_QUEUE_VECTOR_SHARED; in mvpp2_simple_queue_vectors_init()
5808 v->sw_thread_id = 0; in mvpp2_simple_queue_vectors_init()
5809 v->sw_thread_mask = *cpumask_bits(cpu_online_mask); in mvpp2_simple_queue_vectors_init()
5810 v->port = port; in mvpp2_simple_queue_vectors_init()
5811 v->irq = irq_of_parse_and_map(port_node, 0); in mvpp2_simple_queue_vectors_init()
5812 if (v->irq <= 0) in mvpp2_simple_queue_vectors_init()
5813 return -EINVAL; in mvpp2_simple_queue_vectors_init()
5814 netif_napi_add(port->dev, &v->napi, mvpp2_poll); in mvpp2_simple_queue_vectors_init()
5816 port->nqvecs = 1; in mvpp2_simple_queue_vectors_init()
5821 static int mvpp2_multi_queue_vectors_init(struct mvpp2_port *port, in mvpp2_multi_queue_vectors_init() argument
5824 struct mvpp2 *priv = port->priv; in mvpp2_multi_queue_vectors_init()
5830 port->nqvecs = priv->nthreads + 1; in mvpp2_multi_queue_vectors_init()
5833 port->nqvecs = priv->nthreads; in mvpp2_multi_queue_vectors_init()
5837 for (i = 0; i < port->nqvecs; i++) { in mvpp2_multi_queue_vectors_init()
5840 v = port->qvecs + i; in mvpp2_multi_queue_vectors_init()
5842 v->port = port; in mvpp2_multi_queue_vectors_init()
5843 v->type = MVPP2_QUEUE_VECTOR_PRIVATE; in mvpp2_multi_queue_vectors_init()
5844 v->sw_thread_id = i; in mvpp2_multi_queue_vectors_init()
5845 v->sw_thread_mask = BIT(i); in mvpp2_multi_queue_vectors_init()
5847 if (port->flags & MVPP2_F_DT_COMPAT) in mvpp2_multi_queue_vectors_init()
5848 snprintf(irqname, sizeof(irqname), "tx-cpu%d", i); in mvpp2_multi_queue_vectors_init()
5853 v->first_rxq = i; in mvpp2_multi_queue_vectors_init()
5854 v->nrxqs = 1; in mvpp2_multi_queue_vectors_init()
5856 i == (port->nqvecs - 1)) { in mvpp2_multi_queue_vectors_init()
5857 v->first_rxq = 0; in mvpp2_multi_queue_vectors_init()
5858 v->nrxqs = port->nrxqs; in mvpp2_multi_queue_vectors_init()
5859 v->type = MVPP2_QUEUE_VECTOR_SHARED; in mvpp2_multi_queue_vectors_init()
5861 if (port->flags & MVPP2_F_DT_COMPAT) in mvpp2_multi_queue_vectors_init()
5862 strncpy(irqname, "rx-shared", sizeof(irqname)); in mvpp2_multi_queue_vectors_init()
5866 v->irq = of_irq_get_byname(port_node, irqname); in mvpp2_multi_queue_vectors_init()
5868 v->irq = fwnode_irq_get(port->fwnode, i); in mvpp2_multi_queue_vectors_init()
5869 if (v->irq <= 0) { in mvpp2_multi_queue_vectors_init()
5870 ret = -EINVAL; in mvpp2_multi_queue_vectors_init()
5874 netif_napi_add(port->dev, &v->napi, mvpp2_poll); in mvpp2_multi_queue_vectors_init()
5880 for (i = 0; i < port->nqvecs; i++) in mvpp2_multi_queue_vectors_init()
5881 irq_dispose_mapping(port->qvecs[i].irq); in mvpp2_multi_queue_vectors_init()
5885 static int mvpp2_queue_vectors_init(struct mvpp2_port *port, in mvpp2_queue_vectors_init() argument
5888 if (port->has_tx_irqs) in mvpp2_queue_vectors_init()
5889 return mvpp2_multi_queue_vectors_init(port, port_node); in mvpp2_queue_vectors_init()
5891 return mvpp2_simple_queue_vectors_init(port, port_node); in mvpp2_queue_vectors_init()
5894 static void mvpp2_queue_vectors_deinit(struct mvpp2_port *port) in mvpp2_queue_vectors_deinit() argument
5898 for (i = 0; i < port->nqvecs; i++) in mvpp2_queue_vectors_deinit()
5899 irq_dispose_mapping(port->qvecs[i].irq); in mvpp2_queue_vectors_deinit()
5902 /* Configure Rx queue group interrupt for this port */
5903 static void mvpp2_rx_irqs_setup(struct mvpp2_port *port) in mvpp2_rx_irqs_setup() argument
5905 struct mvpp2 *priv = port->priv; in mvpp2_rx_irqs_setup()
5909 if (priv->hw_version == MVPP21) { in mvpp2_rx_irqs_setup()
5910 mvpp2_write(priv, MVPP21_ISR_RXQ_GROUP_REG(port->id), in mvpp2_rx_irqs_setup()
5911 port->nrxqs); in mvpp2_rx_irqs_setup()
5916 for (i = 0; i < port->nqvecs; i++) { in mvpp2_rx_irqs_setup()
5917 struct mvpp2_queue_vector *qv = port->qvecs + i; in mvpp2_rx_irqs_setup()
5919 if (!qv->nrxqs) in mvpp2_rx_irqs_setup()
5922 val = qv->sw_thread_id; in mvpp2_rx_irqs_setup()
5923 val |= port->id << MVPP22_ISR_RXQ_GROUP_INDEX_GROUP_OFFSET; in mvpp2_rx_irqs_setup()
5926 val = qv->first_rxq; in mvpp2_rx_irqs_setup()
5927 val |= qv->nrxqs << MVPP22_ISR_RXQ_SUB_GROUP_SIZE_OFFSET; in mvpp2_rx_irqs_setup()
5932 /* Initialize port HW */
5933 static int mvpp2_port_init(struct mvpp2_port *port) in mvpp2_port_init() argument
5935 struct device *dev = port->dev->dev.parent; in mvpp2_port_init()
5936 struct mvpp2 *priv = port->priv; in mvpp2_port_init()
5942 if (port->first_rxq + port->nrxqs > in mvpp2_port_init()
5943 MVPP2_MAX_PORTS * priv->max_port_rxqs) in mvpp2_port_init()
5944 return -EINVAL; in mvpp2_port_init()
5946 if (port->nrxqs > priv->max_port_rxqs || port->ntxqs > MVPP2_MAX_TXQ) in mvpp2_port_init()
5947 return -EINVAL; in mvpp2_port_init()
5949 /* Disable port */ in mvpp2_port_init()
5950 mvpp2_egress_disable(port); in mvpp2_port_init()
5951 mvpp2_port_disable(port); in mvpp2_port_init()
5953 if (mvpp2_is_xlg(port->phy_interface)) { in mvpp2_port_init()
5954 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_init()
5957 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_port_init()
5959 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_port_init()
5962 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_port_init()
5965 port->tx_time_coal = MVPP2_TXDONE_COAL_USEC; in mvpp2_port_init()
5967 port->txqs = devm_kcalloc(dev, port->ntxqs, sizeof(*port->txqs), in mvpp2_port_init()
5969 if (!port->txqs) in mvpp2_port_init()
5970 return -ENOMEM; in mvpp2_port_init()
5972 /* Associate physical Tx queues to this port and initialize. in mvpp2_port_init()
5975 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_port_init()
5976 int queue_phy_id = mvpp2_txq_phys(port->id, queue); in mvpp2_port_init()
5981 err = -ENOMEM; in mvpp2_port_init()
5985 txq->pcpu = alloc_percpu(struct mvpp2_txq_pcpu); in mvpp2_port_init()
5986 if (!txq->pcpu) { in mvpp2_port_init()
5987 err = -ENOMEM; in mvpp2_port_init()
5991 txq->id = queue_phy_id; in mvpp2_port_init()
5992 txq->log_id = queue; in mvpp2_port_init()
5993 txq->done_pkts_coal = MVPP2_TXDONE_COAL_PKTS_THRESH; in mvpp2_port_init()
5994 for (thread = 0; thread < priv->nthreads; thread++) { in mvpp2_port_init()
5995 txq_pcpu = per_cpu_ptr(txq->pcpu, thread); in mvpp2_port_init()
5996 txq_pcpu->thread = thread; in mvpp2_port_init()
5999 port->txqs[queue] = txq; in mvpp2_port_init()
6002 port->rxqs = devm_kcalloc(dev, port->nrxqs, sizeof(*port->rxqs), in mvpp2_port_init()
6004 if (!port->rxqs) { in mvpp2_port_init()
6005 err = -ENOMEM; in mvpp2_port_init()
6009 /* Allocate and initialize Rx queue for this port */ in mvpp2_port_init()
6010 for (queue = 0; queue < port->nrxqs; queue++) { in mvpp2_port_init()
6013 /* Map physical Rx queue to port's logical Rx queue */ in mvpp2_port_init()
6016 err = -ENOMEM; in mvpp2_port_init()
6020 rxq->id = port->first_rxq + queue; in mvpp2_port_init()
6021 rxq->port = port->id; in mvpp2_port_init()
6022 rxq->logic_rxq = queue; in mvpp2_port_init()
6024 port->rxqs[queue] = rxq; in mvpp2_port_init()
6027 mvpp2_rx_irqs_setup(port); in mvpp2_port_init()
6030 for (queue = 0; queue < port->nrxqs; queue++) { in mvpp2_port_init()
6031 struct mvpp2_rx_queue *rxq = port->rxqs[queue]; in mvpp2_port_init()
6033 rxq->size = port->rx_ring_size; in mvpp2_port_init()
6034 rxq->pkts_coal = MVPP2_RX_COAL_PKTS; in mvpp2_port_init()
6035 rxq->time_coal = MVPP2_RX_COAL_USEC; in mvpp2_port_init()
6038 mvpp2_ingress_disable(port); in mvpp2_port_init()
6040 /* Port default configuration */ in mvpp2_port_init()
6041 mvpp2_defaults_set(port); in mvpp2_port_init()
6043 /* Port's classifier configuration */ in mvpp2_port_init()
6044 mvpp2_cls_oversize_rxq_set(port); in mvpp2_port_init()
6045 mvpp2_cls_port_config(port); in mvpp2_port_init()
6047 if (mvpp22_rss_is_supported(port)) in mvpp2_port_init()
6048 mvpp22_port_rss_init(port); in mvpp2_port_init()
6051 port->pkt_size = MVPP2_RX_PKT_SIZE(port->dev->mtu); in mvpp2_port_init()
6054 err = mvpp2_swf_bm_pool_init(port); in mvpp2_port_init()
6058 /* Clear all port stats */ in mvpp2_port_init()
6059 mvpp2_read_stats(port); in mvpp2_port_init()
6060 memset(port->ethtool_stats, 0, in mvpp2_port_init()
6061 MVPP2_N_ETHTOOL_STATS(port->ntxqs, port->nrxqs) * sizeof(u64)); in mvpp2_port_init()
6066 for (queue = 0; queue < port->ntxqs; queue++) { in mvpp2_port_init()
6067 if (!port->txqs[queue]) in mvpp2_port_init()
6069 free_percpu(port->txqs[queue]->pcpu); in mvpp2_port_init()
6077 char *irqs[5] = { "rx-shared", "tx-cpu0", "tx-cpu1", "tx-cpu2", in mvpp22_port_has_legacy_tx_irqs()
6078 "tx-cpu3" }; in mvpp22_port_has_legacy_tx_irqs()
6082 if (of_property_match_string(port_node, "interrupt-names", in mvpp22_port_has_legacy_tx_irqs()
6090 /* Checks if the port dt description has the required Tx interrupts:
6091 * - PPv2.1: there are no such interrupts.
6092 * - PPv2.2 and PPv2.3:
6093 * - The old DTs have: "rx-shared", "tx-cpuX" with X in [0...3]
6094 * - The new ones have: "hifX" with X in [0..8]
6109 if (priv->hw_version == MVPP21) in mvpp2_port_has_irqs()
6117 if (of_property_match_string(port_node, "interrupt-names", in mvpp2_port_has_irqs()
6129 struct mvpp2_port *port = netdev_priv(dev); in mvpp2_port_copy_mac_addr() local
6140 if (priv->hw_version == MVPP21) { in mvpp2_port_copy_mac_addr()
6141 mvpp21_get_mac_address(port, hw_mac_addr); in mvpp2_port_copy_mac_addr()
6151 if (ret == -EPROBE_DEFER) in mvpp2_port_copy_mac_addr()
6183 struct mvpp2_port *port = mvpp2_pcs_xlg_to_port(pcs); in mvpp2_xlg_pcs_get_state() local
6186 if (port->phy_interface == PHY_INTERFACE_MODE_5GBASER) in mvpp2_xlg_pcs_get_state()
6187 state->speed = SPEED_5000; in mvpp2_xlg_pcs_get_state()
6189 state->speed = SPEED_10000; in mvpp2_xlg_pcs_get_state()
6190 state->duplex = 1; in mvpp2_xlg_pcs_get_state()
6191 state->an_complete = 1; in mvpp2_xlg_pcs_get_state()
6193 val = readl(port->base + MVPP22_XLG_STATUS); in mvpp2_xlg_pcs_get_state()
6194 state->link = !!(val & MVPP22_XLG_STATUS_LINK_UP); in mvpp2_xlg_pcs_get_state()
6196 state->pause = 0; in mvpp2_xlg_pcs_get_state()
6197 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_xlg_pcs_get_state()
6199 state->pause |= MLO_PAUSE_TX; in mvpp2_xlg_pcs_get_state()
6201 state->pause |= MLO_PAUSE_RX; in mvpp2_xlg_pcs_get_state()
6222 * Bit 2 Field InBandAnEn In-band Auto-Negotiation enable. ... in mvpp2_gmac_pcs_validate()
6223 * When <PortType> = 1 (1000BASE-X) this field must be set to 1. in mvpp2_gmac_pcs_validate()
6225 if (phy_interface_mode_is_8023z(state->interface) && in mvpp2_gmac_pcs_validate()
6226 !phylink_test(state->advertising, Autoneg)) in mvpp2_gmac_pcs_validate()
6227 return -EINVAL; in mvpp2_gmac_pcs_validate()
6235 struct mvpp2_port *port = mvpp2_pcs_gmac_to_port(pcs); in mvpp2_gmac_pcs_get_state() local
6238 val = readl(port->base + MVPP2_GMAC_STATUS0); in mvpp2_gmac_pcs_get_state()
6240 state->an_complete = !!(val & MVPP2_GMAC_STATUS0_AN_COMPLETE); in mvpp2_gmac_pcs_get_state()
6241 state->link = !!(val & MVPP2_GMAC_STATUS0_LINK_UP); in mvpp2_gmac_pcs_get_state()
6242 state->duplex = !!(val & MVPP2_GMAC_STATUS0_FULL_DUPLEX); in mvpp2_gmac_pcs_get_state()
6244 switch (port->phy_interface) { in mvpp2_gmac_pcs_get_state()
6246 state->speed = SPEED_1000; in mvpp2_gmac_pcs_get_state()
6249 state->speed = SPEED_2500; in mvpp2_gmac_pcs_get_state()
6253 state->speed = SPEED_1000; in mvpp2_gmac_pcs_get_state()
6255 state->speed = SPEED_100; in mvpp2_gmac_pcs_get_state()
6257 state->speed = SPEED_10; in mvpp2_gmac_pcs_get_state()
6260 state->pause = 0; in mvpp2_gmac_pcs_get_state()
6262 state->pause |= MLO_PAUSE_RX; in mvpp2_gmac_pcs_get_state()
6264 state->pause |= MLO_PAUSE_TX; in mvpp2_gmac_pcs_get_state()
6272 struct mvpp2_port *port = mvpp2_pcs_gmac_to_port(pcs); in mvpp2_gmac_pcs_config() local
6314 old_an = an = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_config()
6318 writel(an, port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_config()
6326 struct mvpp2_port *port = mvpp2_pcs_gmac_to_port(pcs); in mvpp2_gmac_pcs_an_restart() local
6327 u32 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_an_restart()
6330 port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_an_restart()
6332 port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_gmac_pcs_an_restart()
6342 static void mvpp2_xlg_config(struct mvpp2_port *port, unsigned int mode, in mvpp2_xlg_config() argument
6347 mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG, in mvpp2_xlg_config()
6350 mvpp2_modify(port->base + MVPP22_XLG_CTRL4_REG, in mvpp2_xlg_config()
6358 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_xlg_config()
6362 static void mvpp2_gmac_config(struct mvpp2_port *port, unsigned int mode, in mvpp2_gmac_config() argument
6369 old_ctrl0 = ctrl0 = readl(port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_gmac_config()
6370 old_ctrl2 = ctrl2 = readl(port->base + MVPP2_GMAC_CTRL_2_REG); in mvpp2_gmac_config()
6371 old_ctrl4 = ctrl4 = readl(port->base + MVPP22_GMAC_CTRL_4_REG); in mvpp2_gmac_config()
6376 /* Configure port type */ in mvpp2_gmac_config()
6377 if (phy_interface_mode_is_8023z(state->interface)) { in mvpp2_gmac_config()
6383 } else if (state->interface == PHY_INTERFACE_MODE_SGMII) { in mvpp2_gmac_config()
6389 } else if (phy_interface_mode_is_rgmii(state->interface)) { in mvpp2_gmac_config()
6398 /* Phy or fixed speed - no in-band AN, nothing to do, leave the in mvpp2_gmac_config()
6399 * configured speed, duplex and flow control as-is. in mvpp2_gmac_config()
6401 } else if (state->interface == PHY_INTERFACE_MODE_SGMII) { in mvpp2_gmac_config()
6402 /* SGMII in-band mode receives the speed and duplex from in mvpp2_gmac_config()
6404 } else if (phy_interface_mode_is_8023z(state->interface)) { in mvpp2_gmac_config()
6414 writel(ctrl0, port->base + MVPP2_GMAC_CTRL_0_REG); in mvpp2_gmac_config()
6416 writel(ctrl2, port->base + MVPP2_GMAC_CTRL_2_REG); in mvpp2_gmac_config()
6418 writel(ctrl4, port->base + MVPP22_GMAC_CTRL_4_REG); in mvpp2_gmac_config()
6424 struct mvpp2_port *port = mvpp2_phylink_to_port(config); in mvpp2_select_pcs() local
6431 return &port->pcs_xlg; in mvpp2_select_pcs()
6433 return &port->pcs_gmac; in mvpp2_select_pcs()
6439 struct mvpp2_port *port = mvpp2_phylink_to_port(config); in mvpp2_mac_prepare() local
6442 if (mvpp2_is_xlg(interface) && port->gop_id != 0) { in mvpp2_mac_prepare()
6443 netdev_err(port->dev, "Invalid mode on %s\n", port->dev->name); in mvpp2_mac_prepare()
6444 return -EINVAL; in mvpp2_mac_prepare()
6447 if (port->phy_interface != interface || in mvpp2_mac_prepare()
6450 * in-band mode to ensure we do not change the configuration in mvpp2_mac_prepare()
6455 mvpp2_modify(port->base + MVPP2_GMAC_AUTONEG_CONFIG, in mvpp2_mac_prepare()
6460 if (mvpp2_port_supports_xlg(port)) in mvpp2_mac_prepare()
6461 mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG, in mvpp2_mac_prepare()
6467 /* Make sure the port is disabled when reconfiguring the mode */ in mvpp2_mac_prepare()
6468 mvpp2_port_disable(port); in mvpp2_mac_prepare()
6470 if (port->phy_interface != interface) { in mvpp2_mac_prepare()
6472 mvpp2_modify(port->base + MVPP2_GMAC_CTRL_2_REG, in mvpp2_mac_prepare()
6476 if (port->priv->hw_version >= MVPP22) { in mvpp2_mac_prepare()
6477 mvpp22_gop_mask_irq(port); in mvpp2_mac_prepare()
6479 phy_power_off(port->comphy); in mvpp2_mac_prepare()
6482 mvpp22_mode_reconfigure(port, interface); in mvpp2_mac_prepare()
6492 struct mvpp2_port *port = mvpp2_phylink_to_port(config); in mvpp2_mac_config() local
6495 if (mvpp2_is_xlg(state->interface)) in mvpp2_mac_config()
6496 mvpp2_xlg_config(port, mode, state); in mvpp2_mac_config()
6497 else if (phy_interface_mode_is_rgmii(state->interface) || in mvpp2_mac_config()
6498 phy_interface_mode_is_8023z(state->interface) || in mvpp2_mac_config()
6499 state->interface == PHY_INTERFACE_MODE_SGMII) in mvpp2_mac_config()
6500 mvpp2_gmac_config(port, mode, state); in mvpp2_mac_config()
6502 if (port->priv->hw_version == MVPP21 && port->flags & MVPP2_F_LOOPBACK) in mvpp2_mac_config()
6503 mvpp2_port_loopback_set(port, state); in mvpp2_mac_config()
6509 struct mvpp2_port *port = mvpp2_phylink_to_port(config); in mvpp2_mac_finish() local
6511 if (port->priv->hw_version >= MVPP22 && in mvpp2_mac_finish()
6512 port->phy_interface != interface) { in mvpp2_mac_finish()
6513 port->phy_interface = interface; in mvpp2_mac_finish()
6516 mvpp22_gop_unmask_irq(port); in mvpp2_mac_finish()
6521 mvpp2_modify(port->base + MVPP2_GMAC_CTRL_2_REG, in mvpp2_mac_finish()
6524 while (readl(port->base + MVPP2_GMAC_CTRL_2_REG) & in mvpp2_mac_finish()
6529 mvpp2_port_enable(port); in mvpp2_mac_finish()
6531 /* Allow the link to come up if in in-band mode, otherwise the in mvpp2_mac_finish()
6536 mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG, in mvpp2_mac_finish()
6540 mvpp2_modify(port->base + MVPP2_GMAC_AUTONEG_CONFIG, in mvpp2_mac_finish()
6554 struct mvpp2_port *port = mvpp2_phylink_to_port(config); in mvpp2_mac_link_up() local
6566 mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG, in mvpp2_mac_link_up()
6584 mvpp2_modify(port->base + MVPP2_GMAC_AUTONEG_CONFIG, in mvpp2_mac_link_up()
6602 mvpp2_modify(port->base + MVPP22_GMAC_CTRL_4_REG, in mvpp2_mac_link_up()
6607 if (port->priv->global_tx_fc) { in mvpp2_mac_link_up()
6608 port->tx_fc = tx_pause; in mvpp2_mac_link_up()
6610 mvpp2_rxq_enable_fc(port); in mvpp2_mac_link_up()
6612 mvpp2_rxq_disable_fc(port); in mvpp2_mac_link_up()
6613 if (port->priv->percpu_pools) { in mvpp2_mac_link_up()
6614 for (i = 0; i < port->nrxqs; i++) in mvpp2_mac_link_up()
6615 mvpp2_bm_pool_update_fc(port, &port->priv->bm_pools[i], tx_pause); in mvpp2_mac_link_up()
6617 mvpp2_bm_pool_update_fc(port, port->pool_long, tx_pause); in mvpp2_mac_link_up()
6618 mvpp2_bm_pool_update_fc(port, port->pool_short, tx_pause); in mvpp2_mac_link_up()
6620 if (port->priv->hw_version == MVPP23) in mvpp2_mac_link_up()
6621 mvpp23_rx_fifo_fc_en(port->priv, port->id, tx_pause); in mvpp2_mac_link_up()
6624 mvpp2_port_enable(port); in mvpp2_mac_link_up()
6626 mvpp2_egress_enable(port); in mvpp2_mac_link_up()
6627 mvpp2_ingress_enable(port); in mvpp2_mac_link_up()
6628 netif_tx_wake_all_queues(port->dev); in mvpp2_mac_link_up()
6634 struct mvpp2_port *port = mvpp2_phylink_to_port(config); in mvpp2_mac_link_down() local
6639 val = readl(port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_mac_link_down()
6642 writel(val, port->base + MVPP22_XLG_CTRL0_REG); in mvpp2_mac_link_down()
6644 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_mac_link_down()
6647 writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG); in mvpp2_mac_link_down()
6651 netif_tx_stop_all_queues(port->dev); in mvpp2_mac_link_down()
6652 mvpp2_egress_disable(port); in mvpp2_mac_link_down()
6653 mvpp2_ingress_disable(port); in mvpp2_mac_link_down()
6655 mvpp2_port_disable(port); in mvpp2_mac_link_down()
6667 /* Work-around for ACPI */
6668 static void mvpp2_acpi_start(struct mvpp2_port *port) in mvpp2_acpi_start() argument
6675 .interface = port->phy_interface, in mvpp2_acpi_start()
6679 pcs = mvpp2_select_pcs(&port->phylink_config, port->phy_interface); in mvpp2_acpi_start()
6681 mvpp2_mac_prepare(&port->phylink_config, MLO_AN_INBAND, in mvpp2_acpi_start()
6682 port->phy_interface); in mvpp2_acpi_start()
6683 mvpp2_mac_config(&port->phylink_config, MLO_AN_INBAND, &state); in mvpp2_acpi_start()
6684 pcs->ops->pcs_config(pcs, PHYLINK_PCS_NEG_INBAND_ENABLED, in mvpp2_acpi_start()
6685 port->phy_interface, state.advertising, in mvpp2_acpi_start()
6687 mvpp2_mac_finish(&port->phylink_config, MLO_AN_INBAND, in mvpp2_acpi_start()
6688 port->phy_interface); in mvpp2_acpi_start()
6689 mvpp2_mac_link_up(&port->phylink_config, NULL, in mvpp2_acpi_start()
6690 MLO_AN_INBAND, port->phy_interface, in mvpp2_acpi_start()
6694 /* In order to ensure backward compatibility for ACPI, check if the port
6702 return (!fwnode_property_present(port_fwnode, "phy-handle") && in mvpp2_use_acpi_compat_mode()
6704 !fwnode_get_named_child_node(port_fwnode, "fixed-link")); in mvpp2_use_acpi_compat_mode()
6713 struct mvpp2_port *port; in mvpp2_port_probe() local
6729 dev_err(&pdev->dev, in mvpp2_port_probe()
6731 return -EINVAL; in mvpp2_port_probe()
6737 dev = alloc_etherdev_mqs(sizeof(*port), ntxqs, nrxqs); in mvpp2_port_probe()
6739 return -ENOMEM; in mvpp2_port_probe()
6743 dev_err(&pdev->dev, "incorrect phy mode\n"); in mvpp2_port_probe()
6749 * Rewrite 10GBASE-KR to 10GBASE-R for compatibility with existing DT. in mvpp2_port_probe()
6750 * Existing usage of 10GBASE-KR is not correct; no backplane in mvpp2_port_probe()
6752 * 10GBASE-KR. in mvpp2_port_probe()
6758 comphy = devm_of_phy_get(&pdev->dev, port_node, NULL); in mvpp2_port_probe()
6760 if (PTR_ERR(comphy) == -EPROBE_DEFER) { in mvpp2_port_probe()
6761 err = -EPROBE_DEFER; in mvpp2_port_probe()
6768 if (fwnode_property_read_u32(port_fwnode, "port-id", &id)) { in mvpp2_port_probe()
6769 err = -EINVAL; in mvpp2_port_probe()
6770 dev_err(&pdev->dev, "missing port-id value\n"); in mvpp2_port_probe()
6774 dev->tx_queue_len = MVPP2_MAX_TXD_MAX; in mvpp2_port_probe()
6775 dev->watchdog_timeo = 5 * HZ; in mvpp2_port_probe()
6776 dev->netdev_ops = &mvpp2_netdev_ops; in mvpp2_port_probe()
6777 dev->ethtool_ops = &mvpp2_eth_tool_ops; in mvpp2_port_probe()
6779 port = netdev_priv(dev); in mvpp2_port_probe()
6780 port->dev = dev; in mvpp2_port_probe()
6781 port->fwnode = port_fwnode; in mvpp2_port_probe()
6782 port->ntxqs = ntxqs; in mvpp2_port_probe()
6783 port->nrxqs = nrxqs; in mvpp2_port_probe()
6784 port->priv = priv; in mvpp2_port_probe()
6785 port->has_tx_irqs = has_tx_irqs; in mvpp2_port_probe()
6786 port->flags = flags; in mvpp2_port_probe()
6788 err = mvpp2_queue_vectors_init(port, port_node); in mvpp2_port_probe()
6793 port->port_irq = of_irq_get_byname(port_node, "link"); in mvpp2_port_probe()
6795 port->port_irq = fwnode_irq_get(port_fwnode, port->nqvecs + 1); in mvpp2_port_probe()
6796 if (port->port_irq == -EPROBE_DEFER) { in mvpp2_port_probe()
6797 err = -EPROBE_DEFER; in mvpp2_port_probe()
6800 if (port->port_irq <= 0) in mvpp2_port_probe()
6802 port->port_irq = 0; in mvpp2_port_probe()
6805 port->flags |= MVPP2_F_LOOPBACK; in mvpp2_port_probe()
6807 port->id = id; in mvpp2_port_probe()
6808 if (priv->hw_version == MVPP21) in mvpp2_port_probe()
6809 port->first_rxq = port->id * port->nrxqs; in mvpp2_port_probe()
6811 port->first_rxq = port->id * priv->max_port_rxqs; in mvpp2_port_probe()
6813 port->of_node = port_node; in mvpp2_port_probe()
6814 port->phy_interface = phy_mode; in mvpp2_port_probe()
6815 port->comphy = comphy; in mvpp2_port_probe()
6817 if (priv->hw_version == MVPP21) { in mvpp2_port_probe()
6818 port->base = devm_platform_ioremap_resource(pdev, 2 + id); in mvpp2_port_probe()
6819 if (IS_ERR(port->base)) { in mvpp2_port_probe()
6820 err = PTR_ERR(port->base); in mvpp2_port_probe()
6824 port->stats_base = port->priv->lms_base + in mvpp2_port_probe()
6826 port->gop_id * MVPP21_MIB_COUNTERS_PORT_SZ; in mvpp2_port_probe()
6828 if (fwnode_property_read_u32(port_fwnode, "gop-port-id", in mvpp2_port_probe()
6829 &port->gop_id)) { in mvpp2_port_probe()
6830 err = -EINVAL; in mvpp2_port_probe()
6831 dev_err(&pdev->dev, "missing gop-port-id value\n"); in mvpp2_port_probe()
6835 port->base = priv->iface_base + MVPP22_GMAC_BASE(port->gop_id); in mvpp2_port_probe()
6836 port->stats_base = port->priv->iface_base + in mvpp2_port_probe()
6838 port->gop_id * MVPP22_MIB_COUNTERS_PORT_SZ; in mvpp2_port_probe()
6843 if (priv->tai) in mvpp2_port_probe()
6844 port->hwtstamp = true; in mvpp2_port_probe()
6847 /* Alloc per-cpu and ethtool stats */ in mvpp2_port_probe()
6848 port->stats = netdev_alloc_pcpu_stats(struct mvpp2_pcpu_stats); in mvpp2_port_probe()
6849 if (!port->stats) { in mvpp2_port_probe()
6850 err = -ENOMEM; in mvpp2_port_probe()
6854 port->ethtool_stats = devm_kcalloc(&pdev->dev, in mvpp2_port_probe()
6857 if (!port->ethtool_stats) { in mvpp2_port_probe()
6858 err = -ENOMEM; in mvpp2_port_probe()
6862 mutex_init(&port->gather_stats_lock); in mvpp2_port_probe()
6863 INIT_DELAYED_WORK(&port->stats_work, mvpp2_gather_hw_statistics); in mvpp2_port_probe()
6869 port->tx_ring_size = MVPP2_MAX_TXD_DFLT; in mvpp2_port_probe()
6870 port->rx_ring_size = MVPP2_MAX_RXD_DFLT; in mvpp2_port_probe()
6871 SET_NETDEV_DEV(dev, &pdev->dev); in mvpp2_port_probe()
6873 err = mvpp2_port_init(port); in mvpp2_port_probe()
6875 dev_err(&pdev->dev, "failed to init port %d\n", id); in mvpp2_port_probe()
6879 mvpp2_port_periodic_xon_disable(port); in mvpp2_port_probe()
6881 mvpp2_mac_reset_assert(port); in mvpp2_port_probe()
6882 mvpp22_pcs_reset_assert(port); in mvpp2_port_probe()
6884 port->pcpu = alloc_percpu(struct mvpp2_port_pcpu); in mvpp2_port_probe()
6885 if (!port->pcpu) { in mvpp2_port_probe()
6886 err = -ENOMEM; in mvpp2_port_probe()
6890 if (!port->has_tx_irqs) { in mvpp2_port_probe()
6891 for (thread = 0; thread < priv->nthreads; thread++) { in mvpp2_port_probe()
6892 port_pcpu = per_cpu_ptr(port->pcpu, thread); in mvpp2_port_probe()
6894 hrtimer_init(&port_pcpu->tx_done_timer, CLOCK_MONOTONIC, in mvpp2_port_probe()
6896 port_pcpu->tx_done_timer.function = mvpp2_hr_timer_cb; in mvpp2_port_probe()
6897 port_pcpu->timer_scheduled = false; in mvpp2_port_probe()
6898 port_pcpu->dev = dev; in mvpp2_port_probe()
6904 dev->features = features | NETIF_F_RXCSUM; in mvpp2_port_probe()
6905 dev->hw_features |= features | NETIF_F_RXCSUM | NETIF_F_GRO | in mvpp2_port_probe()
6908 if (mvpp22_rss_is_supported(port)) { in mvpp2_port_probe()
6909 dev->hw_features |= NETIF_F_RXHASH; in mvpp2_port_probe()
6910 dev->features |= NETIF_F_NTUPLE; in mvpp2_port_probe()
6913 if (!port->priv->percpu_pools) in mvpp2_port_probe()
6914 mvpp2_set_hw_csum(port, port->pool_long->id); in mvpp2_port_probe()
6915 else if (port->ntxqs >= num_possible_cpus() * 2) in mvpp2_port_probe()
6916 dev->xdp_features = NETDEV_XDP_ACT_BASIC | in mvpp2_port_probe()
6920 dev->vlan_features |= features; in mvpp2_port_probe()
6923 dev->priv_flags |= IFF_UNICAST_FLT; in mvpp2_port_probe()
6925 /* MTU range: 68 - 9704 */ in mvpp2_port_probe()
6926 dev->min_mtu = ETH_MIN_MTU; in mvpp2_port_probe()
6927 /* 9704 == 9728 - 20 and rounding to 8 */ in mvpp2_port_probe()
6928 dev->max_mtu = MVPP2_BM_JUMBO_PKT_SIZE; in mvpp2_port_probe()
6929 dev->dev.of_node = port_node; in mvpp2_port_probe()
6931 port->pcs_gmac.ops = &mvpp2_phylink_gmac_pcs_ops; in mvpp2_port_probe()
6932 port->pcs_gmac.neg_mode = true; in mvpp2_port_probe()
6933 port->pcs_xlg.ops = &mvpp2_phylink_xlg_pcs_ops; in mvpp2_port_probe()
6934 port->pcs_xlg.neg_mode = true; in mvpp2_port_probe()
6937 port->phylink_config.dev = &dev->dev; in mvpp2_port_probe()
6938 port->phylink_config.type = PHYLINK_NETDEV; in mvpp2_port_probe()
6939 port->phylink_config.mac_capabilities = in mvpp2_port_probe()
6942 if (port->priv->global_tx_fc) in mvpp2_port_probe()
6943 port->phylink_config.mac_capabilities |= in mvpp2_port_probe()
6946 if (mvpp2_port_supports_xlg(port)) { in mvpp2_port_probe()
6952 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6954 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6956 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6959 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6962 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6965 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6969 port->phylink_config.mac_capabilities |= in mvpp2_port_probe()
6972 port->phylink_config.mac_capabilities |= in mvpp2_port_probe()
6975 port->phylink_config.mac_capabilities |= in mvpp2_port_probe()
6979 if (mvpp2_port_supports_rgmii(port)) in mvpp2_port_probe()
6980 phy_interface_set_rgmii(port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6987 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6989 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6991 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6993 /* No COMPHY, with only 2500BASE-X mode supported */ in mvpp2_port_probe()
6995 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
6998 /* No COMPHY, we can switch between 1000BASE-X and SGMII in mvpp2_port_probe()
7001 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
7003 port->phylink_config.supported_interfaces); in mvpp2_port_probe()
7006 phylink = phylink_create(&port->phylink_config, port_fwnode, in mvpp2_port_probe()
7012 port->phylink = phylink; in mvpp2_port_probe()
7014 dev_warn(&pdev->dev, "Use link irqs for port#%d. FW update required\n", port->id); in mvpp2_port_probe()
7015 port->phylink = NULL; in mvpp2_port_probe()
7018 /* Cycle the comphy to power it down, saving 270mW per port - in mvpp2_port_probe()
7022 if (port->comphy) { in mvpp2_port_probe()
7023 err = mvpp22_comphy_init(port, port->phy_interface); in mvpp2_port_probe()
7025 phy_power_off(port->comphy); in mvpp2_port_probe()
7030 dev_err(&pdev->dev, "failed to register netdev\n"); in mvpp2_port_probe()
7033 netdev_info(dev, "Using %s mac address %pM\n", mac_from, dev->dev_addr); in mvpp2_port_probe()
7035 priv->port_list[priv->port_count++] = port; in mvpp2_port_probe()
7040 if (port->phylink) in mvpp2_port_probe()
7041 phylink_destroy(port->phylink); in mvpp2_port_probe()
7043 free_percpu(port->pcpu); in mvpp2_port_probe()
7045 for (i = 0; i < port->ntxqs; i++) in mvpp2_port_probe()
7046 free_percpu(port->txqs[i]->pcpu); in mvpp2_port_probe()
7048 free_percpu(port->stats); in mvpp2_port_probe()
7050 if (port->port_irq) in mvpp2_port_probe()
7051 irq_dispose_mapping(port->port_irq); in mvpp2_port_probe()
7053 mvpp2_queue_vectors_deinit(port); in mvpp2_port_probe()
7060 static void mvpp2_port_remove(struct mvpp2_port *port) in mvpp2_port_remove() argument
7064 unregister_netdev(port->dev); in mvpp2_port_remove()
7065 if (port->phylink) in mvpp2_port_remove()
7066 phylink_destroy(port->phylink); in mvpp2_port_remove()
7067 free_percpu(port->pcpu); in mvpp2_port_remove()
7068 free_percpu(port->stats); in mvpp2_port_remove()
7069 for (i = 0; i < port->ntxqs; i++) in mvpp2_port_remove()
7070 free_percpu(port->txqs[i]->pcpu); in mvpp2_port_remove()
7071 mvpp2_queue_vectors_deinit(port); in mvpp2_port_remove()
7072 if (port->port_irq) in mvpp2_port_remove()
7073 irq_dispose_mapping(port->port_irq); in mvpp2_port_remove()
7074 free_netdev(port->dev); in mvpp2_port_remove()
7094 for (i = 0; i < dram->num_cs; i++) { in mvpp2_conf_mbus_windows()
7095 const struct mbus_dram_window *cs = dram->cs + i; in mvpp2_conf_mbus_windows()
7098 (cs->base & 0xffff0000) | (cs->mbus_attr << 8) | in mvpp2_conf_mbus_windows()
7099 dram->mbus_dram_target_id); in mvpp2_conf_mbus_windows()
7102 (cs->size - 1) & 0xffff0000); in mvpp2_conf_mbus_windows()
7113 int port; in mvpp2_rx_fifo_init() local
7115 for (port = 0; port < MVPP2_MAX_PORTS; port++) { in mvpp2_rx_fifo_init()
7116 mvpp2_write(priv, MVPP2_RX_DATA_FIFO_SIZE_REG(port), in mvpp2_rx_fifo_init()
7118 mvpp2_write(priv, MVPP2_RX_ATTR_FIFO_SIZE_REG(port), in mvpp2_rx_fifo_init()
7127 static void mvpp22_rx_fifo_set_hw(struct mvpp2 *priv, int port, int data_size) in mvpp22_rx_fifo_set_hw() argument
7131 mvpp2_write(priv, MVPP2_RX_DATA_FIFO_SIZE_REG(port), data_size); in mvpp22_rx_fifo_set_hw()
7132 mvpp2_write(priv, MVPP2_RX_ATTR_FIFO_SIZE_REG(port), attr_size); in mvpp22_rx_fifo_set_hw()
7136 * 4kB fixed space must be assigned for the loopback port.
7138 * Guarantee minimum 32kB for 10G port and 8kB for port 1, capable of 2.5G
7146 int port, size; in mvpp22_rx_fifo_init() local
7151 port_map = priv->port_map & ~BIT(MVPP2_LOOPBACK_PORT_INDEX); in mvpp22_rx_fifo_init()
7154 for_each_clear_bit(port, &port_map, MVPP2_LOOPBACK_PORT_INDEX) in mvpp22_rx_fifo_init()
7155 mvpp22_rx_fifo_set_hw(priv, port, 0); in mvpp22_rx_fifo_init()
7161 for_each_set_bit(port, &port_map, MVPP2_LOOPBACK_PORT_INDEX) { in mvpp22_rx_fifo_init()
7164 else if (port == 0) in mvpp22_rx_fifo_init()
7167 else if (port == 1) in mvpp22_rx_fifo_init()
7173 size_remainder -= size; in mvpp22_rx_fifo_init()
7174 remaining_ports_count--; in mvpp22_rx_fifo_init()
7176 mvpp22_rx_fifo_set_hw(priv, port, size); in mvpp22_rx_fifo_init()
7187 int port, val; in mvpp23_rx_fifo_fc_set_tresh() local
7189 /* Port 0: maximum speed -10Gb/s port in mvpp23_rx_fifo_fc_set_tresh()
7191 * Port 1: maximum speed -5Gb/s port in mvpp23_rx_fifo_fc_set_tresh()
7193 * Port 2: maximum speed -1Gb/s port in mvpp23_rx_fifo_fc_set_tresh()
7197 /* Without loopback port */ in mvpp23_rx_fifo_fc_set_tresh()
7198 for (port = 0; port < (MVPP2_MAX_PORTS - 1); port++) { in mvpp23_rx_fifo_fc_set_tresh()
7199 if (port == 0) { in mvpp23_rx_fifo_fc_set_tresh()
7203 mvpp2_write(priv, MVPP2_RX_FC_REG(port), val); in mvpp23_rx_fifo_fc_set_tresh()
7204 } else if (port == 1) { in mvpp23_rx_fifo_fc_set_tresh()
7208 mvpp2_write(priv, MVPP2_RX_FC_REG(port), val); in mvpp23_rx_fifo_fc_set_tresh()
7213 mvpp2_write(priv, MVPP2_RX_FC_REG(port), val); in mvpp23_rx_fifo_fc_set_tresh()
7219 void mvpp23_rx_fifo_fc_en(struct mvpp2 *priv, int port, bool en) in mvpp23_rx_fifo_fc_en() argument
7223 val = mvpp2_read(priv, MVPP2_RX_FC_REG(port)); in mvpp23_rx_fifo_fc_en()
7230 mvpp2_write(priv, MVPP2_RX_FC_REG(port), val); in mvpp23_rx_fifo_fc_en()
7233 static void mvpp22_tx_fifo_set_hw(struct mvpp2 *priv, int port, int size) in mvpp22_tx_fifo_set_hw() argument
7237 mvpp2_write(priv, MVPP22_TX_FIFO_SIZE_REG(port), size); in mvpp22_tx_fifo_set_hw()
7238 mvpp2_write(priv, MVPP22_TX_FIFO_THRESH_REG(port), threshold); in mvpp22_tx_fifo_set_hw()
7242 * 1kB fixed space must be assigned for the loopback port.
7245 * per single port).
7252 int port, size; in mvpp22_tx_fifo_init() local
7257 port_map = priv->port_map & ~BIT(MVPP2_LOOPBACK_PORT_INDEX); in mvpp22_tx_fifo_init()
7260 for_each_clear_bit(port, &port_map, MVPP2_LOOPBACK_PORT_INDEX) in mvpp22_tx_fifo_init()
7261 mvpp22_tx_fifo_set_hw(priv, port, 0); in mvpp22_tx_fifo_init()
7267 for_each_set_bit(port, &port_map, MVPP2_LOOPBACK_PORT_INDEX) { in mvpp22_tx_fifo_init()
7271 else if (port == 0) in mvpp22_tx_fifo_init()
7276 size_remainder -= size; in mvpp22_tx_fifo_init()
7277 remaining_ports_count--; in mvpp22_tx_fifo_init()
7279 mvpp22_tx_fifo_set_hw(priv, port, size); in mvpp22_tx_fifo_init()
7349 if (priv->hw_version >= MVPP22) in mvpp2_init()
7353 if (priv->hw_version == MVPP21) { in mvpp2_init()
7354 val = readl(priv->lms_base + MVPP2_PHY_AN_CFG0_REG); in mvpp2_init()
7356 writel(val, priv->lms_base + MVPP2_PHY_AN_CFG0_REG); in mvpp2_init()
7358 val = readl(priv->iface_base + MVPP22_SMI_MISC_CFG_REG); in mvpp2_init()
7360 writel(val, priv->iface_base + MVPP22_SMI_MISC_CFG_REG); in mvpp2_init()
7364 priv->aggr_txqs = devm_kcalloc(&pdev->dev, MVPP2_MAX_THREADS, in mvpp2_init()
7365 sizeof(*priv->aggr_txqs), in mvpp2_init()
7367 if (!priv->aggr_txqs) in mvpp2_init()
7368 return -ENOMEM; in mvpp2_init()
7371 priv->aggr_txqs[i].id = i; in mvpp2_init()
7372 priv->aggr_txqs[i].size = MVPP2_AGGR_TXQ_SIZE; in mvpp2_init()
7373 err = mvpp2_aggr_txq_init(pdev, &priv->aggr_txqs[i], i, priv); in mvpp2_init()
7379 if (priv->hw_version == MVPP21) { in mvpp2_init()
7384 if (priv->hw_version == MVPP23) in mvpp2_init()
7388 if (priv->hw_version == MVPP21) in mvpp2_init()
7390 priv->lms_base + MVPP2_MNG_EXTENDED_GLOBAL_CTRL_REG); in mvpp2_init()
7396 err = mvpp2_bm_init(&pdev->dev, priv); in mvpp2_init()
7419 if (has_acpi_companion(&pdev->dev)) in mvpp2_get_sram()
7420 dev_warn(&pdev->dev, "ACPI is too old, Flow control not supported\n"); in mvpp2_get_sram()
7422 dev_warn(&pdev->dev, "DT is too old, Flow control not supported\n"); in mvpp2_get_sram()
7426 base = devm_ioremap_resource(&pdev->dev, res); in mvpp2_get_sram()
7430 priv->cm3_base = base; in mvpp2_get_sram()
7436 struct fwnode_handle *fwnode = pdev->dev.fwnode; in mvpp2_probe()
7444 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL); in mvpp2_probe()
7446 return -ENOMEM; in mvpp2_probe()
7448 priv->hw_version = (unsigned long)device_get_match_data(&pdev->dev); in mvpp2_probe()
7453 if (priv->hw_version == MVPP21) in mvpp2_probe()
7460 if (priv->hw_version == MVPP21) { in mvpp2_probe()
7461 priv->lms_base = devm_platform_ioremap_resource(pdev, 1); in mvpp2_probe()
7462 if (IS_ERR(priv->lms_base)) in mvpp2_probe()
7463 return PTR_ERR(priv->lms_base); in mvpp2_probe()
7467 dev_err(&pdev->dev, "Invalid resource\n"); in mvpp2_probe()
7468 return -EINVAL; in mvpp2_probe()
7470 if (has_acpi_companion(&pdev->dev)) { in mvpp2_probe()
7472 * the ACPI, it can already appear as 'in-use' in mvpp2_probe()
7481 priv->iface_base = devm_ioremap_resource(&pdev->dev, res); in mvpp2_probe()
7482 if (IS_ERR(priv->iface_base)) in mvpp2_probe()
7483 return PTR_ERR(priv->iface_base); in mvpp2_probe()
7488 dev_warn(&pdev->dev, "Fail to alloc CM3 SRAM\n"); in mvpp2_probe()
7491 if (priv->cm3_base) in mvpp2_probe()
7492 priv->global_tx_fc = true; in mvpp2_probe()
7495 if (priv->hw_version >= MVPP22 && dev_of_node(&pdev->dev)) { in mvpp2_probe()
7496 priv->sysctrl_base = in mvpp2_probe()
7497 syscon_regmap_lookup_by_phandle(pdev->dev.of_node, in mvpp2_probe()
7498 "marvell,system-controller"); in mvpp2_probe()
7499 if (IS_ERR(priv->sysctrl_base)) in mvpp2_probe()
7505 priv->sysctrl_base = NULL; in mvpp2_probe()
7508 if (priv->hw_version >= MVPP22 && in mvpp2_probe()
7510 priv->percpu_pools = 1; in mvpp2_probe()
7515 priv->nthreads = min_t(unsigned int, num_present_cpus(), in mvpp2_probe()
7518 shared = num_present_cpus() - priv->nthreads; in mvpp2_probe()
7520 bitmap_set(&priv->lock_map, 0, in mvpp2_probe()
7526 addr_space_sz = (priv->hw_version == MVPP21 ? in mvpp2_probe()
7528 priv->swth_base[i] = base + i * addr_space_sz; in mvpp2_probe()
7531 if (priv->hw_version == MVPP21) in mvpp2_probe()
7532 priv->max_port_rxqs = 8; in mvpp2_probe()
7534 priv->max_port_rxqs = 32; in mvpp2_probe()
7536 if (dev_of_node(&pdev->dev)) { in mvpp2_probe()
7537 priv->pp_clk = devm_clk_get(&pdev->dev, "pp_clk"); in mvpp2_probe()
7538 if (IS_ERR(priv->pp_clk)) in mvpp2_probe()
7539 return PTR_ERR(priv->pp_clk); in mvpp2_probe()
7540 err = clk_prepare_enable(priv->pp_clk); in mvpp2_probe()
7544 priv->gop_clk = devm_clk_get(&pdev->dev, "gop_clk"); in mvpp2_probe()
7545 if (IS_ERR(priv->gop_clk)) { in mvpp2_probe()
7546 err = PTR_ERR(priv->gop_clk); in mvpp2_probe()
7549 err = clk_prepare_enable(priv->gop_clk); in mvpp2_probe()
7553 if (priv->hw_version >= MVPP22) { in mvpp2_probe()
7554 priv->mg_clk = devm_clk_get(&pdev->dev, "mg_clk"); in mvpp2_probe()
7555 if (IS_ERR(priv->mg_clk)) { in mvpp2_probe()
7556 err = PTR_ERR(priv->mg_clk); in mvpp2_probe()
7560 err = clk_prepare_enable(priv->mg_clk); in mvpp2_probe()
7564 priv->mg_core_clk = devm_clk_get_optional(&pdev->dev, "mg_core_clk"); in mvpp2_probe()
7565 if (IS_ERR(priv->mg_core_clk)) { in mvpp2_probe()
7566 err = PTR_ERR(priv->mg_core_clk); in mvpp2_probe()
7570 err = clk_prepare_enable(priv->mg_core_clk); in mvpp2_probe()
7575 priv->axi_clk = devm_clk_get_optional(&pdev->dev, "axi_clk"); in mvpp2_probe()
7576 if (IS_ERR(priv->axi_clk)) { in mvpp2_probe()
7577 err = PTR_ERR(priv->axi_clk); in mvpp2_probe()
7581 err = clk_prepare_enable(priv->axi_clk); in mvpp2_probe()
7586 priv->tclk = clk_get_rate(priv->pp_clk); in mvpp2_probe()
7588 err = device_property_read_u32(&pdev->dev, "clock-frequency", &priv->tclk); in mvpp2_probe()
7590 dev_err(&pdev->dev, "missing clock-frequency value\n"); in mvpp2_probe()
7595 if (priv->hw_version >= MVPP22) { in mvpp2_probe()
7596 err = dma_set_mask(&pdev->dev, MVPP2_DESC_DMA_MASK); in mvpp2_probe()
7604 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); in mvpp2_probe()
7609 /* Map DTS-active ports. Should be done before FIFO mvpp2_init */ in mvpp2_probe()
7611 if (!fwnode_property_read_u32(port_fwnode, "port-id", &i)) in mvpp2_probe()
7612 priv->port_map |= BIT(i); in mvpp2_probe()
7616 priv->hw_version = MVPP23; in mvpp2_probe()
7619 spin_lock_init(&priv->mss_spinlock); in mvpp2_probe()
7624 dev_err(&pdev->dev, "failed to initialize controller\n"); in mvpp2_probe()
7628 err = mvpp22_tai_probe(&pdev->dev, priv); in mvpp2_probe()
7639 if (priv->port_count == 0) { in mvpp2_probe()
7640 dev_err(&pdev->dev, "no ports enabled\n"); in mvpp2_probe()
7641 err = -ENODEV; in mvpp2_probe()
7646 * packets counters) are 32-bit registers and could overflow quite in mvpp2_probe()
7648 * smallest packets (64B) will overflow a 32-bit counter in less than in mvpp2_probe()
7649 * 30 seconds. Then, use a workqueue to fill 64-bit counters. in mvpp2_probe()
7651 snprintf(priv->queue_name, sizeof(priv->queue_name), in mvpp2_probe()
7652 "stats-wq-%s%s", netdev_name(priv->port_list[0]->dev), in mvpp2_probe()
7653 priv->port_count > 1 ? "+" : ""); in mvpp2_probe()
7654 priv->stats_queue = create_singlethread_workqueue(priv->queue_name); in mvpp2_probe()
7655 if (!priv->stats_queue) { in mvpp2_probe()
7656 err = -ENOMEM; in mvpp2_probe()
7660 if (priv->global_tx_fc && priv->hw_version >= MVPP22) { in mvpp2_probe()
7663 …dev_warn(&pdev->dev, "Minimum of CM3 firmware 18.09 and chip revision B0 required for flow control… in mvpp2_probe()
7666 mvpp2_dbgfs_init(priv, pdev->name); in mvpp2_probe()
7676 if (priv->port_list[i]) in mvpp2_probe()
7677 mvpp2_port_remove(priv->port_list[i]); in mvpp2_probe()
7681 clk_disable_unprepare(priv->axi_clk); in mvpp2_probe()
7683 clk_disable_unprepare(priv->mg_core_clk); in mvpp2_probe()
7685 clk_disable_unprepare(priv->mg_clk); in mvpp2_probe()
7687 clk_disable_unprepare(priv->gop_clk); in mvpp2_probe()
7689 clk_disable_unprepare(priv->pp_clk); in mvpp2_probe()
7696 struct fwnode_handle *fwnode = pdev->dev.fwnode; in mvpp2_remove()
7703 if (priv->port_list[i]) { in mvpp2_remove()
7704 mutex_destroy(&priv->port_list[i]->gather_stats_lock); in mvpp2_remove()
7705 mvpp2_port_remove(priv->port_list[i]); in mvpp2_remove()
7710 destroy_workqueue(priv->stats_queue); in mvpp2_remove()
7712 if (priv->percpu_pools) in mvpp2_remove()
7716 struct mvpp2_bm_pool *bm_pool = &priv->bm_pools[i]; in mvpp2_remove()
7718 mvpp2_bm_pool_destroy(&pdev->dev, priv, bm_pool); in mvpp2_remove()
7722 struct mvpp2_tx_queue *aggr_txq = &priv->aggr_txqs[i]; in mvpp2_remove()
7724 dma_free_coherent(&pdev->dev, in mvpp2_remove()
7726 aggr_txq->descs, in mvpp2_remove()
7727 aggr_txq->descs_dma); in mvpp2_remove()
7733 clk_disable_unprepare(priv->axi_clk); in mvpp2_remove()
7734 clk_disable_unprepare(priv->mg_core_clk); in mvpp2_remove()
7735 clk_disable_unprepare(priv->mg_clk); in mvpp2_remove()
7736 clk_disable_unprepare(priv->pp_clk); in mvpp2_remove()
7737 clk_disable_unprepare(priv->gop_clk); in mvpp2_remove()
7744 .compatible = "marvell,armada-375-pp2",
7748 .compatible = "marvell,armada-7k-pp22",
7786 MODULE_DESCRIPTION("Marvell PPv2 Ethernet Driver - www.marvell.com");