Lines Matching +full:num +full:- +full:rxq
1 // SPDX-License-Identifier: GPL-2.0+
17 * Copyright (c) 2001-2005 Greg Ungerer (gerg@snapgear.com)
20 * Copyright (c) 2004-2006 Macq Electronique SA.
22 * Copyright (C) 2010-2011 Freescale Semiconductor, Inc.
190 .name = "imx25-fec",
193 .name = "imx27-fec",
196 .name = "imx28-fec",
199 .name = "imx6q-fec",
202 .name = "mvf600-fec",
205 .name = "imx6sx-fec",
208 .name = "imx6ul-fec",
211 .name = "imx8mq-fec",
214 .name = "imx8qm-fec",
217 .name = "s32v234-fec",
239 { .compatible = "fsl,imx25-fec", .data = &fec_devtype[IMX25_FEC], },
240 { .compatible = "fsl,imx27-fec", .data = &fec_devtype[IMX27_FEC], },
241 { .compatible = "fsl,imx28-fec", .data = &fec_devtype[IMX28_FEC], },
242 { .compatible = "fsl,imx6q-fec", .data = &fec_devtype[IMX6Q_FEC], },
243 { .compatible = "fsl,mvf600-fec", .data = &fec_devtype[MVF600_FEC], },
244 { .compatible = "fsl,imx6sx-fec", .data = &fec_devtype[IMX6SX_FEC], },
245 { .compatible = "fsl,imx6ul-fec", .data = &fec_devtype[IMX6UL_FEC], },
246 { .compatible = "fsl,imx8mq-fec", .data = &fec_devtype[IMX8MQ_FEC], },
247 { .compatible = "fsl,imx8qm-fec", .data = &fec_devtype[IMX8QM_FEC], },
248 { .compatible = "fsl,s32v234-fec", .data = &fec_devtype[S32V234_FEC], },
260 * if this is non-zero then assume it is the address to get MAC from.
282 #define PKT_MAXBUF_SIZE (round_down(2048 - 64, 64))
353 ((addr >= txq->tso_hdrs_dma) && \
354 (addr < txq->tso_hdrs_dma + txq->bd.ring_size * TSO_HEADER_SIZE))
361 return (bdp >= bd->last) ? bd->base in fec_enet_get_nextdesc()
362 : (struct bufdesc *)(((void *)bdp) + bd->dsize); in fec_enet_get_nextdesc()
368 return (bdp <= bd->base) ? bd->last in fec_enet_get_prevdesc()
369 : (struct bufdesc *)(((void *)bdp) - bd->dsize); in fec_enet_get_prevdesc()
375 return ((const char *)bdp - (const char *)bd->base) >> bd->dsize_log2; in fec_enet_get_bd_index()
382 entries = (((const char *)txq->dirty_tx - in fec_enet_get_free_txdesc_num()
383 (const char *)txq->bd.cur) >> txq->bd.dsize_log2) - 1; in fec_enet_get_free_txdesc_num()
385 return entries >= 0 ? entries : entries + txq->bd.ring_size; in fec_enet_get_free_txdesc_num()
407 txq = fep->tx_queue[0]; in fec_dump()
408 bdp = txq->bd.base; in fec_dump()
413 bdp == txq->bd.cur ? 'S' : ' ', in fec_dump()
414 bdp == txq->dirty_tx ? 'H' : ' ', in fec_dump()
415 fec16_to_cpu(bdp->cbd_sc), in fec_dump()
416 fec32_to_cpu(bdp->cbd_bufaddr), in fec_dump()
417 fec16_to_cpu(bdp->cbd_datlen), in fec_dump()
418 txq->tx_buf[index].buf_p); in fec_dump()
419 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_dump()
421 } while (bdp != txq->bd.base); in fec_dump()
426 return skb->protocol == htons(ETH_P_IP) && ip_hdr(skb)->version == 4; in is_ipv4_pkt()
433 if (skb->ip_summed != CHECKSUM_PARTIAL) in fec_enet_clear_csum()
437 return -1; in fec_enet_clear_csum()
440 ip_hdr(skb)->check = 0; in fec_enet_clear_csum()
441 *(__sum16 *)(skb->head + skb->csum_start + skb->csum_offset) = 0; in fec_enet_clear_csum()
448 struct fec_enet_priv_rx_q *rxq, int size) in fec_enet_create_page_pool() argument
450 struct bpf_prog *xdp_prog = READ_ONCE(fep->xdp_prog); in fec_enet_create_page_pool()
455 .nid = dev_to_node(&fep->pdev->dev), in fec_enet_create_page_pool()
456 .dev = &fep->pdev->dev, in fec_enet_create_page_pool()
463 rxq->page_pool = page_pool_create(&pp_params); in fec_enet_create_page_pool()
464 if (IS_ERR(rxq->page_pool)) { in fec_enet_create_page_pool()
465 err = PTR_ERR(rxq->page_pool); in fec_enet_create_page_pool()
466 rxq->page_pool = NULL; in fec_enet_create_page_pool()
470 err = xdp_rxq_info_reg(&rxq->xdp_rxq, fep->netdev, rxq->id, 0); in fec_enet_create_page_pool()
474 err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq, MEM_TYPE_PAGE_POOL, in fec_enet_create_page_pool()
475 rxq->page_pool); in fec_enet_create_page_pool()
482 xdp_rxq_info_unreg(&rxq->xdp_rxq); in fec_enet_create_page_pool()
484 page_pool_destroy(rxq->page_pool); in fec_enet_create_page_pool()
485 rxq->page_pool = NULL; in fec_enet_create_page_pool()
495 struct bufdesc *bdp = txq->bd.cur; in fec_enet_txq_submit_frag_skb()
497 int nr_frags = skb_shinfo(skb)->nr_frags; in fec_enet_txq_submit_frag_skb()
508 this_frag = &skb_shinfo(skb)->frags[frag]; in fec_enet_txq_submit_frag_skb()
509 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_enet_txq_submit_frag_skb()
512 status = fec16_to_cpu(bdp->cbd_sc); in fec_enet_txq_submit_frag_skb()
515 frag_len = skb_frag_size(&skb_shinfo(skb)->frags[frag]); in fec_enet_txq_submit_frag_skb()
518 if (frag == nr_frags - 1) { in fec_enet_txq_submit_frag_skb()
520 if (fep->bufdesc_ex) { in fec_enet_txq_submit_frag_skb()
522 if (unlikely(skb_shinfo(skb)->tx_flags & in fec_enet_txq_submit_frag_skb()
523 SKBTX_HW_TSTAMP && fep->hwts_tx_en)) in fec_enet_txq_submit_frag_skb()
528 if (fep->bufdesc_ex) { in fec_enet_txq_submit_frag_skb()
529 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_submit_frag_skb()
530 estatus |= FEC_TX_BD_FTYPE(txq->bd.qid); in fec_enet_txq_submit_frag_skb()
531 if (skb->ip_summed == CHECKSUM_PARTIAL) in fec_enet_txq_submit_frag_skb()
534 ebdp->cbd_bdu = 0; in fec_enet_txq_submit_frag_skb()
535 ebdp->cbd_esc = cpu_to_fec32(estatus); in fec_enet_txq_submit_frag_skb()
540 index = fec_enet_get_bd_index(bdp, &txq->bd); in fec_enet_txq_submit_frag_skb()
541 if (((unsigned long) bufaddr) & fep->tx_align || in fec_enet_txq_submit_frag_skb()
542 fep->quirks & FEC_QUIRK_SWAP_FRAME) { in fec_enet_txq_submit_frag_skb()
543 memcpy(txq->tx_bounce[index], bufaddr, frag_len); in fec_enet_txq_submit_frag_skb()
544 bufaddr = txq->tx_bounce[index]; in fec_enet_txq_submit_frag_skb()
546 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_txq_submit_frag_skb()
550 addr = dma_map_single(&fep->pdev->dev, bufaddr, frag_len, in fec_enet_txq_submit_frag_skb()
552 if (dma_mapping_error(&fep->pdev->dev, addr)) { in fec_enet_txq_submit_frag_skb()
558 bdp->cbd_bufaddr = cpu_to_fec32(addr); in fec_enet_txq_submit_frag_skb()
559 bdp->cbd_datlen = cpu_to_fec16(frag_len); in fec_enet_txq_submit_frag_skb()
564 bdp->cbd_sc = cpu_to_fec16(status); in fec_enet_txq_submit_frag_skb()
569 bdp = txq->bd.cur; in fec_enet_txq_submit_frag_skb()
571 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_enet_txq_submit_frag_skb()
572 dma_unmap_single(&fep->pdev->dev, fec32_to_cpu(bdp->cbd_bufaddr), in fec_enet_txq_submit_frag_skb()
573 fec16_to_cpu(bdp->cbd_datlen), DMA_TO_DEVICE); in fec_enet_txq_submit_frag_skb()
575 return ERR_PTR(-ENOMEM); in fec_enet_txq_submit_frag_skb()
582 int nr_frags = skb_shinfo(skb)->nr_frags; in fec_enet_txq_submit_skb()
600 /* Protocol checksum off-load for TCP and UDP. */ in fec_enet_txq_submit_skb()
607 bdp = txq->bd.cur; in fec_enet_txq_submit_skb()
609 status = fec16_to_cpu(bdp->cbd_sc); in fec_enet_txq_submit_skb()
613 bufaddr = skb->data; in fec_enet_txq_submit_skb()
616 index = fec_enet_get_bd_index(bdp, &txq->bd); in fec_enet_txq_submit_skb()
617 if (((unsigned long) bufaddr) & fep->tx_align || in fec_enet_txq_submit_skb()
618 fep->quirks & FEC_QUIRK_SWAP_FRAME) { in fec_enet_txq_submit_skb()
619 memcpy(txq->tx_bounce[index], skb->data, buflen); in fec_enet_txq_submit_skb()
620 bufaddr = txq->tx_bounce[index]; in fec_enet_txq_submit_skb()
622 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_txq_submit_skb()
627 addr = dma_map_single(&fep->pdev->dev, bufaddr, buflen, DMA_TO_DEVICE); in fec_enet_txq_submit_skb()
628 if (dma_mapping_error(&fep->pdev->dev, addr)) { in fec_enet_txq_submit_skb()
638 dma_unmap_single(&fep->pdev->dev, addr, in fec_enet_txq_submit_skb()
645 if (fep->bufdesc_ex) { in fec_enet_txq_submit_skb()
647 if (unlikely(skb_shinfo(skb)->tx_flags & in fec_enet_txq_submit_skb()
648 SKBTX_HW_TSTAMP && fep->hwts_tx_en)) in fec_enet_txq_submit_skb()
652 bdp->cbd_bufaddr = cpu_to_fec32(addr); in fec_enet_txq_submit_skb()
653 bdp->cbd_datlen = cpu_to_fec16(buflen); in fec_enet_txq_submit_skb()
655 if (fep->bufdesc_ex) { in fec_enet_txq_submit_skb()
659 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP && in fec_enet_txq_submit_skb()
660 fep->hwts_tx_en)) in fec_enet_txq_submit_skb()
661 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; in fec_enet_txq_submit_skb()
663 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_submit_skb()
664 estatus |= FEC_TX_BD_FTYPE(txq->bd.qid); in fec_enet_txq_submit_skb()
666 if (skb->ip_summed == CHECKSUM_PARTIAL) in fec_enet_txq_submit_skb()
669 ebdp->cbd_bdu = 0; in fec_enet_txq_submit_skb()
670 ebdp->cbd_esc = cpu_to_fec32(estatus); in fec_enet_txq_submit_skb()
673 index = fec_enet_get_bd_index(last_bdp, &txq->bd); in fec_enet_txq_submit_skb()
675 txq->tx_buf[index].buf_p = skb; in fec_enet_txq_submit_skb()
686 bdp->cbd_sc = cpu_to_fec16(status); in fec_enet_txq_submit_skb()
689 bdp = fec_enet_get_nextdesc(last_bdp, &txq->bd); in fec_enet_txq_submit_skb()
693 /* Make sure the update to bdp is performed before txq->bd.cur. */ in fec_enet_txq_submit_skb()
695 txq->bd.cur = bdp; in fec_enet_txq_submit_skb()
698 writel(0, txq->bd.reg_desc_active); in fec_enet_txq_submit_skb()
715 status = fec16_to_cpu(bdp->cbd_sc); in fec_enet_txq_put_data_tso()
720 if (((unsigned long) data) & fep->tx_align || in fec_enet_txq_put_data_tso()
721 fep->quirks & FEC_QUIRK_SWAP_FRAME) { in fec_enet_txq_put_data_tso()
722 memcpy(txq->tx_bounce[index], data, size); in fec_enet_txq_put_data_tso()
723 data = txq->tx_bounce[index]; in fec_enet_txq_put_data_tso()
725 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_txq_put_data_tso()
729 addr = dma_map_single(&fep->pdev->dev, data, size, DMA_TO_DEVICE); in fec_enet_txq_put_data_tso()
730 if (dma_mapping_error(&fep->pdev->dev, addr)) { in fec_enet_txq_put_data_tso()
737 bdp->cbd_datlen = cpu_to_fec16(size); in fec_enet_txq_put_data_tso()
738 bdp->cbd_bufaddr = cpu_to_fec32(addr); in fec_enet_txq_put_data_tso()
740 if (fep->bufdesc_ex) { in fec_enet_txq_put_data_tso()
741 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_put_data_tso()
742 estatus |= FEC_TX_BD_FTYPE(txq->bd.qid); in fec_enet_txq_put_data_tso()
743 if (skb->ip_summed == CHECKSUM_PARTIAL) in fec_enet_txq_put_data_tso()
745 ebdp->cbd_bdu = 0; in fec_enet_txq_put_data_tso()
746 ebdp->cbd_esc = cpu_to_fec32(estatus); in fec_enet_txq_put_data_tso()
754 if (fep->bufdesc_ex) in fec_enet_txq_put_data_tso()
755 ebdp->cbd_esc |= cpu_to_fec32(BD_ENET_TX_INT); in fec_enet_txq_put_data_tso()
758 bdp->cbd_sc = cpu_to_fec16(status); in fec_enet_txq_put_data_tso()
776 status = fec16_to_cpu(bdp->cbd_sc); in fec_enet_txq_put_hdr_tso()
780 bufaddr = txq->tso_hdrs + index * TSO_HEADER_SIZE; in fec_enet_txq_put_hdr_tso()
781 dmabuf = txq->tso_hdrs_dma + index * TSO_HEADER_SIZE; in fec_enet_txq_put_hdr_tso()
782 if (((unsigned long)bufaddr) & fep->tx_align || in fec_enet_txq_put_hdr_tso()
783 fep->quirks & FEC_QUIRK_SWAP_FRAME) { in fec_enet_txq_put_hdr_tso()
784 memcpy(txq->tx_bounce[index], skb->data, hdr_len); in fec_enet_txq_put_hdr_tso()
785 bufaddr = txq->tx_bounce[index]; in fec_enet_txq_put_hdr_tso()
787 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_txq_put_hdr_tso()
790 dmabuf = dma_map_single(&fep->pdev->dev, bufaddr, in fec_enet_txq_put_hdr_tso()
792 if (dma_mapping_error(&fep->pdev->dev, dmabuf)) { in fec_enet_txq_put_hdr_tso()
800 bdp->cbd_bufaddr = cpu_to_fec32(dmabuf); in fec_enet_txq_put_hdr_tso()
801 bdp->cbd_datlen = cpu_to_fec16(hdr_len); in fec_enet_txq_put_hdr_tso()
803 if (fep->bufdesc_ex) { in fec_enet_txq_put_hdr_tso()
804 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_put_hdr_tso()
805 estatus |= FEC_TX_BD_FTYPE(txq->bd.qid); in fec_enet_txq_put_hdr_tso()
806 if (skb->ip_summed == CHECKSUM_PARTIAL) in fec_enet_txq_put_hdr_tso()
808 ebdp->cbd_bdu = 0; in fec_enet_txq_put_hdr_tso()
809 ebdp->cbd_esc = cpu_to_fec32(estatus); in fec_enet_txq_put_hdr_tso()
812 bdp->cbd_sc = cpu_to_fec16(status); in fec_enet_txq_put_hdr_tso()
823 struct bufdesc *bdp = txq->bd.cur; in fec_enet_txq_submit_tso()
837 /* Protocol checksum off-load for TCP and UDP. */ in fec_enet_txq_submit_tso()
846 total_len = skb->len - hdr_len; in fec_enet_txq_submit_tso()
850 index = fec_enet_get_bd_index(bdp, &txq->bd); in fec_enet_txq_submit_tso()
851 data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len); in fec_enet_txq_submit_tso()
852 total_len -= data_left; in fec_enet_txq_submit_tso()
855 hdr = txq->tso_hdrs + index * TSO_HEADER_SIZE; in fec_enet_txq_submit_tso()
865 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_enet_txq_submit_tso()
866 index = fec_enet_get_bd_index(bdp, &txq->bd); in fec_enet_txq_submit_tso()
875 data_left -= size; in fec_enet_txq_submit_tso()
879 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_enet_txq_submit_tso()
883 txq->tx_buf[index].buf_p = skb; in fec_enet_txq_submit_tso()
886 txq->bd.cur = bdp; in fec_enet_txq_submit_tso()
889 if (!(fep->quirks & FEC_QUIRK_ERR007885) || in fec_enet_txq_submit_tso()
890 !readl(txq->bd.reg_desc_active) || in fec_enet_txq_submit_tso()
891 !readl(txq->bd.reg_desc_active) || in fec_enet_txq_submit_tso()
892 !readl(txq->bd.reg_desc_active) || in fec_enet_txq_submit_tso()
893 !readl(txq->bd.reg_desc_active)) in fec_enet_txq_submit_tso()
894 writel(0, txq->bd.reg_desc_active); in fec_enet_txq_submit_tso()
900 tmp_bdp = txq->bd.cur; in fec_enet_txq_submit_tso()
904 if (tmp_bdp->cbd_bufaddr && in fec_enet_txq_submit_tso()
905 !IS_TSO_HEADER(txq, fec32_to_cpu(tmp_bdp->cbd_bufaddr))) in fec_enet_txq_submit_tso()
906 dma_unmap_single(&fep->pdev->dev, in fec_enet_txq_submit_tso()
907 fec32_to_cpu(tmp_bdp->cbd_bufaddr), in fec_enet_txq_submit_tso()
908 fec16_to_cpu(tmp_bdp->cbd_datlen), in fec_enet_txq_submit_tso()
912 tmp_bdp->cbd_sc = 0; in fec_enet_txq_submit_tso()
913 tmp_bdp->cbd_datlen = 0; in fec_enet_txq_submit_tso()
914 tmp_bdp->cbd_bufaddr = 0; in fec_enet_txq_submit_tso()
917 if (fep->bufdesc_ex) { in fec_enet_txq_submit_tso()
919 ebdp->cbd_esc = 0; in fec_enet_txq_submit_tso()
922 tmp_bdp = fec_enet_get_nextdesc(tmp_bdp, &txq->bd); in fec_enet_txq_submit_tso()
941 txq = fep->tx_queue[queue]; in fec_enet_start_xmit()
952 if (entries_free <= txq->tx_stop_threshold) in fec_enet_start_xmit()
964 struct fec_enet_priv_rx_q *rxq; in fec_enet_bd_init() local
969 for (q = 0; q < fep->num_rx_queues; q++) { in fec_enet_bd_init()
971 rxq = fep->rx_queue[q]; in fec_enet_bd_init()
972 bdp = rxq->bd.base; in fec_enet_bd_init()
974 for (i = 0; i < rxq->bd.ring_size; i++) { in fec_enet_bd_init()
977 if (bdp->cbd_bufaddr) in fec_enet_bd_init()
978 bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY); in fec_enet_bd_init()
980 bdp->cbd_sc = cpu_to_fec16(0); in fec_enet_bd_init()
981 bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); in fec_enet_bd_init()
985 bdp = fec_enet_get_prevdesc(bdp, &rxq->bd); in fec_enet_bd_init()
986 bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); in fec_enet_bd_init()
988 rxq->bd.cur = rxq->bd.base; in fec_enet_bd_init()
991 for (q = 0; q < fep->num_tx_queues; q++) { in fec_enet_bd_init()
993 txq = fep->tx_queue[q]; in fec_enet_bd_init()
994 bdp = txq->bd.base; in fec_enet_bd_init()
995 txq->bd.cur = bdp; in fec_enet_bd_init()
997 for (i = 0; i < txq->bd.ring_size; i++) { in fec_enet_bd_init()
999 bdp->cbd_sc = cpu_to_fec16(0); in fec_enet_bd_init()
1000 if (txq->tx_buf[i].type == FEC_TXBUF_T_SKB) { in fec_enet_bd_init()
1001 if (bdp->cbd_bufaddr && in fec_enet_bd_init()
1002 !IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr))) in fec_enet_bd_init()
1003 dma_unmap_single(&fep->pdev->dev, in fec_enet_bd_init()
1004 fec32_to_cpu(bdp->cbd_bufaddr), in fec_enet_bd_init()
1005 fec16_to_cpu(bdp->cbd_datlen), in fec_enet_bd_init()
1007 if (txq->tx_buf[i].buf_p) in fec_enet_bd_init()
1008 dev_kfree_skb_any(txq->tx_buf[i].buf_p); in fec_enet_bd_init()
1009 } else if (txq->tx_buf[i].type == FEC_TXBUF_T_XDP_NDO) { in fec_enet_bd_init()
1010 if (bdp->cbd_bufaddr) in fec_enet_bd_init()
1011 dma_unmap_single(&fep->pdev->dev, in fec_enet_bd_init()
1012 fec32_to_cpu(bdp->cbd_bufaddr), in fec_enet_bd_init()
1013 fec16_to_cpu(bdp->cbd_datlen), in fec_enet_bd_init()
1016 if (txq->tx_buf[i].buf_p) in fec_enet_bd_init()
1017 xdp_return_frame(txq->tx_buf[i].buf_p); in fec_enet_bd_init()
1019 struct page *page = txq->tx_buf[i].buf_p; in fec_enet_bd_init()
1022 page_pool_put_page(page->pp, page, 0, false); in fec_enet_bd_init()
1025 txq->tx_buf[i].buf_p = NULL; in fec_enet_bd_init()
1027 txq->tx_buf[i].type = FEC_TXBUF_T_SKB; in fec_enet_bd_init()
1028 bdp->cbd_bufaddr = cpu_to_fec32(0); in fec_enet_bd_init()
1029 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_enet_bd_init()
1033 bdp = fec_enet_get_prevdesc(bdp, &txq->bd); in fec_enet_bd_init()
1034 bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); in fec_enet_bd_init()
1035 txq->dirty_tx = bdp; in fec_enet_bd_init()
1044 for (i = 0; i < fep->num_rx_queues; i++) in fec_enet_active_rxring()
1045 writel(0, fep->rx_queue[i]->bd.reg_desc_active); in fec_enet_active_rxring()
1052 struct fec_enet_priv_rx_q *rxq; in fec_enet_enable_ring() local
1055 for (i = 0; i < fep->num_rx_queues; i++) { in fec_enet_enable_ring()
1056 rxq = fep->rx_queue[i]; in fec_enet_enable_ring()
1057 writel(rxq->bd.dma, fep->hwp + FEC_R_DES_START(i)); in fec_enet_enable_ring()
1058 writel(PKT_MAXBUF_SIZE, fep->hwp + FEC_R_BUFF_SIZE(i)); in fec_enet_enable_ring()
1063 fep->hwp + FEC_RCMR(i)); in fec_enet_enable_ring()
1066 for (i = 0; i < fep->num_tx_queues; i++) { in fec_enet_enable_ring()
1067 txq = fep->tx_queue[i]; in fec_enet_enable_ring()
1068 writel(txq->bd.dma, fep->hwp + FEC_X_DES_START(i)); in fec_enet_enable_ring()
1073 fep->hwp + FEC_DMA_CFG(i)); in fec_enet_enable_ring()
1090 if (fep->bufdesc_ex) in fec_restart()
1097 if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES || in fec_restart()
1098 ((fep->quirks & FEC_QUIRK_NO_HARD_RESET) && fep->link)) { in fec_restart()
1099 writel(0, fep->hwp + FEC_ECNTRL); in fec_restart()
1101 writel(1, fep->hwp + FEC_ECNTRL); in fec_restart()
1106 * enet-mac reset will reset mac address registers too, in fec_restart()
1109 memcpy(&temp_mac, ndev->dev_addr, ETH_ALEN); in fec_restart()
1111 fep->hwp + FEC_ADDR_LOW); in fec_restart()
1113 fep->hwp + FEC_ADDR_HIGH); in fec_restart()
1116 writel((0xffffffff & ~FEC_ENET_MII), fep->hwp + FEC_IEVENT); in fec_restart()
1123 if (fep->full_duplex == DUPLEX_FULL) { in fec_restart()
1125 writel(0x04, fep->hwp + FEC_X_CNTRL); in fec_restart()
1129 writel(0x0, fep->hwp + FEC_X_CNTRL); in fec_restart()
1133 writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); in fec_restart()
1136 if (fep->quirks & FEC_QUIRK_HAS_RACC) { in fec_restart()
1137 u32 val = readl(fep->hwp + FEC_RACC); in fec_restart()
1141 if (fep->csum_flags & FLAG_RX_CSUM_ENABLED) in fec_restart()
1146 writel(val, fep->hwp + FEC_RACC); in fec_restart()
1147 writel(PKT_MAXBUF_SIZE, fep->hwp + FEC_FTRL); in fec_restart()
1153 * differently on enet-mac. in fec_restart()
1155 if (fep->quirks & FEC_QUIRK_ENET_MAC) { in fec_restart()
1160 if (fep->phy_interface == PHY_INTERFACE_MODE_RGMII || in fec_restart()
1161 fep->phy_interface == PHY_INTERFACE_MODE_RGMII_ID || in fec_restart()
1162 fep->phy_interface == PHY_INTERFACE_MODE_RGMII_RXID || in fec_restart()
1163 fep->phy_interface == PHY_INTERFACE_MODE_RGMII_TXID) in fec_restart()
1165 else if (fep->phy_interface == PHY_INTERFACE_MODE_RMII) in fec_restart()
1171 if (ndev->phydev) { in fec_restart()
1172 if (ndev->phydev->speed == SPEED_1000) in fec_restart()
1174 else if (ndev->phydev->speed == SPEED_100) in fec_restart()
1181 if (fep->quirks & FEC_QUIRK_USE_GASKET) { in fec_restart()
1184 writel(0, fep->hwp + FEC_MIIGSK_ENR); in fec_restart()
1185 while (readl(fep->hwp + FEC_MIIGSK_ENR) & 4) in fec_restart()
1193 cfgr = (fep->phy_interface == PHY_INTERFACE_MODE_RMII) in fec_restart()
1195 if (ndev->phydev && ndev->phydev->speed == SPEED_10) in fec_restart()
1197 writel(cfgr, fep->hwp + FEC_MIIGSK_CFGR); in fec_restart()
1199 /* re-enable the gasket */ in fec_restart()
1200 writel(2, fep->hwp + FEC_MIIGSK_ENR); in fec_restart()
1207 if ((fep->pause_flag & FEC_PAUSE_FLAG_ENABLE) || in fec_restart()
1208 ((fep->pause_flag & FEC_PAUSE_FLAG_AUTONEG) && in fec_restart()
1209 ndev->phydev && ndev->phydev->pause)) { in fec_restart()
1213 writel(FEC_ENET_RSEM_V, fep->hwp + FEC_R_FIFO_RSEM); in fec_restart()
1214 writel(FEC_ENET_RSFL_V, fep->hwp + FEC_R_FIFO_RSFL); in fec_restart()
1215 writel(FEC_ENET_RAEM_V, fep->hwp + FEC_R_FIFO_RAEM); in fec_restart()
1216 writel(FEC_ENET_RAFL_V, fep->hwp + FEC_R_FIFO_RAFL); in fec_restart()
1219 writel(FEC_ENET_OPD_V, fep->hwp + FEC_OPD); in fec_restart()
1225 writel(rcntl, fep->hwp + FEC_R_CNTRL); in fec_restart()
1230 writel(0, fep->hwp + FEC_HASH_TABLE_HIGH); in fec_restart()
1231 writel(0, fep->hwp + FEC_HASH_TABLE_LOW); in fec_restart()
1234 if (fep->quirks & FEC_QUIRK_ENET_MAC) { in fec_restart()
1238 writel(FEC_TXWMRK_STRFWD, fep->hwp + FEC_X_WMRK); in fec_restart()
1241 if (fep->bufdesc_ex) in fec_restart()
1244 if (fep->quirks & FEC_QUIRK_DELAYED_CLKS_SUPPORT && in fec_restart()
1245 fep->rgmii_txc_dly) in fec_restart()
1247 if (fep->quirks & FEC_QUIRK_DELAYED_CLKS_SUPPORT && in fec_restart()
1248 fep->rgmii_rxc_dly) in fec_restart()
1253 writel(0 << 31, fep->hwp + FEC_MIB_CTRLSTAT); in fec_restart()
1257 writel(ecntl, fep->hwp + FEC_ECNTRL); in fec_restart()
1260 if (fep->bufdesc_ex) { in fec_restart()
1266 if (fep->link) in fec_restart()
1267 writel(FEC_DEFAULT_IMASK, fep->hwp + FEC_IMASK); in fec_restart()
1269 writel(0, fep->hwp + FEC_IMASK); in fec_restart()
1272 if (fep->quirks & FEC_QUIRK_HAS_COALESCE) in fec_restart()
1283 return imx_scu_get_handle(&fep->ipc_handle); in fec_enet_ipc_handle_init()
1288 struct device_node *np = fep->pdev->dev.of_node; in fec_enet_ipg_stop_set()
1292 if (!np || !fep->ipc_handle) in fec_enet_ipg_stop_set()
1301 imx_sc_misc_set_control(fep->ipc_handle, rsrc_id, IMX_SC_C_IPG_STOP, val); in fec_enet_ipg_stop_set()
1306 struct fec_platform_data *pdata = fep->pdev->dev.platform_data; in fec_enet_stop_mode()
1307 struct fec_stop_mode_gpr *stop_gpr = &fep->stop_gpr; in fec_enet_stop_mode()
1309 if (stop_gpr->gpr) { in fec_enet_stop_mode()
1311 regmap_update_bits(stop_gpr->gpr, stop_gpr->reg, in fec_enet_stop_mode()
1312 BIT(stop_gpr->bit), in fec_enet_stop_mode()
1313 BIT(stop_gpr->bit)); in fec_enet_stop_mode()
1315 regmap_update_bits(stop_gpr->gpr, stop_gpr->reg, in fec_enet_stop_mode()
1316 BIT(stop_gpr->bit), 0); in fec_enet_stop_mode()
1317 } else if (pdata && pdata->sleep_mode_enable) { in fec_enet_stop_mode()
1318 pdata->sleep_mode_enable(enabled); in fec_enet_stop_mode()
1328 writel(0, fep->hwp + FEC_IMASK); in fec_irqs_disable()
1335 writel(0, fep->hwp + FEC_IMASK); in fec_irqs_disable_except_wakeup()
1336 writel(FEC_ENET_WAKEUP, fep->hwp + FEC_IMASK); in fec_irqs_disable_except_wakeup()
1343 u32 rmii_mode = readl(fep->hwp + FEC_R_CNTRL) & FEC_RCR_RMII; in fec_stop()
1347 if (fep->link) { in fec_stop()
1348 writel(1, fep->hwp + FEC_X_CNTRL); /* Graceful transmit stop */ in fec_stop()
1350 if (!(readl(fep->hwp + FEC_IEVENT) & FEC_ENET_GRA)) in fec_stop()
1354 if (fep->bufdesc_ex) in fec_stop()
1361 if (!(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { in fec_stop()
1362 if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { in fec_stop()
1363 writel(0, fep->hwp + FEC_ECNTRL); in fec_stop()
1365 writel(FEC_ECR_RESET, fep->hwp + FEC_ECNTRL); in fec_stop()
1369 val = readl(fep->hwp + FEC_ECNTRL); in fec_stop()
1371 writel(val, fep->hwp + FEC_ECNTRL); in fec_stop()
1373 writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); in fec_stop()
1374 writel(FEC_DEFAULT_IMASK, fep->hwp + FEC_IMASK); in fec_stop()
1377 if (fep->quirks & FEC_QUIRK_ENET_MAC && in fec_stop()
1378 !(fep->wol_flag & FEC_WOL_FLAG_SLEEP_ON)) { in fec_stop()
1379 writel(FEC_ECR_ETHEREN, fep->hwp + FEC_ECNTRL); in fec_stop()
1380 writel(rmii_mode, fep->hwp + FEC_R_CNTRL); in fec_stop()
1383 if (fep->bufdesc_ex) { in fec_stop()
1384 val = readl(fep->hwp + FEC_ECNTRL); in fec_stop()
1386 writel(val, fep->hwp + FEC_ECNTRL); in fec_stop()
1400 ndev->stats.tx_errors++; in fec_timeout()
1402 schedule_work(&fep->tx_timeout_work); in fec_timeout()
1409 struct net_device *ndev = fep->netdev; in fec_enet_timeout_work()
1413 napi_disable(&fep->napi); in fec_enet_timeout_work()
1418 napi_enable(&fep->napi); in fec_enet_timeout_work()
1430 spin_lock_irqsave(&fep->tmreg_lock, flags); in fec_enet_hwtstamp()
1431 ns = timecounter_cyc2time(&fep->tc, ts); in fec_enet_hwtstamp()
1432 spin_unlock_irqrestore(&fep->tmreg_lock, flags); in fec_enet_hwtstamp()
1435 hwtstamps->hwtstamp = ns_to_ktime(ns); in fec_enet_hwtstamp()
1455 txq = fep->tx_queue[queue_id]; in fec_enet_tx_queue()
1458 bdp = txq->dirty_tx; in fec_enet_tx_queue()
1461 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_enet_tx_queue()
1463 while (bdp != READ_ONCE(txq->bd.cur)) { in fec_enet_tx_queue()
1466 status = fec16_to_cpu(READ_ONCE(bdp->cbd_sc)); in fec_enet_tx_queue()
1470 index = fec_enet_get_bd_index(bdp, &txq->bd); in fec_enet_tx_queue()
1472 if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB) { in fec_enet_tx_queue()
1473 skb = txq->tx_buf[index].buf_p; in fec_enet_tx_queue()
1474 if (bdp->cbd_bufaddr && in fec_enet_tx_queue()
1475 !IS_TSO_HEADER(txq, fec32_to_cpu(bdp->cbd_bufaddr))) in fec_enet_tx_queue()
1476 dma_unmap_single(&fep->pdev->dev, in fec_enet_tx_queue()
1477 fec32_to_cpu(bdp->cbd_bufaddr), in fec_enet_tx_queue()
1478 fec16_to_cpu(bdp->cbd_datlen), in fec_enet_tx_queue()
1480 bdp->cbd_bufaddr = cpu_to_fec32(0); in fec_enet_tx_queue()
1492 if (txq->tx_buf[index].type == FEC_TXBUF_T_XDP_NDO) { in fec_enet_tx_queue()
1493 xdpf = txq->tx_buf[index].buf_p; in fec_enet_tx_queue()
1494 if (bdp->cbd_bufaddr) in fec_enet_tx_queue()
1495 dma_unmap_single(&fep->pdev->dev, in fec_enet_tx_queue()
1496 fec32_to_cpu(bdp->cbd_bufaddr), in fec_enet_tx_queue()
1497 fec16_to_cpu(bdp->cbd_datlen), in fec_enet_tx_queue()
1500 page = txq->tx_buf[index].buf_p; in fec_enet_tx_queue()
1503 bdp->cbd_bufaddr = cpu_to_fec32(0); in fec_enet_tx_queue()
1504 if (unlikely(!txq->tx_buf[index].buf_p)) { in fec_enet_tx_queue()
1505 txq->tx_buf[index].type = FEC_TXBUF_T_SKB; in fec_enet_tx_queue()
1509 frame_len = fec16_to_cpu(bdp->cbd_datlen); in fec_enet_tx_queue()
1516 ndev->stats.tx_errors++; in fec_enet_tx_queue()
1518 ndev->stats.tx_heartbeat_errors++; in fec_enet_tx_queue()
1520 ndev->stats.tx_window_errors++; in fec_enet_tx_queue()
1522 ndev->stats.tx_aborted_errors++; in fec_enet_tx_queue()
1524 ndev->stats.tx_fifo_errors++; in fec_enet_tx_queue()
1526 ndev->stats.tx_carrier_errors++; in fec_enet_tx_queue()
1528 ndev->stats.tx_packets++; in fec_enet_tx_queue()
1530 if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB) in fec_enet_tx_queue()
1531 ndev->stats.tx_bytes += skb->len; in fec_enet_tx_queue()
1533 ndev->stats.tx_bytes += frame_len; in fec_enet_tx_queue()
1540 ndev->stats.collisions++; in fec_enet_tx_queue()
1542 if (txq->tx_buf[index].type == FEC_TXBUF_T_SKB) { in fec_enet_tx_queue()
1547 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS && in fec_enet_tx_queue()
1548 fep->hwts_tx_en) && fep->bufdesc_ex) { in fec_enet_tx_queue()
1552 fec_enet_hwtstamp(fep, fec32_to_cpu(ebdp->ts), &shhwtstamps); in fec_enet_tx_queue()
1558 } else if (txq->tx_buf[index].type == FEC_TXBUF_T_XDP_NDO) { in fec_enet_tx_queue()
1562 page_pool_put_page(page->pp, page, 0, true); in fec_enet_tx_queue()
1565 txq->tx_buf[index].buf_p = NULL; in fec_enet_tx_queue()
1567 txq->tx_buf[index].type = FEC_TXBUF_T_SKB; in fec_enet_tx_queue()
1574 txq->dirty_tx = bdp; in fec_enet_tx_queue()
1577 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_enet_tx_queue()
1583 if (entries_free >= txq->tx_wake_threshold) in fec_enet_tx_queue()
1589 if (bdp != txq->bd.cur && in fec_enet_tx_queue()
1590 readl(txq->bd.reg_desc_active) == 0) in fec_enet_tx_queue()
1591 writel(0, txq->bd.reg_desc_active); in fec_enet_tx_queue()
1600 for (i = fep->num_tx_queues - 1; i >= 0; i--) in fec_enet_tx()
1604 static int fec_enet_update_cbd(struct fec_enet_priv_rx_q *rxq, in fec_enet_update_cbd() argument
1610 new_page = page_pool_dev_alloc_pages(rxq->page_pool); in fec_enet_update_cbd()
1612 return -ENOMEM; in fec_enet_update_cbd()
1614 rxq->rx_skb_info[index].page = new_page; in fec_enet_update_cbd()
1615 rxq->rx_skb_info[index].offset = FEC_ENET_XDP_HEADROOM; in fec_enet_update_cbd()
1617 bdp->cbd_bufaddr = cpu_to_fec32(phys_addr); in fec_enet_update_cbd()
1624 struct xdp_buff *xdp, struct fec_enet_priv_rx_q *rxq, int cpu) in fec_enet_run_xdp() argument
1626 unsigned int sync, len = xdp->data_end - xdp->data; in fec_enet_run_xdp()
1637 sync = xdp->data_end - xdp->data; in fec_enet_run_xdp()
1642 rxq->stats[RX_XDP_PASS]++; in fec_enet_run_xdp()
1647 rxq->stats[RX_XDP_REDIRECT]++; in fec_enet_run_xdp()
1648 err = xdp_do_redirect(fep->netdev, xdp, prog); in fec_enet_run_xdp()
1656 rxq->stats[RX_XDP_TX]++; in fec_enet_run_xdp()
1659 rxq->stats[RX_XDP_TX_ERRORS]++; in fec_enet_run_xdp()
1667 bpf_warn_invalid_xdp_action(fep->netdev, prog, act); in fec_enet_run_xdp()
1674 rxq->stats[RX_XDP_DROP]++; in fec_enet_run_xdp()
1677 page = virt_to_head_page(xdp->data); in fec_enet_run_xdp()
1678 page_pool_put_page(rxq->page_pool, page, sync, true); in fec_enet_run_xdp()
1680 trace_xdp_exception(fep->netdev, prog, act); in fec_enet_run_xdp()
1696 struct fec_enet_priv_rx_q *rxq; in fec_enet_rx_queue() local
1707 bool need_swap = fep->quirks & FEC_QUIRK_SWAP_FRAME; in fec_enet_rx_queue()
1708 struct bpf_prog *xdp_prog = READ_ONCE(fep->xdp_prog); in fec_enet_rx_queue()
1721 if (fep->quirks & FEC_QUIRK_HAS_RACC) { in fec_enet_rx_queue()
1730 rxq = fep->rx_queue[queue_id]; in fec_enet_rx_queue()
1735 bdp = rxq->bd.cur; in fec_enet_rx_queue()
1736 xdp_init_buff(&xdp, PAGE_SIZE, &rxq->xdp_rxq); in fec_enet_rx_queue()
1738 while (!((status = fec16_to_cpu(bdp->cbd_sc)) & BD_ENET_RX_EMPTY)) { in fec_enet_rx_queue()
1744 writel(FEC_ENET_RXF_GET(queue_id), fep->hwp + FEC_IEVENT); in fec_enet_rx_queue()
1751 ndev->stats.rx_errors++; in fec_enet_rx_queue()
1754 ndev->stats.rx_fifo_errors++; in fec_enet_rx_queue()
1760 ndev->stats.rx_length_errors++; in fec_enet_rx_queue()
1765 ndev->stats.rx_crc_errors++; in fec_enet_rx_queue()
1768 ndev->stats.rx_frame_errors++; in fec_enet_rx_queue()
1773 ndev->stats.rx_packets++; in fec_enet_rx_queue()
1774 pkt_len = fec16_to_cpu(bdp->cbd_datlen); in fec_enet_rx_queue()
1775 ndev->stats.rx_bytes += pkt_len; in fec_enet_rx_queue()
1777 index = fec_enet_get_bd_index(bdp, &rxq->bd); in fec_enet_rx_queue()
1778 page = rxq->rx_skb_info[index].page; in fec_enet_rx_queue()
1779 cbd_bufaddr = bdp->cbd_bufaddr; in fec_enet_rx_queue()
1780 if (fec_enet_update_cbd(rxq, bdp, index)) { in fec_enet_rx_queue()
1781 ndev->stats.rx_dropped++; in fec_enet_rx_queue()
1785 dma_sync_single_for_cpu(&fep->pdev->dev, in fec_enet_rx_queue()
1795 data_start, pkt_len - sub_len, false); in fec_enet_rx_queue()
1796 ret = fec_enet_run_xdp(fep, xdp_prog, &xdp, rxq, cpu); in fec_enet_rx_queue()
1808 page_pool_recycle_direct(rxq->page_pool, page); in fec_enet_rx_queue()
1809 ndev->stats.rx_dropped++; in fec_enet_rx_queue()
1816 skb_put(skb, pkt_len - sub_len); in fec_enet_rx_queue()
1823 data = skb->data; in fec_enet_rx_queue()
1827 if (fep->bufdesc_ex) in fec_enet_rx_queue()
1832 if ((ndev->features & NETIF_F_HW_VLAN_CTAG_RX) && in fec_enet_rx_queue()
1833 fep->bufdesc_ex && in fec_enet_rx_queue()
1834 (ebdp->cbd_esc & cpu_to_fec32(BD_ENET_RX_VLAN))) { in fec_enet_rx_queue()
1838 vlan_tag = ntohs(vlan_header->h_vlan_TCI); in fec_enet_rx_queue()
1842 memmove(skb->data + VLAN_HLEN, data, ETH_ALEN * 2); in fec_enet_rx_queue()
1846 skb->protocol = eth_type_trans(skb, ndev); in fec_enet_rx_queue()
1849 if (fep->hwts_rx_en && fep->bufdesc_ex) in fec_enet_rx_queue()
1850 fec_enet_hwtstamp(fep, fec32_to_cpu(ebdp->ts), in fec_enet_rx_queue()
1853 if (fep->bufdesc_ex && in fec_enet_rx_queue()
1854 (fep->csum_flags & FLAG_RX_CSUM_ENABLED)) { in fec_enet_rx_queue()
1855 if (!(ebdp->cbd_esc & cpu_to_fec32(FLAG_RX_CSUM_ERROR))) { in fec_enet_rx_queue()
1857 skb->ip_summed = CHECKSUM_UNNECESSARY; in fec_enet_rx_queue()
1870 napi_gro_receive(&fep->napi, skb); in fec_enet_rx_queue()
1879 if (fep->bufdesc_ex) { in fec_enet_rx_queue()
1882 ebdp->cbd_esc = cpu_to_fec32(BD_ENET_RX_INT); in fec_enet_rx_queue()
1883 ebdp->cbd_prot = 0; in fec_enet_rx_queue()
1884 ebdp->cbd_bdu = 0; in fec_enet_rx_queue()
1890 bdp->cbd_sc = cpu_to_fec16(status); in fec_enet_rx_queue()
1893 bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); in fec_enet_rx_queue()
1899 writel(0, rxq->bd.reg_desc_active); in fec_enet_rx_queue()
1901 rxq->bd.cur = bdp; in fec_enet_rx_queue()
1915 for (i = fep->num_rx_queues - 1; i >= 0; i--) in fec_enet_rx()
1916 done += fec_enet_rx_queue(ndev, budget - done, i); in fec_enet_rx()
1925 int_events = readl(fep->hwp + FEC_IEVENT); in fec_enet_collect_events()
1930 writel(int_events, fep->hwp + FEC_IEVENT); in fec_enet_collect_events()
1942 if (fec_enet_collect_events(fep) && fep->link) { in fec_enet_interrupt()
1945 if (napi_schedule_prep(&fep->napi)) { in fec_enet_interrupt()
1947 writel(0, fep->hwp + FEC_IMASK); in fec_enet_interrupt()
1948 __napi_schedule(&fep->napi); in fec_enet_interrupt()
1957 struct net_device *ndev = napi->dev; in fec_enet_rx_napi()
1962 done += fec_enet_rx(ndev, budget - done); in fec_enet_rx_napi()
1968 writel(FEC_DEFAULT_IMASK, fep->hwp + FEC_IMASK); in fec_enet_rx_napi()
1974 /* ------------------------------------------------------------------------- */
1993 struct device_node *np = fep->pdev->dev.of_node; in fec_get_mac()
1998 else if (ret == -EPROBE_DEFER) in fec_get_mac()
2011 struct fec_platform_data *pdata = dev_get_platdata(&fep->pdev->dev); in fec_get_mac()
2014 iap = (unsigned char *)&pdata->mac; in fec_get_mac()
2023 cpu_to_be32(readl(fep->hwp + FEC_ADDR_LOW)); in fec_get_mac()
2025 cpu_to_be16(readl(fep->hwp + FEC_ADDR_HIGH) >> 16); in fec_get_mac()
2034 dev_err(&fep->pdev->dev, "Invalid MAC address: %pM\n", iap); in fec_get_mac()
2036 dev_info(&fep->pdev->dev, "Using random MAC address: %pM\n", in fec_get_mac()
2037 ndev->dev_addr); in fec_get_mac()
2042 eth_hw_addr_gen(ndev, iap, iap == macaddr ? fep->dev_id : 0); in fec_get_mac()
2047 /* ------------------------------------------------------------------------- */
2055 struct phy_device *phy_dev = ndev->phydev; in fec_enet_adjust_link()
2064 fep->link = 0; in fec_enet_adjust_link()
2065 } else if (phy_dev->link) { in fec_enet_adjust_link()
2066 if (!fep->link) { in fec_enet_adjust_link()
2067 fep->link = phy_dev->link; in fec_enet_adjust_link()
2071 if (fep->full_duplex != phy_dev->duplex) { in fec_enet_adjust_link()
2072 fep->full_duplex = phy_dev->duplex; in fec_enet_adjust_link()
2076 if (phy_dev->speed != fep->speed) { in fec_enet_adjust_link()
2077 fep->speed = phy_dev->speed; in fec_enet_adjust_link()
2084 napi_disable(&fep->napi); in fec_enet_adjust_link()
2089 napi_enable(&fep->napi); in fec_enet_adjust_link()
2092 if (fep->link) { in fec_enet_adjust_link()
2094 napi_disable(&fep->napi); in fec_enet_adjust_link()
2098 napi_enable(&fep->napi); in fec_enet_adjust_link()
2099 fep->link = phy_dev->link; in fec_enet_adjust_link()
2113 ret = readl_poll_timeout_atomic(fep->hwp + FEC_IEVENT, ievent, in fec_enet_mdio_wait()
2117 writel(FEC_ENET_MII, fep->hwp + FEC_IEVENT); in fec_enet_mdio_wait()
2124 struct fec_enet_private *fep = bus->priv; in fec_enet_mdio_read_c22()
2125 struct device *dev = &fep->pdev->dev; in fec_enet_mdio_read_c22()
2140 FEC_MMFR_TA, fep->hwp + FEC_MII_DATA); in fec_enet_mdio_read_c22()
2145 netdev_err(fep->netdev, "MDIO read timeout\n"); in fec_enet_mdio_read_c22()
2149 ret = FEC_MMFR_DATA(readl(fep->hwp + FEC_MII_DATA)); in fec_enet_mdio_read_c22()
2161 struct fec_enet_private *fep = bus->priv; in fec_enet_mdio_read_c45()
2162 struct device *dev = &fep->pdev->dev; in fec_enet_mdio_read_c45()
2175 fep->hwp + FEC_MII_DATA); in fec_enet_mdio_read_c45()
2180 netdev_err(fep->netdev, "MDIO address write timeout\n"); in fec_enet_mdio_read_c45()
2189 FEC_MMFR_TA, fep->hwp + FEC_MII_DATA); in fec_enet_mdio_read_c45()
2194 netdev_err(fep->netdev, "MDIO read timeout\n"); in fec_enet_mdio_read_c45()
2198 ret = FEC_MMFR_DATA(readl(fep->hwp + FEC_MII_DATA)); in fec_enet_mdio_read_c45()
2210 struct fec_enet_private *fep = bus->priv; in fec_enet_mdio_write_c22()
2211 struct device *dev = &fep->pdev->dev; in fec_enet_mdio_write_c22()
2226 fep->hwp + FEC_MII_DATA); in fec_enet_mdio_write_c22()
2231 netdev_err(fep->netdev, "MDIO write timeout\n"); in fec_enet_mdio_write_c22()
2242 struct fec_enet_private *fep = bus->priv; in fec_enet_mdio_write_c45()
2243 struct device *dev = &fep->pdev->dev; in fec_enet_mdio_write_c45()
2256 fep->hwp + FEC_MII_DATA); in fec_enet_mdio_write_c45()
2261 netdev_err(fep->netdev, "MDIO address write timeout\n"); in fec_enet_mdio_write_c45()
2269 fep->hwp + FEC_MII_DATA); in fec_enet_mdio_write_c45()
2274 netdev_err(fep->netdev, "MDIO write timeout\n"); in fec_enet_mdio_write_c45()
2286 struct phy_device *phy_dev = ndev->phydev; in fec_enet_phy_reset_after_clk_enable()
2290 } else if (fep->phy_node) { in fec_enet_phy_reset_after_clk_enable()
2298 phy_dev = of_phy_find_device(fep->phy_node); in fec_enet_phy_reset_after_clk_enable()
2300 put_device(&phy_dev->mdio.dev); in fec_enet_phy_reset_after_clk_enable()
2310 ret = clk_prepare_enable(fep->clk_enet_out); in fec_enet_clk_enable()
2314 if (fep->clk_ptp) { in fec_enet_clk_enable()
2315 mutex_lock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2316 ret = clk_prepare_enable(fep->clk_ptp); in fec_enet_clk_enable()
2318 mutex_unlock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2321 fep->ptp_clk_on = true; in fec_enet_clk_enable()
2323 mutex_unlock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2326 ret = clk_prepare_enable(fep->clk_ref); in fec_enet_clk_enable()
2330 ret = clk_prepare_enable(fep->clk_2x_txclk); in fec_enet_clk_enable()
2336 clk_disable_unprepare(fep->clk_enet_out); in fec_enet_clk_enable()
2337 if (fep->clk_ptp) { in fec_enet_clk_enable()
2338 mutex_lock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2339 clk_disable_unprepare(fep->clk_ptp); in fec_enet_clk_enable()
2340 fep->ptp_clk_on = false; in fec_enet_clk_enable()
2341 mutex_unlock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2343 clk_disable_unprepare(fep->clk_ref); in fec_enet_clk_enable()
2344 clk_disable_unprepare(fep->clk_2x_txclk); in fec_enet_clk_enable()
2350 if (fep->clk_ref) in fec_enet_clk_enable()
2351 clk_disable_unprepare(fep->clk_ref); in fec_enet_clk_enable()
2353 if (fep->clk_ptp) { in fec_enet_clk_enable()
2354 mutex_lock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2355 clk_disable_unprepare(fep->clk_ptp); in fec_enet_clk_enable()
2356 fep->ptp_clk_on = false; in fec_enet_clk_enable()
2357 mutex_unlock(&fep->ptp_clk_mutex); in fec_enet_clk_enable()
2360 clk_disable_unprepare(fep->clk_enet_out); in fec_enet_clk_enable()
2371 if (!of_property_read_u32(np, "tx-internal-delay-ps", &rgmii_tx_delay)) { in fec_enet_parse_rgmii_delay()
2373 dev_err(&fep->pdev->dev, "The only allowed RGMII TX delay values are: 0ps, 2000ps"); in fec_enet_parse_rgmii_delay()
2374 return -EINVAL; in fec_enet_parse_rgmii_delay()
2376 fep->rgmii_txc_dly = true; in fec_enet_parse_rgmii_delay()
2381 if (!of_property_read_u32(np, "rx-internal-delay-ps", &rgmii_rx_delay)) { in fec_enet_parse_rgmii_delay()
2383 dev_err(&fep->pdev->dev, "The only allowed RGMII RX delay values are: 0ps, 2000ps"); in fec_enet_parse_rgmii_delay()
2384 return -EINVAL; in fec_enet_parse_rgmii_delay()
2386 fep->rgmii_rxc_dly = true; in fec_enet_parse_rgmii_delay()
2400 int dev_id = fep->dev_id; in fec_enet_mii_probe()
2402 if (fep->phy_node) { in fec_enet_mii_probe()
2403 phy_dev = of_phy_connect(ndev, fep->phy_node, in fec_enet_mii_probe()
2405 fep->phy_interface); in fec_enet_mii_probe()
2408 return -ENODEV; in fec_enet_mii_probe()
2413 if (!mdiobus_is_registered_device(fep->mii_bus, phy_id)) in fec_enet_mii_probe()
2415 if (dev_id--) in fec_enet_mii_probe()
2417 strscpy(mdio_bus_id, fep->mii_bus->id, MII_BUS_ID_SIZE); in fec_enet_mii_probe()
2423 strscpy(mdio_bus_id, "fixed-0", MII_BUS_ID_SIZE); in fec_enet_mii_probe()
2430 fep->phy_interface); in fec_enet_mii_probe()
2439 if (fep->quirks & FEC_QUIRK_HAS_GBIT) { in fec_enet_mii_probe()
2450 fep->link = 0; in fec_enet_mii_probe()
2451 fep->full_duplex = 0; in fec_enet_mii_probe()
2466 int err = -ENXIO; in fec_enet_mii_init()
2475 * - fec0 supports MII & RMII modes while fec1 only supports RMII in fec_enet_mii_init()
2476 * - fec0 acts as the 1588 time master while fec1 is slave in fec_enet_mii_init()
2477 * - external phys can only be configured by fec0 in fec_enet_mii_init()
2487 if ((fep->quirks & FEC_QUIRK_SINGLE_MDIO) && fep->dev_id > 0) { in fec_enet_mii_init()
2490 fep->mii_bus = fec0_mii_bus; in fec_enet_mii_init()
2494 return -ENOENT; in fec_enet_mii_init()
2498 node = of_get_child_by_name(pdev->dev.of_node, "mdio"); in fec_enet_mii_init()
2500 of_property_read_u32(node, "clock-frequency", &bus_freq); in fec_enet_mii_init()
2502 "suppress-preamble"); in fec_enet_mii_init()
2509 * for ENET-MAC is 'ref_freq / ((MII_SPEED + 1) x 2)'. The i.MX28 in fec_enet_mii_init()
2513 mii_speed = DIV_ROUND_UP(clk_get_rate(fep->clk_ipg), bus_freq * 2); in fec_enet_mii_init()
2514 if (fep->quirks & FEC_QUIRK_ENET_MAC) in fec_enet_mii_init()
2515 mii_speed--; in fec_enet_mii_init()
2517 dev_err(&pdev->dev, in fec_enet_mii_init()
2519 clk_get_rate(fep->clk_ipg)); in fec_enet_mii_init()
2520 err = -EINVAL; in fec_enet_mii_init()
2536 holdtime = DIV_ROUND_UP(clk_get_rate(fep->clk_ipg), 100000000) - 1; in fec_enet_mii_init()
2538 fep->phy_speed = mii_speed << 1 | holdtime << 8; in fec_enet_mii_init()
2541 fep->phy_speed |= BIT(7); in fec_enet_mii_init()
2543 if (fep->quirks & FEC_QUIRK_CLEAR_SETUP_MII) { in fec_enet_mii_init()
2546 * - writing MSCR: in fec_enet_mii_init()
2547 * - mmfr[31:0]_not_zero & mscr[7:0]_is_zero & in fec_enet_mii_init()
2549 * - writing MMFR: in fec_enet_mii_init()
2550 * - mscr[7:0]_not_zero in fec_enet_mii_init()
2552 writel(0, fep->hwp + FEC_MII_DATA); in fec_enet_mii_init()
2555 writel(fep->phy_speed, fep->hwp + FEC_MII_SPEED); in fec_enet_mii_init()
2558 writel(FEC_ENET_MII, fep->hwp + FEC_IEVENT); in fec_enet_mii_init()
2560 fep->mii_bus = mdiobus_alloc(); in fec_enet_mii_init()
2561 if (fep->mii_bus == NULL) { in fec_enet_mii_init()
2562 err = -ENOMEM; in fec_enet_mii_init()
2566 fep->mii_bus->name = "fec_enet_mii_bus"; in fec_enet_mii_init()
2567 fep->mii_bus->read = fec_enet_mdio_read_c22; in fec_enet_mii_init()
2568 fep->mii_bus->write = fec_enet_mdio_write_c22; in fec_enet_mii_init()
2569 if (fep->quirks & FEC_QUIRK_HAS_MDIO_C45) { in fec_enet_mii_init()
2570 fep->mii_bus->read_c45 = fec_enet_mdio_read_c45; in fec_enet_mii_init()
2571 fep->mii_bus->write_c45 = fec_enet_mdio_write_c45; in fec_enet_mii_init()
2573 snprintf(fep->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x", in fec_enet_mii_init()
2574 pdev->name, fep->dev_id + 1); in fec_enet_mii_init()
2575 fep->mii_bus->priv = fep; in fec_enet_mii_init()
2576 fep->mii_bus->parent = &pdev->dev; in fec_enet_mii_init()
2578 err = of_mdiobus_register(fep->mii_bus, node); in fec_enet_mii_init()
2585 phydev = mdiobus_get_phy(fep->mii_bus, addr); in fec_enet_mii_init()
2587 phydev->mac_managed_pm = true; in fec_enet_mii_init()
2593 if (fep->quirks & FEC_QUIRK_SINGLE_MDIO) in fec_enet_mii_init()
2594 fec0_mii_bus = fep->mii_bus; in fec_enet_mii_init()
2599 mdiobus_free(fep->mii_bus); in fec_enet_mii_init()
2607 if (--mii_cnt == 0) { in fec_enet_mii_remove()
2608 mdiobus_unregister(fep->mii_bus); in fec_enet_mii_remove()
2609 mdiobus_free(fep->mii_bus); in fec_enet_mii_remove()
2618 strscpy(info->driver, fep->pdev->dev.driver->name, in fec_enet_get_drvinfo()
2619 sizeof(info->driver)); in fec_enet_get_drvinfo()
2620 strscpy(info->bus_info, dev_name(&ndev->dev), sizeof(info->bus_info)); in fec_enet_get_drvinfo()
2629 r = platform_get_resource(fep->pdev, IORESOURCE_MEM, 0); in fec_enet_get_regs_len()
2714 u32 __iomem *theregs = (u32 __iomem *)fep->hwp; in fec_enet_get_regs()
2715 struct device *dev = &fep->pdev->dev; in fec_enet_get_regs()
2741 regs->version = fec_enet_register_version; in fec_enet_get_regs()
2743 memset(buf, 0, regs->len); in fec_enet_get_regs()
2749 !(fep->quirks & FEC_QUIRK_HAS_FRREG)) in fec_enet_get_regs()
2765 if (fep->bufdesc_ex) { in fec_enet_get_ts_info()
2767 info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | in fec_enet_get_ts_info()
2773 if (fep->ptp_clock) in fec_enet_get_ts_info()
2774 info->phc_index = ptp_clock_index(fep->ptp_clock); in fec_enet_get_ts_info()
2776 info->phc_index = -1; in fec_enet_get_ts_info()
2778 info->tx_types = (1 << HWTSTAMP_TX_OFF) | in fec_enet_get_ts_info()
2781 info->rx_filters = (1 << HWTSTAMP_FILTER_NONE) | in fec_enet_get_ts_info()
2796 pause->autoneg = (fep->pause_flag & FEC_PAUSE_FLAG_AUTONEG) != 0; in fec_enet_get_pauseparam()
2797 pause->tx_pause = (fep->pause_flag & FEC_PAUSE_FLAG_ENABLE) != 0; in fec_enet_get_pauseparam()
2798 pause->rx_pause = pause->tx_pause; in fec_enet_get_pauseparam()
2806 if (!ndev->phydev) in fec_enet_set_pauseparam()
2807 return -ENODEV; in fec_enet_set_pauseparam()
2809 if (pause->tx_pause != pause->rx_pause) { in fec_enet_set_pauseparam()
2812 return -EINVAL; in fec_enet_set_pauseparam()
2815 fep->pause_flag = 0; in fec_enet_set_pauseparam()
2818 fep->pause_flag |= pause->rx_pause ? FEC_PAUSE_FLAG_ENABLE : 0; in fec_enet_set_pauseparam()
2819 fep->pause_flag |= pause->autoneg ? FEC_PAUSE_FLAG_AUTONEG : 0; in fec_enet_set_pauseparam()
2821 phy_set_sym_pause(ndev->phydev, pause->rx_pause, pause->tx_pause, in fec_enet_set_pauseparam()
2822 pause->autoneg); in fec_enet_set_pauseparam()
2824 if (pause->autoneg) { in fec_enet_set_pauseparam()
2827 phy_start_aneg(ndev->phydev); in fec_enet_set_pauseparam()
2830 napi_disable(&fep->napi); in fec_enet_set_pauseparam()
2835 napi_enable(&fep->napi); in fec_enet_set_pauseparam()
2925 fep->ethtool_stats[i] = readl(fep->hwp + fec_stats[i].offset); in fec_enet_update_ethtool_stats()
2931 struct fec_enet_priv_rx_q *rxq; in fec_enet_get_xdp_stats() local
2934 for (i = fep->num_rx_queues - 1; i >= 0; i--) { in fec_enet_get_xdp_stats()
2935 rxq = fep->rx_queue[i]; in fec_enet_get_xdp_stats()
2938 xdp_stats[j] += rxq->stats[j]; in fec_enet_get_xdp_stats()
2948 struct fec_enet_priv_rx_q *rxq; in fec_enet_page_pool_stats() local
2951 for (i = fep->num_rx_queues - 1; i >= 0; i--) { in fec_enet_page_pool_stats()
2952 rxq = fep->rx_queue[i]; in fec_enet_page_pool_stats()
2954 if (!rxq->page_pool) in fec_enet_page_pool_stats()
2957 page_pool_get_stats(rxq->page_pool, &stats); in fec_enet_page_pool_stats()
2972 memcpy(data, fep->ethtool_stats, FEC_STATS_SIZE); in fec_enet_get_ethtool_stats()
3017 return -EOPNOTSUPP; in fec_enet_get_sset_count()
3024 struct fec_enet_priv_rx_q *rxq; in fec_enet_clear_ethtool_stats() local
3028 writel(FEC_MIB_CTRLSTAT_DISABLE, fep->hwp + FEC_MIB_CTRLSTAT); in fec_enet_clear_ethtool_stats()
3031 writel(0, fep->hwp + fec_stats[i].offset); in fec_enet_clear_ethtool_stats()
3033 for (i = fep->num_rx_queues - 1; i >= 0; i--) { in fec_enet_clear_ethtool_stats()
3034 rxq = fep->rx_queue[i]; in fec_enet_clear_ethtool_stats()
3036 rxq->stats[j] = 0; in fec_enet_clear_ethtool_stats()
3040 writel(0, fep->hwp + FEC_MIB_CTRLSTAT); in fec_enet_clear_ethtool_stats()
3062 return us * (fep->itr_clk_rate / 64000) / 1000; in fec_enet_us_to_itr_clock()
3072 if (!fep->rx_time_itr || !fep->rx_pkts_itr || in fec_enet_itr_coal_set()
3073 !fep->tx_time_itr || !fep->tx_pkts_itr) in fec_enet_itr_coal_set()
3083 rx_itr |= FEC_ITR_ICFT(fep->rx_pkts_itr); in fec_enet_itr_coal_set()
3084 rx_itr |= FEC_ITR_ICTT(fec_enet_us_to_itr_clock(ndev, fep->rx_time_itr)); in fec_enet_itr_coal_set()
3085 tx_itr |= FEC_ITR_ICFT(fep->tx_pkts_itr); in fec_enet_itr_coal_set()
3086 tx_itr |= FEC_ITR_ICTT(fec_enet_us_to_itr_clock(ndev, fep->tx_time_itr)); in fec_enet_itr_coal_set()
3091 writel(tx_itr, fep->hwp + FEC_TXIC0); in fec_enet_itr_coal_set()
3092 writel(rx_itr, fep->hwp + FEC_RXIC0); in fec_enet_itr_coal_set()
3093 if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { in fec_enet_itr_coal_set()
3094 writel(tx_itr, fep->hwp + FEC_TXIC1); in fec_enet_itr_coal_set()
3095 writel(rx_itr, fep->hwp + FEC_RXIC1); in fec_enet_itr_coal_set()
3096 writel(tx_itr, fep->hwp + FEC_TXIC2); in fec_enet_itr_coal_set()
3097 writel(rx_itr, fep->hwp + FEC_RXIC2); in fec_enet_itr_coal_set()
3108 if (!(fep->quirks & FEC_QUIRK_HAS_COALESCE)) in fec_enet_get_coalesce()
3109 return -EOPNOTSUPP; in fec_enet_get_coalesce()
3111 ec->rx_coalesce_usecs = fep->rx_time_itr; in fec_enet_get_coalesce()
3112 ec->rx_max_coalesced_frames = fep->rx_pkts_itr; in fec_enet_get_coalesce()
3114 ec->tx_coalesce_usecs = fep->tx_time_itr; in fec_enet_get_coalesce()
3115 ec->tx_max_coalesced_frames = fep->tx_pkts_itr; in fec_enet_get_coalesce()
3126 struct device *dev = &fep->pdev->dev; in fec_enet_set_coalesce()
3129 if (!(fep->quirks & FEC_QUIRK_HAS_COALESCE)) in fec_enet_set_coalesce()
3130 return -EOPNOTSUPP; in fec_enet_set_coalesce()
3132 if (ec->rx_max_coalesced_frames > 255) { in fec_enet_set_coalesce()
3134 return -EINVAL; in fec_enet_set_coalesce()
3137 if (ec->tx_max_coalesced_frames > 255) { in fec_enet_set_coalesce()
3139 return -EINVAL; in fec_enet_set_coalesce()
3142 cycle = fec_enet_us_to_itr_clock(ndev, ec->rx_coalesce_usecs); in fec_enet_set_coalesce()
3145 return -EINVAL; in fec_enet_set_coalesce()
3148 cycle = fec_enet_us_to_itr_clock(ndev, ec->tx_coalesce_usecs); in fec_enet_set_coalesce()
3151 return -EINVAL; in fec_enet_set_coalesce()
3154 fep->rx_time_itr = ec->rx_coalesce_usecs; in fec_enet_set_coalesce()
3155 fep->rx_pkts_itr = ec->rx_max_coalesced_frames; in fec_enet_set_coalesce()
3157 fep->tx_time_itr = ec->tx_coalesce_usecs; in fec_enet_set_coalesce()
3158 fep->tx_pkts_itr = ec->tx_max_coalesced_frames; in fec_enet_set_coalesce()
3172 return us * (fep->clk_ref_rate / 1000) / 1000; in fec_enet_us_to_tx_cycle()
3178 struct ethtool_eee *p = &fep->eee; in fec_enet_eee_mode_set()
3183 ret = phy_init_eee(ndev->phydev, false); in fec_enet_eee_mode_set()
3187 sleep_cycle = fec_enet_us_to_tx_cycle(ndev, p->tx_lpi_timer); in fec_enet_eee_mode_set()
3194 p->tx_lpi_enabled = enable; in fec_enet_eee_mode_set()
3195 p->eee_enabled = enable; in fec_enet_eee_mode_set()
3196 p->eee_active = enable; in fec_enet_eee_mode_set()
3198 writel(sleep_cycle, fep->hwp + FEC_LPI_SLEEP); in fec_enet_eee_mode_set()
3199 writel(wake_cycle, fep->hwp + FEC_LPI_WAKE); in fec_enet_eee_mode_set()
3208 struct ethtool_eee *p = &fep->eee; in fec_enet_get_eee()
3210 if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) in fec_enet_get_eee()
3211 return -EOPNOTSUPP; in fec_enet_get_eee()
3214 return -ENETDOWN; in fec_enet_get_eee()
3216 edata->eee_enabled = p->eee_enabled; in fec_enet_get_eee()
3217 edata->eee_active = p->eee_active; in fec_enet_get_eee()
3218 edata->tx_lpi_timer = p->tx_lpi_timer; in fec_enet_get_eee()
3219 edata->tx_lpi_enabled = p->tx_lpi_enabled; in fec_enet_get_eee()
3221 return phy_ethtool_get_eee(ndev->phydev, edata); in fec_enet_get_eee()
3228 struct ethtool_eee *p = &fep->eee; in fec_enet_set_eee()
3231 if (!(fep->quirks & FEC_QUIRK_HAS_EEE)) in fec_enet_set_eee()
3232 return -EOPNOTSUPP; in fec_enet_set_eee()
3235 return -ENETDOWN; in fec_enet_set_eee()
3237 p->tx_lpi_timer = edata->tx_lpi_timer; in fec_enet_set_eee()
3239 if (!edata->eee_enabled || !edata->tx_lpi_enabled || in fec_enet_set_eee()
3240 !edata->tx_lpi_timer) in fec_enet_set_eee()
3248 return phy_ethtool_set_eee(ndev->phydev, edata); in fec_enet_set_eee()
3256 if (fep->wol_flag & FEC_WOL_HAS_MAGIC_PACKET) { in fec_enet_get_wol()
3257 wol->supported = WAKE_MAGIC; in fec_enet_get_wol()
3258 wol->wolopts = fep->wol_flag & FEC_WOL_FLAG_ENABLE ? WAKE_MAGIC : 0; in fec_enet_get_wol()
3260 wol->supported = wol->wolopts = 0; in fec_enet_get_wol()
3269 if (!(fep->wol_flag & FEC_WOL_HAS_MAGIC_PACKET)) in fec_enet_set_wol()
3270 return -EINVAL; in fec_enet_set_wol()
3272 if (wol->wolopts & ~WAKE_MAGIC) in fec_enet_set_wol()
3273 return -EINVAL; in fec_enet_set_wol()
3275 device_set_wakeup_enable(&ndev->dev, wol->wolopts & WAKE_MAGIC); in fec_enet_set_wol()
3276 if (device_may_wakeup(&ndev->dev)) in fec_enet_set_wol()
3277 fep->wol_flag |= FEC_WOL_FLAG_ENABLE; in fec_enet_set_wol()
3279 fep->wol_flag &= (~FEC_WOL_FLAG_ENABLE); in fec_enet_set_wol()
3316 struct fec_enet_priv_rx_q *rxq; in fec_enet_free_buffers() local
3319 for (q = 0; q < fep->num_rx_queues; q++) { in fec_enet_free_buffers()
3320 rxq = fep->rx_queue[q]; in fec_enet_free_buffers()
3321 for (i = 0; i < rxq->bd.ring_size; i++) in fec_enet_free_buffers()
3322 page_pool_put_full_page(rxq->page_pool, rxq->rx_skb_info[i].page, false); in fec_enet_free_buffers()
3325 rxq->stats[i] = 0; in fec_enet_free_buffers()
3327 if (xdp_rxq_info_is_reg(&rxq->xdp_rxq)) in fec_enet_free_buffers()
3328 xdp_rxq_info_unreg(&rxq->xdp_rxq); in fec_enet_free_buffers()
3329 page_pool_destroy(rxq->page_pool); in fec_enet_free_buffers()
3330 rxq->page_pool = NULL; in fec_enet_free_buffers()
3333 for (q = 0; q < fep->num_tx_queues; q++) { in fec_enet_free_buffers()
3334 txq = fep->tx_queue[q]; in fec_enet_free_buffers()
3335 for (i = 0; i < txq->bd.ring_size; i++) { in fec_enet_free_buffers()
3336 kfree(txq->tx_bounce[i]); in fec_enet_free_buffers()
3337 txq->tx_bounce[i] = NULL; in fec_enet_free_buffers()
3339 if (!txq->tx_buf[i].buf_p) { in fec_enet_free_buffers()
3340 txq->tx_buf[i].type = FEC_TXBUF_T_SKB; in fec_enet_free_buffers()
3344 if (txq->tx_buf[i].type == FEC_TXBUF_T_SKB) { in fec_enet_free_buffers()
3345 dev_kfree_skb(txq->tx_buf[i].buf_p); in fec_enet_free_buffers()
3346 } else if (txq->tx_buf[i].type == FEC_TXBUF_T_XDP_NDO) { in fec_enet_free_buffers()
3347 xdp_return_frame(txq->tx_buf[i].buf_p); in fec_enet_free_buffers()
3349 struct page *page = txq->tx_buf[i].buf_p; in fec_enet_free_buffers()
3351 page_pool_put_page(page->pp, page, 0, false); in fec_enet_free_buffers()
3354 txq->tx_buf[i].buf_p = NULL; in fec_enet_free_buffers()
3355 txq->tx_buf[i].type = FEC_TXBUF_T_SKB; in fec_enet_free_buffers()
3366 for (i = 0; i < fep->num_tx_queues; i++) in fec_enet_free_queue()
3367 if (fep->tx_queue[i] && fep->tx_queue[i]->tso_hdrs) { in fec_enet_free_queue()
3368 txq = fep->tx_queue[i]; in fec_enet_free_queue()
3369 dma_free_coherent(&fep->pdev->dev, in fec_enet_free_queue()
3370 txq->bd.ring_size * TSO_HEADER_SIZE, in fec_enet_free_queue()
3371 txq->tso_hdrs, in fec_enet_free_queue()
3372 txq->tso_hdrs_dma); in fec_enet_free_queue()
3375 for (i = 0; i < fep->num_rx_queues; i++) in fec_enet_free_queue()
3376 kfree(fep->rx_queue[i]); in fec_enet_free_queue()
3377 for (i = 0; i < fep->num_tx_queues; i++) in fec_enet_free_queue()
3378 kfree(fep->tx_queue[i]); in fec_enet_free_queue()
3388 for (i = 0; i < fep->num_tx_queues; i++) { in fec_enet_alloc_queue()
3391 ret = -ENOMEM; in fec_enet_alloc_queue()
3395 fep->tx_queue[i] = txq; in fec_enet_alloc_queue()
3396 txq->bd.ring_size = TX_RING_SIZE; in fec_enet_alloc_queue()
3397 fep->total_tx_ring_size += fep->tx_queue[i]->bd.ring_size; in fec_enet_alloc_queue()
3399 txq->tx_stop_threshold = FEC_MAX_SKB_DESCS; in fec_enet_alloc_queue()
3400 txq->tx_wake_threshold = FEC_MAX_SKB_DESCS + 2 * MAX_SKB_FRAGS; in fec_enet_alloc_queue()
3402 txq->tso_hdrs = dma_alloc_coherent(&fep->pdev->dev, in fec_enet_alloc_queue()
3403 txq->bd.ring_size * TSO_HEADER_SIZE, in fec_enet_alloc_queue()
3404 &txq->tso_hdrs_dma, in fec_enet_alloc_queue()
3406 if (!txq->tso_hdrs) { in fec_enet_alloc_queue()
3407 ret = -ENOMEM; in fec_enet_alloc_queue()
3412 for (i = 0; i < fep->num_rx_queues; i++) { in fec_enet_alloc_queue()
3413 fep->rx_queue[i] = kzalloc(sizeof(*fep->rx_queue[i]), in fec_enet_alloc_queue()
3415 if (!fep->rx_queue[i]) { in fec_enet_alloc_queue()
3416 ret = -ENOMEM; in fec_enet_alloc_queue()
3420 fep->rx_queue[i]->bd.ring_size = RX_RING_SIZE; in fec_enet_alloc_queue()
3421 fep->total_rx_ring_size += fep->rx_queue[i]->bd.ring_size; in fec_enet_alloc_queue()
3434 struct fec_enet_priv_rx_q *rxq; in fec_enet_alloc_rxq_buffers() local
3440 rxq = fep->rx_queue[queue]; in fec_enet_alloc_rxq_buffers()
3441 bdp = rxq->bd.base; in fec_enet_alloc_rxq_buffers()
3443 err = fec_enet_create_page_pool(fep, rxq, rxq->bd.ring_size); in fec_enet_alloc_rxq_buffers()
3449 for (i = 0; i < rxq->bd.ring_size; i++) { in fec_enet_alloc_rxq_buffers()
3450 page = page_pool_dev_alloc_pages(rxq->page_pool); in fec_enet_alloc_rxq_buffers()
3455 bdp->cbd_bufaddr = cpu_to_fec32(phys_addr); in fec_enet_alloc_rxq_buffers()
3457 rxq->rx_skb_info[i].page = page; in fec_enet_alloc_rxq_buffers()
3458 rxq->rx_skb_info[i].offset = FEC_ENET_XDP_HEADROOM; in fec_enet_alloc_rxq_buffers()
3459 bdp->cbd_sc = cpu_to_fec16(BD_ENET_RX_EMPTY); in fec_enet_alloc_rxq_buffers()
3461 if (fep->bufdesc_ex) { in fec_enet_alloc_rxq_buffers()
3463 ebdp->cbd_esc = cpu_to_fec32(BD_ENET_RX_INT); in fec_enet_alloc_rxq_buffers()
3466 bdp = fec_enet_get_nextdesc(bdp, &rxq->bd); in fec_enet_alloc_rxq_buffers()
3470 bdp = fec_enet_get_prevdesc(bdp, &rxq->bd); in fec_enet_alloc_rxq_buffers()
3471 bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); in fec_enet_alloc_rxq_buffers()
3476 return -ENOMEM; in fec_enet_alloc_rxq_buffers()
3487 txq = fep->tx_queue[queue]; in fec_enet_alloc_txq_buffers()
3488 bdp = txq->bd.base; in fec_enet_alloc_txq_buffers()
3489 for (i = 0; i < txq->bd.ring_size; i++) { in fec_enet_alloc_txq_buffers()
3490 txq->tx_bounce[i] = kmalloc(FEC_ENET_TX_FRSIZE, GFP_KERNEL); in fec_enet_alloc_txq_buffers()
3491 if (!txq->tx_bounce[i]) in fec_enet_alloc_txq_buffers()
3494 bdp->cbd_sc = cpu_to_fec16(0); in fec_enet_alloc_txq_buffers()
3495 bdp->cbd_bufaddr = cpu_to_fec32(0); in fec_enet_alloc_txq_buffers()
3497 if (fep->bufdesc_ex) { in fec_enet_alloc_txq_buffers()
3499 ebdp->cbd_esc = cpu_to_fec32(BD_ENET_TX_INT); in fec_enet_alloc_txq_buffers()
3502 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_enet_alloc_txq_buffers()
3506 bdp = fec_enet_get_prevdesc(bdp, &txq->bd); in fec_enet_alloc_txq_buffers()
3507 bdp->cbd_sc |= cpu_to_fec16(BD_SC_WRAP); in fec_enet_alloc_txq_buffers()
3513 return -ENOMEM; in fec_enet_alloc_txq_buffers()
3521 for (i = 0; i < fep->num_rx_queues; i++) in fec_enet_alloc_buffers()
3523 return -ENOMEM; in fec_enet_alloc_buffers()
3525 for (i = 0; i < fep->num_tx_queues; i++) in fec_enet_alloc_buffers()
3527 return -ENOMEM; in fec_enet_alloc_buffers()
3538 ret = pm_runtime_resume_and_get(&fep->pdev->dev); in fec_enet_open()
3542 pinctrl_pm_select_default_state(&fep->pdev->dev); in fec_enet_open()
3553 if (ndev->phydev && ndev->phydev->drv) in fec_enet_open()
3580 if (fep->quirks & FEC_QUIRK_ERR006687) in fec_enet_open()
3583 if (fep->quirks & FEC_QUIRK_HAS_PMQOS) in fec_enet_open()
3584 cpu_latency_qos_add_request(&fep->pm_qos_req, 0); in fec_enet_open()
3586 napi_enable(&fep->napi); in fec_enet_open()
3587 phy_start(ndev->phydev); in fec_enet_open()
3590 device_set_wakeup_enable(&ndev->dev, fep->wol_flag & in fec_enet_open()
3600 pm_runtime_mark_last_busy(&fep->pdev->dev); in fec_enet_open()
3601 pm_runtime_put_autosuspend(&fep->pdev->dev); in fec_enet_open()
3602 pinctrl_pm_select_sleep_state(&fep->pdev->dev); in fec_enet_open()
3611 phy_stop(ndev->phydev); in fec_enet_close()
3614 napi_disable(&fep->napi); in fec_enet_close()
3619 phy_disconnect(ndev->phydev); in fec_enet_close()
3621 if (fep->quirks & FEC_QUIRK_ERR006687) in fec_enet_close()
3627 if (fep->quirks & FEC_QUIRK_HAS_PMQOS) in fec_enet_close()
3628 cpu_latency_qos_remove_request(&fep->pm_qos_req); in fec_enet_close()
3630 pinctrl_pm_select_sleep_state(&fep->pdev->dev); in fec_enet_close()
3631 pm_runtime_mark_last_busy(&fep->pdev->dev); in fec_enet_close()
3632 pm_runtime_put_autosuspend(&fep->pdev->dev); in fec_enet_close()
3659 if (ndev->flags & IFF_PROMISC) { in set_multicast_list()
3660 tmp = readl(fep->hwp + FEC_R_CNTRL); in set_multicast_list()
3662 writel(tmp, fep->hwp + FEC_R_CNTRL); in set_multicast_list()
3666 tmp = readl(fep->hwp + FEC_R_CNTRL); in set_multicast_list()
3668 writel(tmp, fep->hwp + FEC_R_CNTRL); in set_multicast_list()
3670 if (ndev->flags & IFF_ALLMULTI) { in set_multicast_list()
3674 writel(0xffffffff, fep->hwp + FEC_GRP_HASH_TABLE_HIGH); in set_multicast_list()
3675 writel(0xffffffff, fep->hwp + FEC_GRP_HASH_TABLE_LOW); in set_multicast_list()
3683 crc = ether_crc_le(ndev->addr_len, ha->addr); in set_multicast_list()
3688 hash = (crc >> (32 - FEC_HASH_BITS)) & 0x3f; in set_multicast_list()
3691 hash_high |= 1 << (hash - 32); in set_multicast_list()
3696 writel(hash_high, fep->hwp + FEC_GRP_HASH_TABLE_HIGH); in set_multicast_list()
3697 writel(hash_low, fep->hwp + FEC_GRP_HASH_TABLE_LOW); in set_multicast_list()
3708 if (!is_valid_ether_addr(addr->sa_data)) in fec_set_mac_address()
3709 return -EADDRNOTAVAIL; in fec_set_mac_address()
3710 eth_hw_addr_set(ndev, addr->sa_data); in fec_set_mac_address()
3721 writel(ndev->dev_addr[3] | (ndev->dev_addr[2] << 8) | in fec_set_mac_address()
3722 (ndev->dev_addr[1] << 16) | (ndev->dev_addr[0] << 24), in fec_set_mac_address()
3723 fep->hwp + FEC_ADDR_LOW); in fec_set_mac_address()
3724 writel((ndev->dev_addr[5] << 16) | (ndev->dev_addr[4] << 24), in fec_set_mac_address()
3725 fep->hwp + FEC_ADDR_HIGH); in fec_set_mac_address()
3733 netdev_features_t changed = features ^ netdev->features; in fec_enet_set_netdev_features()
3735 netdev->features = features; in fec_enet_set_netdev_features()
3740 fep->csum_flags |= FLAG_RX_CSUM_ENABLED; in fec_enet_set_netdev_features()
3742 fep->csum_flags &= ~FLAG_RX_CSUM_ENABLED; in fec_enet_set_netdev_features()
3750 netdev_features_t changed = features ^ netdev->features; in fec_set_features()
3753 napi_disable(&fep->napi); in fec_set_features()
3760 napi_enable(&fep->napi); in fec_set_features()
3774 if (!(fep->quirks & FEC_QUIRK_HAS_AVB)) in fec_enet_select_queue()
3778 if (eth_type_vlan(skb->protocol)) { in fec_enet_select_queue()
3781 vlan_tag = ntohs(vhdr->h_vlan_TCI); in fec_enet_select_queue()
3784 vlan_tag = skb->vlan_tci; in fec_enet_select_queue()
3798 switch (bpf->command) { in fec_enet_bpf()
3804 if (fep->quirks & FEC_QUIRK_SWAP_FRAME) in fec_enet_bpf()
3805 return -EOPNOTSUPP; in fec_enet_bpf()
3807 if (!bpf->prog) in fec_enet_bpf()
3811 napi_disable(&fep->napi); in fec_enet_bpf()
3815 old_prog = xchg(&fep->xdp_prog, bpf->prog); in fec_enet_bpf()
3822 napi_enable(&fep->napi); in fec_enet_bpf()
3826 if (bpf->prog) in fec_enet_bpf()
3832 return -EOPNOTSUPP; in fec_enet_bpf()
3835 return -EOPNOTSUPP; in fec_enet_bpf()
3845 return (index % fep->num_tx_queues); in fec_enet_xdp_get_tx_queue()
3861 netdev_err_once(fep->netdev, "NOT enough BD for SG!\n"); in fec_enet_txq_xmit_frame()
3862 return -EBUSY; in fec_enet_txq_xmit_frame()
3866 bdp = txq->bd.cur; in fec_enet_txq_xmit_frame()
3867 status = fec16_to_cpu(bdp->cbd_sc); in fec_enet_txq_xmit_frame()
3870 index = fec_enet_get_bd_index(bdp, &txq->bd); in fec_enet_txq_xmit_frame()
3875 dma_addr = dma_map_single(&fep->pdev->dev, xdpf->data, in fec_enet_txq_xmit_frame()
3876 xdpf->len, DMA_TO_DEVICE); in fec_enet_txq_xmit_frame()
3877 if (dma_mapping_error(&fep->pdev->dev, dma_addr)) in fec_enet_txq_xmit_frame()
3878 return -ENOMEM; in fec_enet_txq_xmit_frame()
3880 frame_len = xdpf->len; in fec_enet_txq_xmit_frame()
3881 txq->tx_buf[index].buf_p = xdpf; in fec_enet_txq_xmit_frame()
3882 txq->tx_buf[index].type = FEC_TXBUF_T_XDP_NDO; in fec_enet_txq_xmit_frame()
3887 page = virt_to_page(xdpb->data); in fec_enet_txq_xmit_frame()
3889 (xdpb->data - xdpb->data_hard_start); in fec_enet_txq_xmit_frame()
3890 dma_sync_single_for_device(&fep->pdev->dev, dma_addr, in fec_enet_txq_xmit_frame()
3892 frame_len = xdpb->data_end - xdpb->data; in fec_enet_txq_xmit_frame()
3893 txq->tx_buf[index].buf_p = page; in fec_enet_txq_xmit_frame()
3894 txq->tx_buf[index].type = FEC_TXBUF_T_XDP_TX; in fec_enet_txq_xmit_frame()
3898 if (fep->bufdesc_ex) in fec_enet_txq_xmit_frame()
3901 bdp->cbd_bufaddr = cpu_to_fec32(dma_addr); in fec_enet_txq_xmit_frame()
3902 bdp->cbd_datlen = cpu_to_fec16(frame_len); in fec_enet_txq_xmit_frame()
3904 if (fep->bufdesc_ex) { in fec_enet_txq_xmit_frame()
3907 if (fep->quirks & FEC_QUIRK_HAS_AVB) in fec_enet_txq_xmit_frame()
3908 estatus |= FEC_TX_BD_FTYPE(txq->bd.qid); in fec_enet_txq_xmit_frame()
3910 ebdp->cbd_bdu = 0; in fec_enet_txq_xmit_frame()
3911 ebdp->cbd_esc = cpu_to_fec32(estatus); in fec_enet_txq_xmit_frame()
3923 bdp->cbd_sc = cpu_to_fec16(status); in fec_enet_txq_xmit_frame()
3926 bdp = fec_enet_get_nextdesc(bdp, &txq->bd); in fec_enet_txq_xmit_frame()
3928 /* Make sure the update to bdp are performed before txq->bd.cur. */ in fec_enet_txq_xmit_frame()
3931 txq->bd.cur = bdp; in fec_enet_txq_xmit_frame()
3934 writel(0, txq->bd.reg_desc_active); in fec_enet_txq_xmit_frame()
3948 txq = fep->tx_queue[queue]; in fec_enet_xdp_tx_xmit()
3949 nq = netdev_get_tx_queue(fep->netdev, queue); in fec_enet_xdp_tx_xmit()
3976 txq = fep->tx_queue[queue]; in fec_enet_xdp_xmit()
3977 nq = netdev_get_tx_queue(fep->netdev, queue); in fec_enet_xdp_xmit()
4000 return -EINVAL; in fec_hwtstamp_get()
4002 if (!fep->bufdesc_ex) in fec_hwtstamp_get()
4003 return -EOPNOTSUPP; in fec_hwtstamp_get()
4017 return -EINVAL; in fec_hwtstamp_set()
4019 if (!fep->bufdesc_ex) in fec_hwtstamp_set()
4020 return -EOPNOTSUPP; in fec_hwtstamp_set()
4061 unsigned dsize = fep->bufdesc_ex ? sizeof(struct bufdesc_ex) : in fec_enet_init()
4068 fep->rx_align = 0xf; in fec_enet_init()
4069 fep->tx_align = 0xf; in fec_enet_init()
4071 fep->rx_align = 0x3; in fec_enet_init()
4072 fep->tx_align = 0x3; in fec_enet_init()
4074 fep->rx_pkts_itr = FEC_ITR_ICFT_DEFAULT; in fec_enet_init()
4075 fep->tx_pkts_itr = FEC_ITR_ICFT_DEFAULT; in fec_enet_init()
4076 fep->rx_time_itr = FEC_ITR_ICTT_DEFAULT; in fec_enet_init()
4077 fep->tx_time_itr = FEC_ITR_ICTT_DEFAULT; in fec_enet_init()
4080 ret = dma_set_mask_and_coherent(&fep->pdev->dev, DMA_BIT_MASK(32)); in fec_enet_init()
4082 dev_warn(&fep->pdev->dev, "No suitable DMA available\n"); in fec_enet_init()
4090 bd_size = (fep->total_tx_ring_size + fep->total_rx_ring_size) * dsize; in fec_enet_init()
4093 cbd_base = dmam_alloc_coherent(&fep->pdev->dev, bd_size, &bd_dma, in fec_enet_init()
4096 ret = -ENOMEM; in fec_enet_init()
4106 for (i = 0; i < fep->num_rx_queues; i++) { in fec_enet_init()
4107 struct fec_enet_priv_rx_q *rxq = fep->rx_queue[i]; in fec_enet_init() local
4108 unsigned size = dsize * rxq->bd.ring_size; in fec_enet_init()
4110 rxq->bd.qid = i; in fec_enet_init()
4111 rxq->bd.base = cbd_base; in fec_enet_init()
4112 rxq->bd.cur = cbd_base; in fec_enet_init()
4113 rxq->bd.dma = bd_dma; in fec_enet_init()
4114 rxq->bd.dsize = dsize; in fec_enet_init()
4115 rxq->bd.dsize_log2 = dsize_log2; in fec_enet_init()
4116 rxq->bd.reg_desc_active = fep->hwp + offset_des_active_rxq[i]; in fec_enet_init()
4119 rxq->bd.last = (struct bufdesc *)(((void *)cbd_base) - dsize); in fec_enet_init()
4122 for (i = 0; i < fep->num_tx_queues; i++) { in fec_enet_init()
4123 struct fec_enet_priv_tx_q *txq = fep->tx_queue[i]; in fec_enet_init()
4124 unsigned size = dsize * txq->bd.ring_size; in fec_enet_init()
4126 txq->bd.qid = i; in fec_enet_init()
4127 txq->bd.base = cbd_base; in fec_enet_init()
4128 txq->bd.cur = cbd_base; in fec_enet_init()
4129 txq->bd.dma = bd_dma; in fec_enet_init()
4130 txq->bd.dsize = dsize; in fec_enet_init()
4131 txq->bd.dsize_log2 = dsize_log2; in fec_enet_init()
4132 txq->bd.reg_desc_active = fep->hwp + offset_des_active_txq[i]; in fec_enet_init()
4135 txq->bd.last = (struct bufdesc *)(((void *)cbd_base) - dsize); in fec_enet_init()
4140 ndev->watchdog_timeo = TX_TIMEOUT; in fec_enet_init()
4141 ndev->netdev_ops = &fec_netdev_ops; in fec_enet_init()
4142 ndev->ethtool_ops = &fec_enet_ethtool_ops; in fec_enet_init()
4144 writel(FEC_RX_DISABLED_IMASK, fep->hwp + FEC_IMASK); in fec_enet_init()
4145 netif_napi_add(ndev, &fep->napi, fec_enet_rx_napi); in fec_enet_init()
4147 if (fep->quirks & FEC_QUIRK_HAS_VLAN) in fec_enet_init()
4149 ndev->features |= NETIF_F_HW_VLAN_CTAG_RX; in fec_enet_init()
4151 if (fep->quirks & FEC_QUIRK_HAS_CSUM) { in fec_enet_init()
4155 ndev->features |= (NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM in fec_enet_init()
4157 fep->csum_flags |= FLAG_RX_CSUM_ENABLED; in fec_enet_init()
4160 if (fep->quirks & FEC_QUIRK_HAS_MULTI_QUEUES) { in fec_enet_init()
4161 fep->tx_align = 0; in fec_enet_init()
4162 fep->rx_align = 0x3f; in fec_enet_init()
4165 ndev->hw_features = ndev->features; in fec_enet_init()
4167 if (!(fep->quirks & FEC_QUIRK_SWAP_FRAME)) in fec_enet_init()
4168 ndev->xdp_features = NETDEV_XDP_ACT_BASIC | in fec_enet_init()
4173 if (fep->quirks & FEC_QUIRK_MIB_CLEAR) in fec_enet_init()
4189 netif_napi_del(&fep->napi); in fec_enet_deinit()
4198 struct device_node *np = pdev->dev.of_node; in fec_reset_phy()
4204 err = of_property_read_u32(np, "phy-reset-duration", &msec); in fec_reset_phy()
4209 err = of_property_read_u32(np, "phy-reset-post-delay", &phy_post_delay); in fec_reset_phy()
4212 return -EINVAL; in fec_reset_phy()
4214 phy_reset = devm_gpiod_get_optional(&pdev->dev, "phy-reset", in fec_reset_phy()
4217 return dev_err_probe(&pdev->dev, PTR_ERR(phy_reset), in fec_reset_phy()
4218 "failed to get phy-reset-gpios\n"); in fec_reset_phy()
4255 struct device_node *np = pdev->dev.of_node; in fec_enet_get_queue_num()
4262 /* parse the num of tx and rx queues */ in fec_enet_get_queue_num()
4263 of_property_read_u32(np, "fsl,num-tx-queues", num_tx); in fec_enet_get_queue_num()
4265 of_property_read_u32(np, "fsl,num-rx-queues", num_rx); in fec_enet_get_queue_num()
4268 dev_warn(&pdev->dev, "Invalid num_tx(=%d), fall back to 1\n", in fec_enet_get_queue_num()
4275 dev_warn(&pdev->dev, "Invalid num_rx(=%d), fall back to 1\n", in fec_enet_get_queue_num()
4301 if (fep->quirks & FEC_QUIRK_WAKEUP_FROM_INT2) in fec_enet_get_wakeup_irq()
4302 fep->wake_irq = fep->irq[2]; in fec_enet_get_wakeup_irq()
4304 fep->wake_irq = fep->irq[0]; in fec_enet_get_wakeup_irq()
4314 gpr_np = of_parse_phandle(np, "fsl,stop-mode", 0); in fec_enet_init_stop_mode()
4318 ret = of_property_read_u32_array(np, "fsl,stop-mode", out_val, in fec_enet_init_stop_mode()
4321 dev_dbg(&fep->pdev->dev, "no stop mode property\n"); in fec_enet_init_stop_mode()
4325 fep->stop_gpr.gpr = syscon_node_to_regmap(gpr_np); in fec_enet_init_stop_mode()
4326 if (IS_ERR(fep->stop_gpr.gpr)) { in fec_enet_init_stop_mode()
4327 dev_err(&fep->pdev->dev, "could not find gpr regmap\n"); in fec_enet_init_stop_mode()
4328 ret = PTR_ERR(fep->stop_gpr.gpr); in fec_enet_init_stop_mode()
4329 fep->stop_gpr.gpr = NULL; in fec_enet_init_stop_mode()
4333 fep->stop_gpr.reg = out_val[1]; in fec_enet_init_stop_mode()
4334 fep->stop_gpr.bit = out_val[2]; in fec_enet_init_stop_mode()
4352 struct device_node *np = pdev->dev.of_node, *phy_node; in fec_probe()
4365 return -ENOMEM; in fec_probe()
4367 SET_NETDEV_DEV(ndev, &pdev->dev); in fec_probe()
4372 of_id = of_match_device(fec_dt_ids, &pdev->dev); in fec_probe()
4374 pdev->id_entry = of_id->data; in fec_probe()
4375 dev_info = (struct fec_devinfo *)pdev->id_entry->driver_data; in fec_probe()
4377 fep->quirks = dev_info->quirks; in fec_probe()
4379 fep->netdev = ndev; in fec_probe()
4380 fep->num_rx_queues = num_rx_qs; in fec_probe()
4381 fep->num_tx_queues = num_tx_qs; in fec_probe()
4385 if (fep->quirks & FEC_QUIRK_HAS_GBIT) in fec_probe()
4386 fep->pause_flag |= FEC_PAUSE_FLAG_AUTONEG; in fec_probe()
4390 pinctrl_pm_select_default_state(&pdev->dev); in fec_probe()
4392 fep->hwp = devm_platform_ioremap_resource(pdev, 0); in fec_probe()
4393 if (IS_ERR(fep->hwp)) { in fec_probe()
4394 ret = PTR_ERR(fep->hwp); in fec_probe()
4398 fep->pdev = pdev; in fec_probe()
4399 fep->dev_id = dev_id++; in fec_probe()
4405 !of_property_read_bool(np, "fsl,err006687-workaround-present")) in fec_probe()
4406 fep->quirks |= FEC_QUIRK_ERR006687; in fec_probe()
4412 if (of_property_read_bool(np, "fsl,magic-packet")) in fec_probe()
4413 fep->wol_flag |= FEC_WOL_HAS_MAGIC_PACKET; in fec_probe()
4419 phy_node = of_parse_phandle(np, "phy-handle", 0); in fec_probe()
4423 dev_err(&pdev->dev, in fec_probe()
4424 "broken fixed-link specification\n"); in fec_probe()
4429 fep->phy_node = phy_node; in fec_probe()
4431 ret = of_get_phy_mode(pdev->dev.of_node, &interface); in fec_probe()
4433 pdata = dev_get_platdata(&pdev->dev); in fec_probe()
4435 fep->phy_interface = pdata->phy; in fec_probe()
4437 fep->phy_interface = PHY_INTERFACE_MODE_MII; in fec_probe()
4439 fep->phy_interface = interface; in fec_probe()
4446 fep->clk_ipg = devm_clk_get(&pdev->dev, "ipg"); in fec_probe()
4447 if (IS_ERR(fep->clk_ipg)) { in fec_probe()
4448 ret = PTR_ERR(fep->clk_ipg); in fec_probe()
4452 fep->clk_ahb = devm_clk_get(&pdev->dev, "ahb"); in fec_probe()
4453 if (IS_ERR(fep->clk_ahb)) { in fec_probe()
4454 ret = PTR_ERR(fep->clk_ahb); in fec_probe()
4458 fep->itr_clk_rate = clk_get_rate(fep->clk_ahb); in fec_probe()
4461 fep->clk_enet_out = devm_clk_get_optional(&pdev->dev, "enet_out"); in fec_probe()
4462 if (IS_ERR(fep->clk_enet_out)) { in fec_probe()
4463 ret = PTR_ERR(fep->clk_enet_out); in fec_probe()
4467 fep->ptp_clk_on = false; in fec_probe()
4468 mutex_init(&fep->ptp_clk_mutex); in fec_probe()
4471 fep->clk_ref = devm_clk_get_optional(&pdev->dev, "enet_clk_ref"); in fec_probe()
4472 if (IS_ERR(fep->clk_ref)) { in fec_probe()
4473 ret = PTR_ERR(fep->clk_ref); in fec_probe()
4476 fep->clk_ref_rate = clk_get_rate(fep->clk_ref); in fec_probe()
4479 if (fep->rgmii_txc_dly || fep->rgmii_rxc_dly) { in fec_probe()
4480 fep->clk_2x_txclk = devm_clk_get(&pdev->dev, "enet_2x_txclk"); in fec_probe()
4481 if (IS_ERR(fep->clk_2x_txclk)) in fec_probe()
4482 fep->clk_2x_txclk = NULL; in fec_probe()
4485 fep->bufdesc_ex = fep->quirks & FEC_QUIRK_HAS_BUFDESC_EX; in fec_probe()
4486 fep->clk_ptp = devm_clk_get(&pdev->dev, "ptp"); in fec_probe()
4487 if (IS_ERR(fep->clk_ptp)) { in fec_probe()
4488 fep->clk_ptp = NULL; in fec_probe()
4489 fep->bufdesc_ex = false; in fec_probe()
4496 ret = clk_prepare_enable(fep->clk_ipg); in fec_probe()
4499 ret = clk_prepare_enable(fep->clk_ahb); in fec_probe()
4503 fep->reg_phy = devm_regulator_get_optional(&pdev->dev, "phy"); in fec_probe()
4504 if (!IS_ERR(fep->reg_phy)) { in fec_probe()
4505 ret = regulator_enable(fep->reg_phy); in fec_probe()
4507 dev_err(&pdev->dev, in fec_probe()
4512 if (PTR_ERR(fep->reg_phy) == -EPROBE_DEFER) { in fec_probe()
4513 ret = -EPROBE_DEFER; in fec_probe()
4516 fep->reg_phy = NULL; in fec_probe()
4519 pm_runtime_set_autosuspend_delay(&pdev->dev, FEC_MDIO_PM_TIMEOUT); in fec_probe()
4520 pm_runtime_use_autosuspend(&pdev->dev); in fec_probe()
4521 pm_runtime_get_noresume(&pdev->dev); in fec_probe()
4522 pm_runtime_set_active(&pdev->dev); in fec_probe()
4523 pm_runtime_enable(&pdev->dev); in fec_probe()
4530 if (fep->bufdesc_ex) in fec_probe()
4546 ret = devm_request_irq(&pdev->dev, irq, fec_enet_interrupt, in fec_probe()
4547 0, pdev->name, ndev); in fec_probe()
4551 fep->irq[i] = irq; in fec_probe()
4564 pinctrl_pm_select_sleep_state(&pdev->dev); in fec_probe()
4566 ndev->max_mtu = PKT_MAXBUF_SIZE - ETH_HLEN - ETH_FCS_LEN; in fec_probe()
4572 device_init_wakeup(&ndev->dev, fep->wol_flag & in fec_probe()
4575 if (fep->bufdesc_ex && fep->ptp_clock) in fec_probe()
4576 netdev_info(ndev, "registered PHC device %d\n", fep->dev_id); in fec_probe()
4578 INIT_WORK(&fep->tx_timeout_work, fec_enet_timeout_work); in fec_probe()
4580 pm_runtime_mark_last_busy(&pdev->dev); in fec_probe()
4581 pm_runtime_put_autosuspend(&pdev->dev); in fec_probe()
4593 pm_runtime_put_noidle(&pdev->dev); in fec_probe()
4594 pm_runtime_disable(&pdev->dev); in fec_probe()
4595 if (fep->reg_phy) in fec_probe()
4596 regulator_disable(fep->reg_phy); in fec_probe()
4598 clk_disable_unprepare(fep->clk_ahb); in fec_probe()
4600 clk_disable_unprepare(fep->clk_ipg); in fec_probe()
4611 dev_id--; in fec_probe()
4623 struct device_node *np = pdev->dev.of_node; in fec_drv_remove()
4626 ret = pm_runtime_get_sync(&pdev->dev); in fec_drv_remove()
4628 dev_err(&pdev->dev, in fec_drv_remove()
4632 cancel_work_sync(&fep->tx_timeout_work); in fec_drv_remove()
4636 if (fep->reg_phy) in fec_drv_remove()
4637 regulator_disable(fep->reg_phy); in fec_drv_remove()
4641 of_node_put(fep->phy_node); in fec_drv_remove()
4647 clk_disable_unprepare(fep->clk_ahb); in fec_drv_remove()
4648 clk_disable_unprepare(fep->clk_ipg); in fec_drv_remove()
4650 pm_runtime_put_noidle(&pdev->dev); in fec_drv_remove()
4651 pm_runtime_disable(&pdev->dev); in fec_drv_remove()
4665 if (fep->wol_flag & FEC_WOL_FLAG_ENABLE) in fec_suspend()
4666 fep->wol_flag |= FEC_WOL_FLAG_SLEEP_ON; in fec_suspend()
4667 phy_stop(ndev->phydev); in fec_suspend()
4668 napi_disable(&fep->napi); in fec_suspend()
4673 if (!(fep->wol_flag & FEC_WOL_FLAG_ENABLE)) { in fec_suspend()
4675 pinctrl_pm_select_sleep_state(&fep->pdev->dev); in fec_suspend()
4678 if (fep->wake_irq > 0) { in fec_suspend()
4679 disable_irq(fep->wake_irq); in fec_suspend()
4680 enable_irq_wake(fep->wake_irq); in fec_suspend()
4687 fep->rpm_active = !pm_runtime_status_suspended(dev); in fec_suspend()
4688 if (fep->rpm_active) { in fec_suspend()
4698 if (fep->reg_phy && !(fep->wol_flag & FEC_WOL_FLAG_ENABLE)) in fec_suspend()
4699 regulator_disable(fep->reg_phy); in fec_suspend()
4704 if (fep->clk_enet_out || fep->reg_phy) in fec_suspend()
4705 fep->link = 0; in fec_suspend()
4717 if (fep->reg_phy && !(fep->wol_flag & FEC_WOL_FLAG_ENABLE)) { in fec_resume()
4718 ret = regulator_enable(fep->reg_phy); in fec_resume()
4725 if (fep->rpm_active) in fec_resume()
4733 if (fep->wol_flag & FEC_WOL_FLAG_ENABLE) { in fec_resume()
4735 if (fep->wake_irq) { in fec_resume()
4736 disable_irq_wake(fep->wake_irq); in fec_resume()
4737 enable_irq(fep->wake_irq); in fec_resume()
4740 val = readl(fep->hwp + FEC_ECNTRL); in fec_resume()
4742 writel(val, fep->hwp + FEC_ECNTRL); in fec_resume()
4743 fep->wol_flag &= ~FEC_WOL_FLAG_SLEEP_ON; in fec_resume()
4745 pinctrl_pm_select_default_state(&fep->pdev->dev); in fec_resume()
4751 napi_enable(&fep->napi); in fec_resume()
4752 phy_init_hw(ndev->phydev); in fec_resume()
4753 phy_start(ndev->phydev); in fec_resume()
4760 if (fep->reg_phy) in fec_resume()
4761 regulator_disable(fep->reg_phy); in fec_resume()
4770 clk_disable_unprepare(fep->clk_ahb); in fec_runtime_suspend()
4771 clk_disable_unprepare(fep->clk_ipg); in fec_runtime_suspend()
4782 ret = clk_prepare_enable(fep->clk_ahb); in fec_runtime_resume()
4785 ret = clk_prepare_enable(fep->clk_ipg); in fec_runtime_resume()
4792 clk_disable_unprepare(fep->clk_ahb); in fec_runtime_resume()