1 // SPDX-License-Identifier: GPL-2.0-only
5  * Copyright (C) 2004-2006 Atmel Corporation
10 #include <linux/clk-provider.h>
25 #include <linux/dma-mapping.h>
40 #include <linux/firmware/xlnx-zynqmp.h>
46 	unsigned long rate;  member
57 				 * (bp)->rx_ring_size)
63 				 * (bp)->tx_ring_size)
66 #define MACB_TX_WAKEUP_THRESH(bp)	(3 * (bp)->tx_ring_size / 4)
77 …MAX_TX_LEN		((unsigned int)((1 << MACB_TX_FRMLEN_SIZE) - 1) & ~((unsigned int)(MACB_TX_LEN_ALIGN -…
94  * 1 frame time (10 Mbits/s, full-duplex, ignoring collisions)
97 #define MACB_PM_TIMEOUT  100 /* ms */
133 	switch (bp->hw_dma_cap) {  in macb_dma_desc_get_size()
158 	switch (bp->hw_dma_cap) {  in macb_adj_dma_desc_idx()
184 	return index & (bp->tx_ring_size - 1);  in macb_tx_ring_wrap()
190 	index = macb_tx_ring_wrap(queue->bp, index);  in macb_tx_desc()
191 	index = macb_adj_dma_desc_idx(queue->bp, index);  in macb_tx_desc()
192 	return &queue->tx_ring[index];  in macb_tx_desc()
198 	return &queue->tx_skb[macb_tx_ring_wrap(queue->bp, index)];  in macb_tx_skb()
205 	offset = macb_tx_ring_wrap(queue->bp, index) *  in macb_tx_dma()
206 			macb_dma_desc_get_size(queue->bp);  in macb_tx_dma()
208 	return queue->tx_ring_dma + offset;  in macb_tx_dma()
213 	return index & (bp->rx_ring_size - 1);  in macb_rx_ring_wrap()
218 	index = macb_rx_ring_wrap(queue->bp, index);  in macb_rx_desc()
219 	index = macb_adj_dma_desc_idx(queue->bp, index);  in macb_rx_desc()
220 	return &queue->rx_ring[index];  in macb_rx_desc()
225 	return queue->rx_buffers + queue->bp->rx_buffer_size *  in macb_rx_buffer()
226 	       macb_rx_ring_wrap(queue->bp, index);  in macb_rx_buffer()
232 	return __raw_readl(bp->regs + offset);  in hw_readl_native()
237 	__raw_writel(value, bp->regs + offset);  in hw_writel_native()
242 	return readl_relaxed(bp->regs + offset);  in hw_readl()
247 	writel_relaxed(value, bp->regs + offset);  in hw_writel()
284 	bottom = cpu_to_le32(*((u32 *)bp->dev->dev_addr));  in macb_set_hwaddr()
286 	top = cpu_to_le16(*((u16 *)(bp->dev->dev_addr + 4)));  in macb_set_hwaddr()
323 			eth_hw_addr_set(bp->dev, addr);  in macb_get_hwaddr()
328 	dev_info(&bp->pdev->dev, "invalid hw address, using random\n");  in macb_get_hwaddr()
329 	eth_hw_addr_random(bp->dev);  in macb_get_hwaddr()
342 	struct macb *bp = bus->priv;  in macb_mdio_read_c22()
345 	status = pm_runtime_resume_and_get(&bp->pdev->dev);  in macb_mdio_read_c22()
366 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_read_c22()
367 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_read_c22()
375 	struct macb *bp = bus->priv;  in macb_mdio_read_c45()
378 	status = pm_runtime_get_sync(&bp->pdev->dev);  in macb_mdio_read_c45()
380 		pm_runtime_put_noidle(&bp->pdev->dev);  in macb_mdio_read_c45()
412 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_read_c45()
413 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_read_c45()
421 	struct macb *bp = bus->priv;  in macb_mdio_write_c22()
424 	status = pm_runtime_resume_and_get(&bp->pdev->dev);  in macb_mdio_write_c22()
444 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_write_c22()
445 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_write_c22()
454 	struct macb *bp = bus->priv;  in macb_mdio_write_c45()
457 	status = pm_runtime_get_sync(&bp->pdev->dev);  in macb_mdio_write_c45()
459 		pm_runtime_put_noidle(&bp->pdev->dev);  in macb_mdio_write_c45()
490 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_mdio_write_c45()
491 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_mdio_write_c45()
501 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_init_buffers()
502 		queue_writel(queue, RBQP, lower_32_bits(queue->rx_ring_dma));  in macb_init_buffers()
504 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_init_buffers()
506 				     upper_32_bits(queue->rx_ring_dma));  in macb_init_buffers()
508 		queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));  in macb_init_buffers()
510 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_init_buffers()
512 				     upper_32_bits(queue->tx_ring_dma));  in macb_init_buffers()
518  * macb_set_tx_clk() - Set a clock to a new frequency
524 	long ferr, rate, rate_rounded;  in macb_set_tx_clk()  local
526 	if (!bp->tx_clk || (bp->caps & MACB_CAPS_CLK_HW_CHG))  in macb_set_tx_clk()
530 	if (bp->phy_interface == PHY_INTERFACE_MODE_MII)  in macb_set_tx_clk()
535 		rate = 2500000;  in macb_set_tx_clk()
538 		rate = 25000000;  in macb_set_tx_clk()
541 		rate = 125000000;  in macb_set_tx_clk()
547 	rate_rounded = clk_round_rate(bp->tx_clk, rate);  in macb_set_tx_clk()
554 	ferr = abs(rate_rounded - rate);  in macb_set_tx_clk()
555 	ferr = DIV_ROUND_UP(ferr, rate / 100000);  in macb_set_tx_clk()
557 		netdev_warn(bp->dev,  in macb_set_tx_clk()
559 			    rate);  in macb_set_tx_clk()
561 	if (clk_set_rate(bp->tx_clk, rate_rounded))  in macb_set_tx_clk()
562 		netdev_err(bp->dev, "adjusting tx_clk failed.\n");  in macb_set_tx_clk()
586 	state->speed = SPEED_10000;  in macb_usx_pcs_get_state()
587 	state->duplex = 1;  in macb_usx_pcs_get_state()
588 	state->an_complete = 1;  in macb_usx_pcs_get_state()
591 	state->link = !!(val & GEM_BIT(USX_BLOCK_LOCK));  in macb_usx_pcs_get_state()
594 		state->pause = MLO_PAUSE_RX;  in macb_usx_pcs_get_state()
614 	state->link = 0;  in macb_pcs_get_state()
646 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_config()
652 	spin_lock_irqsave(&bp->lock, flags);  in macb_mac_config()
657 	if (bp->caps & MACB_CAPS_MACB_IS_EMAC) {  in macb_mac_config()
658 		if (state->interface == PHY_INTERFACE_MODE_RMII)  in macb_mac_config()
664 		if (state->interface == PHY_INTERFACE_MODE_SGMII) {  in macb_mac_config()
666 		} else if (state->interface == PHY_INTERFACE_MODE_10GBASER) {  in macb_mac_config()
669 		} else if (bp->caps & MACB_CAPS_MIIONRGMII &&  in macb_mac_config()
670 			   bp->phy_interface == PHY_INTERFACE_MODE_MII) {  in macb_mac_config()
686 	if (macb_is_gem(bp) && state->interface == PHY_INTERFACE_MODE_SGMII) {  in macb_mac_config()
698 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_mac_config()
704 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_link_down()
710 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))  in macb_mac_link_down()
711 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_mac_link_down()
713 				     bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));  in macb_mac_link_down()
728 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_link_up()
735 	spin_lock_irqsave(&bp->lock, flags);  in macb_mac_link_up()
747 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC)) {  in macb_mac_link_up()
762 		bp->macbgem_ops.mog_init_rings(bp);  in macb_mac_link_up()
765 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_mac_link_up()
767 				     bp->rx_intr_mask | MACB_TX_INT_FLAGS | MACB_BIT(HRESP));  in macb_mac_link_up()
772 	if (bp->phy_interface == PHY_INTERFACE_MODE_10GBASER)  in macb_mac_link_up()
776 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_mac_link_up()
778 	if (!(bp->caps & MACB_CAPS_MACB_IS_EMAC))  in macb_mac_link_up()
794 	struct net_device *ndev = to_net_dev(config->dev);  in macb_mac_select_pcs()
798 		return &bp->phylink_usx_pcs;  in macb_mac_select_pcs()
800 		return &bp->phylink_sgmii_pcs;  in macb_mac_select_pcs()
814 	dn = of_parse_phandle(dn, "phy-handle", 0);  in macb_phy_handle_exists()
821 	struct device_node *dn = bp->pdev->dev.of_node;  in macb_phylink_connect()
822 	struct net_device *dev = bp->dev;  in macb_phylink_connect()
827 		ret = phylink_of_phy_connect(bp->phylink, dn, 0);  in macb_phylink_connect()
830 		phydev = phy_find_first(bp->mii_bus);  in macb_phylink_connect()
833 			return -ENXIO;  in macb_phylink_connect()
837 		ret = phylink_connect_phy(bp->phylink, phydev);  in macb_phylink_connect()
845 	phylink_start(bp->phylink);  in macb_phylink_connect()
853 	struct net_device *ndev = to_net_dev(config->dev);  in macb_get_pcs_fixed_state()
856 	state->link = (macb_readl(bp, NSR) & MACB_BIT(NSR_LINK)) != 0;  in macb_get_pcs_fixed_state()
864 	bp->phylink_sgmii_pcs.ops = &macb_phylink_pcs_ops;  in macb_mii_probe()
865 	bp->phylink_sgmii_pcs.neg_mode = true;  in macb_mii_probe()
866 	bp->phylink_usx_pcs.ops = &macb_phylink_usx_pcs_ops;  in macb_mii_probe()
867 	bp->phylink_usx_pcs.neg_mode = true;  in macb_mii_probe()
869 	bp->phylink_config.dev = &dev->dev;  in macb_mii_probe()
870 	bp->phylink_config.type = PHYLINK_NETDEV;  in macb_mii_probe()
871 	bp->phylink_config.mac_managed_pm = true;  in macb_mii_probe()
873 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {  in macb_mii_probe()
874 		bp->phylink_config.poll_fixed_state = true;  in macb_mii_probe()
875 		bp->phylink_config.get_fixed_state = macb_get_pcs_fixed_state;  in macb_mii_probe()
878 	bp->phylink_config.mac_capabilities = MAC_ASYM_PAUSE |  in macb_mii_probe()
882 		  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
884 		  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
887 	if (macb_is_gem(bp) && (bp->caps & MACB_CAPS_GIGABIT_MODE_AVAILABLE)) {  in macb_mii_probe()
888 		bp->phylink_config.mac_capabilities |= MAC_1000FD;  in macb_mii_probe()
889 		if (!(bp->caps & MACB_CAPS_NO_GIGABIT_HALF))  in macb_mii_probe()
890 			bp->phylink_config.mac_capabilities |= MAC_1000HD;  in macb_mii_probe()
893 			  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
894 		phy_interface_set_rgmii(bp->phylink_config.supported_interfaces);  in macb_mii_probe()
896 		if (bp->caps & MACB_CAPS_PCS)  in macb_mii_probe()
898 				  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
900 		if (bp->caps & MACB_CAPS_HIGH_SPEED) {  in macb_mii_probe()
902 				  bp->phylink_config.supported_interfaces);  in macb_mii_probe()
903 			bp->phylink_config.mac_capabilities |= MAC_10000FD;  in macb_mii_probe()
907 	bp->phylink = phylink_create(&bp->phylink_config, bp->pdev->dev.fwnode,  in macb_mii_probe()
908 				     bp->phy_interface, &macb_phylink_ops);  in macb_mii_probe()
909 	if (IS_ERR(bp->phylink)) {  in macb_mii_probe()
911 			   PTR_ERR(bp->phylink));  in macb_mii_probe()
912 		return PTR_ERR(bp->phylink);  in macb_mii_probe()
920 	struct device_node *child, *np = bp->pdev->dev.of_node;  in macb_mdiobus_register()
927 		int ret = of_mdiobus_register(bp->mii_bus, child);  in macb_mdiobus_register()
945 			return of_mdiobus_register(bp->mii_bus, np);  in macb_mdiobus_register()
948 	return mdiobus_register(bp->mii_bus);  in macb_mdiobus_register()
953 	struct device_node *child, *np = bp->pdev->dev.of_node;  in macb_mii_init()
954 	int err = -ENXIO;  in macb_mii_init()
956 	/* With fixed-link, we don't need to register the MDIO bus,  in macb_mii_init()
964 		return macb_mii_probe(bp->dev);  in macb_mii_init()
969 	bp->mii_bus = mdiobus_alloc();  in macb_mii_init()
970 	if (!bp->mii_bus) {  in macb_mii_init()
971 		err = -ENOMEM;  in macb_mii_init()
975 	bp->mii_bus->name = "MACB_mii_bus";  in macb_mii_init()
976 	bp->mii_bus->read = &macb_mdio_read_c22;  in macb_mii_init()
977 	bp->mii_bus->write = &macb_mdio_write_c22;  in macb_mii_init()
978 	bp->mii_bus->read_c45 = &macb_mdio_read_c45;  in macb_mii_init()
979 	bp->mii_bus->write_c45 = &macb_mdio_write_c45;  in macb_mii_init()
980 	snprintf(bp->mii_bus->id, MII_BUS_ID_SIZE, "%s-%x",  in macb_mii_init()
981 		 bp->pdev->name, bp->pdev->id);  in macb_mii_init()
982 	bp->mii_bus->priv = bp;  in macb_mii_init()
983 	bp->mii_bus->parent = &bp->pdev->dev;  in macb_mii_init()
985 	dev_set_drvdata(&bp->dev->dev, bp->mii_bus);  in macb_mii_init()
991 	err = macb_mii_probe(bp->dev);  in macb_mii_init()
998 	mdiobus_unregister(bp->mii_bus);  in macb_mii_init()
1000 	mdiobus_free(bp->mii_bus);  in macb_mii_init()
1007 	u32 *p = &bp->hw_stats.macb.rx_pause_frames;  in macb_update_stats()
1008 	u32 *end = &bp->hw_stats.macb.tx_pause_frames + 1;  in macb_update_stats()
1011 	WARN_ON((unsigned long)(end - p - 1) != (MACB_TPF - MACB_PFR) / 4);  in macb_update_stats()
1014 		*p += bp->macb_reg_readl(bp, offset);  in macb_update_stats()
1034 	return -ETIMEDOUT;  in macb_halt_tx()
1039 	if (tx_skb->mapping) {  in macb_tx_unmap()
1040 		if (tx_skb->mapped_as_page)  in macb_tx_unmap()
1041 			dma_unmap_page(&bp->pdev->dev, tx_skb->mapping,  in macb_tx_unmap()
1042 				       tx_skb->size, DMA_TO_DEVICE);  in macb_tx_unmap()
1044 			dma_unmap_single(&bp->pdev->dev, tx_skb->mapping,  in macb_tx_unmap()
1045 					 tx_skb->size, DMA_TO_DEVICE);  in macb_tx_unmap()
1046 		tx_skb->mapping = 0;  in macb_tx_unmap()
1049 	if (tx_skb->skb) {  in macb_tx_unmap()
1050 		napi_consume_skb(tx_skb->skb, budget);  in macb_tx_unmap()
1051 		tx_skb->skb = NULL;  in macb_tx_unmap()
1060 	if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_set_addr()
1062 		desc_64->addrh = upper_32_bits(addr);  in macb_set_addr()
1070 	desc->addr = lower_32_bits(addr);  in macb_set_addr()
1079 	if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_get_addr()
1081 		addr = ((u64)(desc_64->addrh) << 32);  in macb_get_addr()
1084 	addr |= MACB_BF(RX_WADDR, MACB_BFEXT(RX_WADDR, desc->addr));  in macb_get_addr()
1086 	if (bp->hw_dma_cap & HW_DMA_CAP_PTP)  in macb_get_addr()
1097 	struct macb		*bp = queue->bp;  in macb_tx_error_task()
1104 	netdev_vdbg(bp->dev, "macb_tx_error_task: q = %u, t = %u, h = %u\n",  in macb_tx_error_task()
1105 		    (unsigned int)(queue - bp->queues),  in macb_tx_error_task()
1106 		    queue->tx_tail, queue->tx_head);  in macb_tx_error_task()
1114 	napi_disable(&queue->napi_tx);  in macb_tx_error_task()
1115 	spin_lock_irqsave(&bp->lock, flags);  in macb_tx_error_task()
1118 	netif_tx_stop_all_queues(bp->dev);  in macb_tx_error_task()
1125 		netdev_err(bp->dev, "BUG: halt tx timed out\n");  in macb_tx_error_task()
1133 	for (tail = queue->tx_tail; tail != queue->tx_head; tail++) {  in macb_tx_error_task()
1137 		ctrl = desc->ctrl;  in macb_tx_error_task()
1139 		skb = tx_skb->skb;  in macb_tx_error_task()
1147 				skb = tx_skb->skb;  in macb_tx_error_task()
1154 				netdev_vdbg(bp->dev, "txerr skb %u (data %p) TX complete\n",  in macb_tx_error_task()
1156 					    skb->data);  in macb_tx_error_task()
1157 				bp->dev->stats.tx_packets++;  in macb_tx_error_task()
1158 				queue->stats.tx_packets++;  in macb_tx_error_task()
1159 				bp->dev->stats.tx_bytes += skb->len;  in macb_tx_error_task()
1160 				queue->stats.tx_bytes += skb->len;  in macb_tx_error_task()
1163 			/* "Buffers exhausted mid-frame" errors may only happen  in macb_tx_error_task()
1168 				netdev_err(bp->dev,  in macb_tx_error_task()
1169 					   "BUG: TX buffers exhausted mid-frame\n");  in macb_tx_error_task()
1171 			desc->ctrl = ctrl | MACB_BIT(TX_USED);  in macb_tx_error_task()
1180 	desc->ctrl = MACB_BIT(TX_USED);  in macb_tx_error_task()
1186 	queue_writel(queue, TBQP, lower_32_bits(queue->tx_ring_dma));  in macb_tx_error_task()
1188 	if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_tx_error_task()
1189 		queue_writel(queue, TBQPH, upper_32_bits(queue->tx_ring_dma));  in macb_tx_error_task()
1192 	queue->tx_head = 0;  in macb_tx_error_task()
1193 	queue->tx_tail = 0;  in macb_tx_error_task()
1203 	netif_tx_start_all_queues(bp->dev);  in macb_tx_error_task()
1206 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_tx_error_task()
1207 	napi_enable(&queue->napi_tx);  in macb_tx_error_task()
1217 	if (likely(!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)))  in ptp_one_step_sync()
1229 	if (hdr->flag_field[0] & PTP_FLAG_TWOSTEP)  in ptp_one_step_sync()
1242 	struct macb *bp = queue->bp;  in macb_tx_complete()
1243 	u16 queue_index = queue - bp->queues;  in macb_tx_complete()
1248 	spin_lock(&queue->tx_ptr_lock);  in macb_tx_complete()
1249 	head = queue->tx_head;  in macb_tx_complete()
1250 	for (tail = queue->tx_tail; tail != head && packets < budget; tail++) {  in macb_tx_complete()
1261 		ctrl = desc->ctrl;  in macb_tx_complete()
1272 			skb = tx_skb->skb;  in macb_tx_complete()
1276 				if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&  in macb_tx_complete()
1280 				netdev_vdbg(bp->dev, "skb %u (data %p) TX complete\n",  in macb_tx_complete()
1282 					    skb->data);  in macb_tx_complete()
1283 				bp->dev->stats.tx_packets++;  in macb_tx_complete()
1284 				queue->stats.tx_packets++;  in macb_tx_complete()
1285 				bp->dev->stats.tx_bytes += skb->len;  in macb_tx_complete()
1286 				queue->stats.tx_bytes += skb->len;  in macb_tx_complete()
1302 	queue->tx_tail = tail;  in macb_tx_complete()
1303 	if (__netif_subqueue_stopped(bp->dev, queue_index) &&  in macb_tx_complete()
1304 	    CIRC_CNT(queue->tx_head, queue->tx_tail,  in macb_tx_complete()
1305 		     bp->tx_ring_size) <= MACB_TX_WAKEUP_THRESH(bp))  in macb_tx_complete()
1306 		netif_wake_subqueue(bp->dev, queue_index);  in macb_tx_complete()
1307 	spin_unlock(&queue->tx_ptr_lock);  in macb_tx_complete()
1317 	struct macb *bp = queue->bp;  in gem_rx_refill()
1320 	while (CIRC_SPACE(queue->rx_prepared_head, queue->rx_tail,  in gem_rx_refill()
1321 			bp->rx_ring_size) > 0) {  in gem_rx_refill()
1322 		entry = macb_rx_ring_wrap(bp, queue->rx_prepared_head);  in gem_rx_refill()
1329 		if (!queue->rx_skbuff[entry]) {  in gem_rx_refill()
1331 			skb = netdev_alloc_skb(bp->dev, bp->rx_buffer_size);  in gem_rx_refill()
1333 				netdev_err(bp->dev,  in gem_rx_refill()
1339 			paddr = dma_map_single(&bp->pdev->dev, skb->data,  in gem_rx_refill()
1340 					       bp->rx_buffer_size,  in gem_rx_refill()
1342 			if (dma_mapping_error(&bp->pdev->dev, paddr)) {  in gem_rx_refill()
1347 			queue->rx_skbuff[entry] = skb;  in gem_rx_refill()
1349 			if (entry == bp->rx_ring_size - 1)  in gem_rx_refill()
1351 			desc->ctrl = 0;  in gem_rx_refill()
1361 			desc->ctrl = 0;  in gem_rx_refill()
1363 			desc->addr &= ~MACB_BIT(RX_USED);  in gem_rx_refill()
1365 		queue->rx_prepared_head++;  in gem_rx_refill()
1371 	netdev_vdbg(bp->dev, "rx ring: queue: %p, prepared head %d, tail %d\n",  in gem_rx_refill()
1372 			queue, queue->rx_prepared_head, queue->rx_tail);  in gem_rx_refill()
1384 		desc->addr &= ~MACB_BIT(RX_USED);  in discard_partial_frame()
1399 	struct macb *bp = queue->bp;  in gem_rx()
1411 		entry = macb_rx_ring_wrap(bp, queue->rx_tail);  in gem_rx()
1417 		rxused = (desc->addr & MACB_BIT(RX_USED)) ? true : false;  in gem_rx()
1423 		/* Ensure ctrl is at least as up-to-date as rxused */  in gem_rx()
1426 		ctrl = desc->ctrl;  in gem_rx()
1428 		queue->rx_tail++;  in gem_rx()
1432 			netdev_err(bp->dev,  in gem_rx()
1434 			bp->dev->stats.rx_dropped++;  in gem_rx()
1435 			queue->stats.rx_dropped++;  in gem_rx()
1438 		skb = queue->rx_skbuff[entry];  in gem_rx()
1440 			netdev_err(bp->dev,  in gem_rx()
1442 			bp->dev->stats.rx_dropped++;  in gem_rx()
1443 			queue->stats.rx_dropped++;  in gem_rx()
1447 		queue->rx_skbuff[entry] = NULL;  in gem_rx()
1448 		len = ctrl & bp->rx_frm_len_mask;  in gem_rx()
1450 		netdev_vdbg(bp->dev, "gem_rx %u (len %u)\n", entry, len);  in gem_rx()
1453 		dma_unmap_single(&bp->pdev->dev, addr,  in gem_rx()
1454 				 bp->rx_buffer_size, DMA_FROM_DEVICE);  in gem_rx()
1456 		skb->protocol = eth_type_trans(skb, bp->dev);  in gem_rx()
1458 		if (bp->dev->features & NETIF_F_RXCSUM &&  in gem_rx()
1459 		    !(bp->dev->flags & IFF_PROMISC) &&  in gem_rx()
1461 			skb->ip_summed = CHECKSUM_UNNECESSARY;  in gem_rx()
1463 		bp->dev->stats.rx_packets++;  in gem_rx()
1464 		queue->stats.rx_packets++;  in gem_rx()
1465 		bp->dev->stats.rx_bytes += skb->len;  in gem_rx()
1466 		queue->stats.rx_bytes += skb->len;  in gem_rx()
1471 		netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",  in gem_rx()
1472 			    skb->len, skb->csum);  in gem_rx()
1476 			       skb->data, 32, true);  in gem_rx()
1495 	struct macb *bp = queue->bp;  in macb_rx_frame()
1498 	len = desc->ctrl & bp->rx_frm_len_mask;  in macb_rx_frame()
1500 	netdev_vdbg(bp->dev, "macb_rx_frame frags %u - %u (len %u)\n",  in macb_rx_frame()
1506 	 * payload word-aligned.  in macb_rx_frame()
1512 	skb = netdev_alloc_skb(bp->dev, len + NET_IP_ALIGN);  in macb_rx_frame()
1514 		bp->dev->stats.rx_dropped++;  in macb_rx_frame()
1517 			desc->addr &= ~MACB_BIT(RX_USED);  in macb_rx_frame()
1534 		unsigned int frag_len = bp->rx_buffer_size;  in macb_rx_frame()
1539 				return -1;  in macb_rx_frame()
1541 			frag_len = len - offset;  in macb_rx_frame()
1546 		offset += bp->rx_buffer_size;  in macb_rx_frame()
1548 		desc->addr &= ~MACB_BIT(RX_USED);  in macb_rx_frame()
1558 	skb->protocol = eth_type_trans(skb, bp->dev);  in macb_rx_frame()
1560 	bp->dev->stats.rx_packets++;  in macb_rx_frame()
1561 	bp->dev->stats.rx_bytes += skb->len;  in macb_rx_frame()
1562 	netdev_vdbg(bp->dev, "received skb of length %u, csum: %08x\n",  in macb_rx_frame()
1563 		    skb->len, skb->csum);  in macb_rx_frame()
1571 	struct macb *bp = queue->bp;  in macb_init_rx_ring()
1576 	addr = queue->rx_buffers_dma;  in macb_init_rx_ring()
1577 	for (i = 0; i < bp->rx_ring_size; i++) {  in macb_init_rx_ring()
1580 		desc->ctrl = 0;  in macb_init_rx_ring()
1581 		addr += bp->rx_buffer_size;  in macb_init_rx_ring()
1583 	desc->addr |= MACB_BIT(RX_WRAP);  in macb_init_rx_ring()
1584 	queue->rx_tail = 0;  in macb_init_rx_ring()
1590 	struct macb *bp = queue->bp;  in macb_rx()
1594 	int first_frag = -1;  in macb_rx()
1596 	for (tail = queue->rx_tail; budget > 0; tail++) {  in macb_rx()
1603 		if (!(desc->addr & MACB_BIT(RX_USED)))  in macb_rx()
1606 		/* Ensure ctrl is at least as up-to-date as addr */  in macb_rx()
1609 		ctrl = desc->ctrl;  in macb_rx()
1612 			if (first_frag != -1)  in macb_rx()
1620 			if (unlikely(first_frag == -1)) {  in macb_rx()
1626 			first_frag = -1;  in macb_rx()
1633 				budget--;  in macb_rx()
1642 		netdev_err(bp->dev, "RX queue corruption: reset it\n");  in macb_rx()
1644 		spin_lock_irqsave(&bp->lock, flags);  in macb_rx()
1650 		queue_writel(queue, RBQP, queue->rx_ring_dma);  in macb_rx()
1654 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_rx()
1658 	if (first_frag != -1)  in macb_rx()
1659 		queue->rx_tail = first_frag;  in macb_rx()
1661 		queue->rx_tail = tail;  in macb_rx()
1668 	struct macb *bp = queue->bp;  in macb_rx_pending()
1672 	entry = macb_rx_ring_wrap(bp, queue->rx_tail);  in macb_rx_pending()
1678 	return (desc->addr & MACB_BIT(RX_USED)) != 0;  in macb_rx_pending()
1684 	struct macb *bp = queue->bp;  in macb_rx_poll()
1687 	work_done = bp->macbgem_ops.mog_rx(queue, napi, budget);  in macb_rx_poll()
1689 	netdev_vdbg(bp->dev, "RX poll: queue = %u, work_done = %d, budget = %d\n",  in macb_rx_poll()
1690 		    (unsigned int)(queue - bp->queues), work_done, budget);  in macb_rx_poll()
1693 		queue_writel(queue, IER, bp->rx_intr_mask);  in macb_rx_poll()
1699 		 * interrupts are re-enabled.  in macb_rx_poll()
1706 			queue_writel(queue, IDR, bp->rx_intr_mask);  in macb_rx_poll()
1707 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_rx_poll()
1709 			netdev_vdbg(bp->dev, "poll: packets pending, reschedule\n");  in macb_rx_poll()
1721 	struct macb *bp = queue->bp;  in macb_tx_restart()
1724 	spin_lock(&queue->tx_ptr_lock);  in macb_tx_restart()
1726 	if (queue->tx_head == queue->tx_tail)  in macb_tx_restart()
1731 	head_idx = macb_adj_dma_desc_idx(bp, macb_tx_ring_wrap(bp, queue->tx_head));  in macb_tx_restart()
1736 	spin_lock_irq(&bp->lock);  in macb_tx_restart()
1738 	spin_unlock_irq(&bp->lock);  in macb_tx_restart()
1741 	spin_unlock(&queue->tx_ptr_lock);  in macb_tx_restart()
1748 	spin_lock(&queue->tx_ptr_lock);  in macb_tx_complete_pending()
1749 	if (queue->tx_head != queue->tx_tail) {  in macb_tx_complete_pending()
1753 		if (macb_tx_desc(queue, queue->tx_tail)->ctrl & MACB_BIT(TX_USED))  in macb_tx_complete_pending()
1756 	spin_unlock(&queue->tx_ptr_lock);  in macb_tx_complete_pending()
1763 	struct macb *bp = queue->bp;  in macb_tx_poll()
1769 	if (queue->txubr_pending) {  in macb_tx_poll()
1770 		queue->txubr_pending = false;  in macb_tx_poll()
1771 		netdev_vdbg(bp->dev, "poll: tx restart\n");  in macb_tx_poll()
1775 	netdev_vdbg(bp->dev, "TX poll: queue = %u, work_done = %d, budget = %d\n",  in macb_tx_poll()
1776 		    (unsigned int)(queue - bp->queues), work_done, budget);  in macb_tx_poll()
1785 		 * interrupts are re-enabled.  in macb_tx_poll()
1793 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_tx_poll()
1795 			netdev_vdbg(bp->dev, "TX poll: packets pending, reschedule\n");  in macb_tx_poll()
1806 	struct net_device *dev = bp->dev;  in macb_hresp_error_task()
1811 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_hresp_error_task()
1812 		queue_writel(queue, IDR, bp->rx_intr_mask |  in macb_hresp_error_task()
1823 	bp->macbgem_ops.mog_init_rings(bp);  in macb_hresp_error_task()
1829 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_hresp_error_task()
1831 			     bp->rx_intr_mask |  in macb_hresp_error_task()
1845 	struct macb *bp = queue->bp;  in macb_wol_interrupt()
1853 	spin_lock(&bp->lock);  in macb_wol_interrupt()
1858 		netdev_vdbg(bp->dev, "MACB WoL: queue = %u, isr = 0x%08lx\n",  in macb_wol_interrupt()
1859 			    (unsigned int)(queue - bp->queues),  in macb_wol_interrupt()
1861 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_wol_interrupt()
1863 		pm_wakeup_event(&bp->pdev->dev, 0);  in macb_wol_interrupt()
1866 	spin_unlock(&bp->lock);  in macb_wol_interrupt()
1874 	struct macb *bp = queue->bp;  in gem_wol_interrupt()
1882 	spin_lock(&bp->lock);  in gem_wol_interrupt()
1887 		netdev_vdbg(bp->dev, "GEM WoL: queue = %u, isr = 0x%08lx\n",  in gem_wol_interrupt()
1888 			    (unsigned int)(queue - bp->queues),  in gem_wol_interrupt()
1890 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in gem_wol_interrupt()
1892 		pm_wakeup_event(&bp->pdev->dev, 0);  in gem_wol_interrupt()
1895 	spin_unlock(&bp->lock);  in gem_wol_interrupt()
1903 	struct macb *bp = queue->bp;  in macb_interrupt()
1904 	struct net_device *dev = bp->dev;  in macb_interrupt()
1912 	spin_lock(&bp->lock);  in macb_interrupt()
1917 			queue_writel(queue, IDR, -1);  in macb_interrupt()
1918 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1919 				queue_writel(queue, ISR, -1);  in macb_interrupt()
1923 		netdev_vdbg(bp->dev, "queue = %u, isr = 0x%08lx\n",  in macb_interrupt()
1924 			    (unsigned int)(queue - bp->queues),  in macb_interrupt()
1927 		if (status & bp->rx_intr_mask) {  in macb_interrupt()
1934 			queue_writel(queue, IDR, bp->rx_intr_mask);  in macb_interrupt()
1935 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1938 			if (napi_schedule_prep(&queue->napi_rx)) {  in macb_interrupt()
1939 				netdev_vdbg(bp->dev, "scheduling RX softirq\n");  in macb_interrupt()
1940 				__napi_schedule(&queue->napi_rx);  in macb_interrupt()
1947 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1952 				queue->txubr_pending = true;  in macb_interrupt()
1956 			if (napi_schedule_prep(&queue->napi_tx)) {  in macb_interrupt()
1957 				netdev_vdbg(bp->dev, "scheduling TX softirq\n");  in macb_interrupt()
1958 				__napi_schedule(&queue->napi_tx);  in macb_interrupt()
1964 			schedule_work(&queue->tx_error_task);  in macb_interrupt()
1966 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1973 		 * add that if/when we get our hands on a full-blown MII PHY.  in macb_interrupt()
1978 		 * interrupts but it can be cleared by re-enabling RX. See  in macb_interrupt()
1989 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
1995 			spin_lock(&bp->stats_lock);  in macb_interrupt()
1997 				bp->hw_stats.gem.rx_overruns++;  in macb_interrupt()
1999 				bp->hw_stats.macb.rx_overruns++;  in macb_interrupt()
2000 			spin_unlock(&bp->stats_lock);  in macb_interrupt()
2002 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
2007 			tasklet_schedule(&bp->hresp_err_tasklet);  in macb_interrupt()
2010 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_interrupt()
2016 	spin_unlock(&bp->lock);  in macb_interrupt()
2022 /* Polling receive - used by netconsole and other diagnostic tools
2033 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in macb_poll_controller()
2034 		macb_interrupt(dev->irq, queue);  in macb_poll_controller()
2045 	unsigned int len, entry, i, tx_head = queue->tx_head;  in macb_tx_map()
2049 	unsigned int f, nr_frags = skb_shinfo(skb)->nr_frags;  in macb_tx_map()
2054 	if (skb_shinfo(skb)->gso_size != 0) {  in macb_tx_map()
2055 		if (ip_hdr(skb)->protocol == IPPROTO_UDP)  in macb_tx_map()
2056 			/* UDP - UFO */  in macb_tx_map()
2059 			/* TCP - TSO */  in macb_tx_map()
2063 	/* First, map non-paged data */  in macb_tx_map()
2072 		tx_skb = &queue->tx_skb[entry];  in macb_tx_map()
2074 		mapping = dma_map_single(&bp->pdev->dev,  in macb_tx_map()
2075 					 skb->data + offset,  in macb_tx_map()
2077 		if (dma_mapping_error(&bp->pdev->dev, mapping))  in macb_tx_map()
2081 		tx_skb->skb = NULL;  in macb_tx_map()
2082 		tx_skb->mapping = mapping;  in macb_tx_map()
2083 		tx_skb->size = size;  in macb_tx_map()
2084 		tx_skb->mapped_as_page = false;  in macb_tx_map()
2086 		len -= size;  in macb_tx_map()
2091 		size = min(len, bp->max_tx_length);  in macb_tx_map()
2096 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];  in macb_tx_map()
2101 			size = min(len, bp->max_tx_length);  in macb_tx_map()
2103 			tx_skb = &queue->tx_skb[entry];  in macb_tx_map()
2105 			mapping = skb_frag_dma_map(&bp->pdev->dev, frag,  in macb_tx_map()
2107 			if (dma_mapping_error(&bp->pdev->dev, mapping))  in macb_tx_map()
2111 			tx_skb->skb = NULL;  in macb_tx_map()
2112 			tx_skb->mapping = mapping;  in macb_tx_map()
2113 			tx_skb->size = size;  in macb_tx_map()
2114 			tx_skb->mapped_as_page = true;  in macb_tx_map()
2116 			len -= size;  in macb_tx_map()
2125 		netdev_err(bp->dev, "BUG! empty skb!\n");  in macb_tx_map()
2130 	tx_skb->skb = skb;  in macb_tx_map()
2143 	desc->ctrl = ctrl;  in macb_tx_map()
2148 			mss_mfs = skb_shinfo(skb)->gso_size +  in macb_tx_map()
2152 			mss_mfs = skb_shinfo(skb)->gso_size;  in macb_tx_map()
2161 		i--;  in macb_tx_map()
2163 		tx_skb = &queue->tx_skb[entry];  in macb_tx_map()
2166 		ctrl = (u32)tx_skb->size;  in macb_tx_map()
2171 		if (unlikely(entry == (bp->tx_ring_size - 1)))  in macb_tx_map()
2175 		if (i == queue->tx_head) {  in macb_tx_map()
2178 			if ((bp->dev->features & NETIF_F_HW_CSUM) &&  in macb_tx_map()
2179 			    skb->ip_summed != CHECKSUM_PARTIAL && !lso_ctrl &&  in macb_tx_map()
2189 		macb_set_addr(bp, desc, tx_skb->mapping);  in macb_tx_map()
2190 		/* desc->addr must be visible to hardware before clearing  in macb_tx_map()
2191 		 * 'TX_USED' bit in desc->ctrl.  in macb_tx_map()
2194 		desc->ctrl = ctrl;  in macb_tx_map()
2195 	} while (i != queue->tx_head);  in macb_tx_map()
2197 	queue->tx_head = tx_head;  in macb_tx_map()
2202 	netdev_err(bp->dev, "TX DMA map failed\n");  in macb_tx_map()
2204 	for (i = queue->tx_head; i != tx_head; i++) {  in macb_tx_map()
2223 	if (!skb_is_nonlinear(skb) || (ip_hdr(skb)->protocol != IPPROTO_UDP))  in macb_features_check()
2233 	if (!IS_ALIGNED(skb_headlen(skb) - hdrlen, MACB_TX_LEN_ALIGN))  in macb_features_check()
2236 	nr_frags = skb_shinfo(skb)->nr_frags;  in macb_features_check()
2238 	nr_frags--;  in macb_features_check()
2240 		const skb_frag_t *frag = &skb_shinfo(skb)->frags[f];  in macb_features_check()
2251 	if (skb->ip_summed != CHECKSUM_PARTIAL)  in macb_clear_csum()
2256 		return -1;  in macb_clear_csum()
2259 	 * This is required - at least for Zynq, which otherwise calculates  in macb_clear_csum()
2262 	*(__sum16 *)(skb_checksum_start(skb) + skb->csum_offset) = 0;  in macb_clear_csum()
2270 	int padlen = ETH_ZLEN - (*skb)->len;  in macb_pad_and_fcs()
2275 	if (!(ndev->features & NETIF_F_HW_CSUM) ||  in macb_pad_and_fcs()
2276 	    !((*skb)->ip_summed != CHECKSUM_PARTIAL) ||  in macb_pad_and_fcs()
2277 	    skb_shinfo(*skb)->gso_size || ptp_one_step_sync(*skb))  in macb_pad_and_fcs()
2295 			return -ENOMEM;  in macb_pad_and_fcs()
2302 		skb_put_zero(*skb, padlen - ETH_FCS_LEN);  in macb_pad_and_fcs()
2306 	fcs = crc32_le(~0, (*skb)->data, (*skb)->len);  in macb_pad_and_fcs()
2321 	struct macb_queue *queue = &bp->queues[queue_index];  in macb_start_xmit()
2338 	if ((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&  in macb_start_xmit()
2339 	    (bp->hw_dma_cap & HW_DMA_CAP_PTP))  in macb_start_xmit()
2340 		skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;  in macb_start_xmit()
2343 	is_lso = (skb_shinfo(skb)->gso_size != 0);  in macb_start_xmit()
2347 		if (ip_hdr(skb)->protocol == IPPROTO_UDP)  in macb_start_xmit()
2353 			netdev_err(bp->dev, "Error - LSO headers fragmented!!!\n");  in macb_start_xmit()
2358 		hdrlen = min(skb_headlen(skb), bp->max_tx_length);  in macb_start_xmit()
2361 	netdev_vdbg(bp->dev,  in macb_start_xmit()
2363 		    queue_index, skb->len, skb->head, skb->data,  in macb_start_xmit()
2366 		       skb->data, 16, true);  in macb_start_xmit()
2375 		desc_cnt = DIV_ROUND_UP((skb_headlen(skb) - hdrlen), bp->max_tx_length) + 1;  in macb_start_xmit()
2377 		desc_cnt = DIV_ROUND_UP(skb_headlen(skb), bp->max_tx_length);  in macb_start_xmit()
2378 	nr_frags = skb_shinfo(skb)->nr_frags;  in macb_start_xmit()
2380 		frag_size = skb_frag_size(&skb_shinfo(skb)->frags[f]);  in macb_start_xmit()
2381 		desc_cnt += DIV_ROUND_UP(frag_size, bp->max_tx_length);  in macb_start_xmit()
2384 	spin_lock_bh(&queue->tx_ptr_lock);  in macb_start_xmit()
2387 	if (CIRC_SPACE(queue->tx_head, queue->tx_tail,  in macb_start_xmit()
2388 		       bp->tx_ring_size) < desc_cnt) {  in macb_start_xmit()
2390 		netdev_dbg(bp->dev, "tx_head = %u, tx_tail = %u\n",  in macb_start_xmit()
2391 			   queue->tx_head, queue->tx_tail);  in macb_start_xmit()
2406 	spin_lock_irq(&bp->lock);  in macb_start_xmit()
2408 	spin_unlock_irq(&bp->lock);  in macb_start_xmit()
2410 	if (CIRC_SPACE(queue->tx_head, queue->tx_tail, bp->tx_ring_size) < 1)  in macb_start_xmit()
2414 	spin_unlock_bh(&queue->tx_ptr_lock);  in macb_start_xmit()
2422 		bp->rx_buffer_size = MACB_RX_BUFFER_SIZE;  in macb_init_rx_buffer_size()
2424 		bp->rx_buffer_size = size;  in macb_init_rx_buffer_size()
2426 		if (bp->rx_buffer_size % RX_BUFFER_MULTIPLE) {  in macb_init_rx_buffer_size()
2427 			netdev_dbg(bp->dev,  in macb_init_rx_buffer_size()
2430 			bp->rx_buffer_size =  in macb_init_rx_buffer_size()
2431 				roundup(bp->rx_buffer_size, RX_BUFFER_MULTIPLE);  in macb_init_rx_buffer_size()
2435 	netdev_dbg(bp->dev, "mtu [%u] rx_buffer_size [%zu]\n",  in macb_init_rx_buffer_size()
2436 		   bp->dev->mtu, bp->rx_buffer_size);  in macb_init_rx_buffer_size()
2448 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_free_rx_buffers()
2449 		if (!queue->rx_skbuff)  in gem_free_rx_buffers()
2452 		for (i = 0; i < bp->rx_ring_size; i++) {  in gem_free_rx_buffers()
2453 			skb = queue->rx_skbuff[i];  in gem_free_rx_buffers()
2461 			dma_unmap_single(&bp->pdev->dev, addr, bp->rx_buffer_size,  in gem_free_rx_buffers()
2467 		kfree(queue->rx_skbuff);  in gem_free_rx_buffers()
2468 		queue->rx_skbuff = NULL;  in gem_free_rx_buffers()
2474 	struct macb_queue *queue = &bp->queues[0];  in macb_free_rx_buffers()
2476 	if (queue->rx_buffers) {  in macb_free_rx_buffers()
2477 		dma_free_coherent(&bp->pdev->dev,  in macb_free_rx_buffers()
2478 				  bp->rx_ring_size * bp->rx_buffer_size,  in macb_free_rx_buffers()
2479 				  queue->rx_buffers, queue->rx_buffers_dma);  in macb_free_rx_buffers()
2480 		queue->rx_buffers = NULL;  in macb_free_rx_buffers()
2490 	bp->macbgem_ops.mog_free_rx_buffers(bp);  in macb_free_consistent()
2492 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_free_consistent()
2493 		kfree(queue->tx_skb);  in macb_free_consistent()
2494 		queue->tx_skb = NULL;  in macb_free_consistent()
2495 		if (queue->tx_ring) {  in macb_free_consistent()
2496 			size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;  in macb_free_consistent()
2497 			dma_free_coherent(&bp->pdev->dev, size,  in macb_free_consistent()
2498 					  queue->tx_ring, queue->tx_ring_dma);  in macb_free_consistent()
2499 			queue->tx_ring = NULL;  in macb_free_consistent()
2501 		if (queue->rx_ring) {  in macb_free_consistent()
2502 			size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;  in macb_free_consistent()
2503 			dma_free_coherent(&bp->pdev->dev, size,  in macb_free_consistent()
2504 					  queue->rx_ring, queue->rx_ring_dma);  in macb_free_consistent()
2505 			queue->rx_ring = NULL;  in macb_free_consistent()
2516 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_alloc_rx_buffers()
2517 		size = bp->rx_ring_size * sizeof(struct sk_buff *);  in gem_alloc_rx_buffers()
2518 		queue->rx_skbuff = kzalloc(size, GFP_KERNEL);  in gem_alloc_rx_buffers()
2519 		if (!queue->rx_skbuff)  in gem_alloc_rx_buffers()
2520 			return -ENOMEM;  in gem_alloc_rx_buffers()
2522 			netdev_dbg(bp->dev,  in gem_alloc_rx_buffers()
2524 				   bp->rx_ring_size, queue->rx_skbuff);  in gem_alloc_rx_buffers()
2531 	struct macb_queue *queue = &bp->queues[0];  in macb_alloc_rx_buffers()
2534 	size = bp->rx_ring_size * bp->rx_buffer_size;  in macb_alloc_rx_buffers()
2535 	queue->rx_buffers = dma_alloc_coherent(&bp->pdev->dev, size,  in macb_alloc_rx_buffers()
2536 					    &queue->rx_buffers_dma, GFP_KERNEL);  in macb_alloc_rx_buffers()
2537 	if (!queue->rx_buffers)  in macb_alloc_rx_buffers()
2538 		return -ENOMEM;  in macb_alloc_rx_buffers()
2540 	netdev_dbg(bp->dev,  in macb_alloc_rx_buffers()
2542 		   size, (unsigned long)queue->rx_buffers_dma, queue->rx_buffers);  in macb_alloc_rx_buffers()
2552 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_alloc_consistent()
2553 		size = TX_RING_BYTES(bp) + bp->tx_bd_rd_prefetch;  in macb_alloc_consistent()
2554 		queue->tx_ring = dma_alloc_coherent(&bp->pdev->dev, size,  in macb_alloc_consistent()
2555 						    &queue->tx_ring_dma,  in macb_alloc_consistent()
2557 		if (!queue->tx_ring)  in macb_alloc_consistent()
2559 		netdev_dbg(bp->dev,  in macb_alloc_consistent()
2561 			   q, size, (unsigned long)queue->tx_ring_dma,  in macb_alloc_consistent()
2562 			   queue->tx_ring);  in macb_alloc_consistent()
2564 		size = bp->tx_ring_size * sizeof(struct macb_tx_skb);  in macb_alloc_consistent()
2565 		queue->tx_skb = kmalloc(size, GFP_KERNEL);  in macb_alloc_consistent()
2566 		if (!queue->tx_skb)  in macb_alloc_consistent()
2569 		size = RX_RING_BYTES(bp) + bp->rx_bd_rd_prefetch;  in macb_alloc_consistent()
2570 		queue->rx_ring = dma_alloc_coherent(&bp->pdev->dev, size,  in macb_alloc_consistent()
2571 						 &queue->rx_ring_dma, GFP_KERNEL);  in macb_alloc_consistent()
2572 		if (!queue->rx_ring)  in macb_alloc_consistent()
2574 		netdev_dbg(bp->dev,  in macb_alloc_consistent()
2576 			   size, (unsigned long)queue->rx_ring_dma, queue->rx_ring);  in macb_alloc_consistent()
2578 	if (bp->macbgem_ops.mog_alloc_rx_buffers(bp))  in macb_alloc_consistent()
2585 	return -ENOMEM;  in macb_alloc_consistent()
2595 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_init_rings()
2596 		for (i = 0; i < bp->tx_ring_size; i++) {  in gem_init_rings()
2599 			desc->ctrl = MACB_BIT(TX_USED);  in gem_init_rings()
2601 		desc->ctrl |= MACB_BIT(TX_WRAP);  in gem_init_rings()
2602 		queue->tx_head = 0;  in gem_init_rings()
2603 		queue->tx_tail = 0;  in gem_init_rings()
2605 		queue->rx_tail = 0;  in gem_init_rings()
2606 		queue->rx_prepared_head = 0;  in gem_init_rings()
2618 	macb_init_rx_ring(&bp->queues[0]);  in macb_init_rings()
2620 	for (i = 0; i < bp->tx_ring_size; i++) {  in macb_init_rings()
2621 		desc = macb_tx_desc(&bp->queues[0], i);  in macb_init_rings()
2623 		desc->ctrl = MACB_BIT(TX_USED);  in macb_init_rings()
2625 	bp->queues[0].tx_head = 0;  in macb_init_rings()
2626 	bp->queues[0].tx_tail = 0;  in macb_init_rings()
2627 	desc->ctrl |= MACB_BIT(TX_WRAP);  in macb_init_rings()
2647 	macb_writel(bp, TSR, -1);  in macb_reset_hw()
2648 	macb_writel(bp, RSR, -1);  in macb_reset_hw()
2654 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_reset_hw()
2655 		queue_writel(queue, IDR, -1);  in macb_reset_hw()
2657 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_reset_hw()
2658 			queue_writel(queue, ISR, -1);  in macb_reset_hw()
2665 	unsigned long pclk_hz = clk_get_rate(bp->pclk);  in gem_mdc_clk_div()
2695 	pclk_hz = clk_get_rate(bp->pclk);  in macb_mdc_clk_div()
2729  * - use the correct receive buffer size
2730  * - set best burst length for DMA operations
2732  * - set both rx/tx packet buffers to full memory size
2742 	buffer_size = bp->rx_buffer_size / RX_BUFFER_MULTIPLE;  in macb_configure_dma()
2744 		dmacfg = gem_readl(bp, DMACFG) & ~GEM_BF(RXBS, -1L);  in macb_configure_dma()
2745 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_configure_dma()
2751 		if (bp->dma_burst_length)  in macb_configure_dma()
2752 			dmacfg = GEM_BFINS(FBLDO, bp->dma_burst_length, dmacfg);  in macb_configure_dma()
2753 		dmacfg |= GEM_BIT(TXPBMS) | GEM_BF(RXBMS, -1L);  in macb_configure_dma()
2756 		if (bp->native_io)  in macb_configure_dma()
2761 		if (bp->dev->features & NETIF_F_HW_CSUM)  in macb_configure_dma()
2768 		if (bp->hw_dma_cap & HW_DMA_CAP_64B)  in macb_configure_dma()
2772 		if (bp->hw_dma_cap & HW_DMA_CAP_PTP)  in macb_configure_dma()
2775 		netdev_dbg(bp->dev, "Cadence configure DMA with 0x%08x\n",  in macb_configure_dma()
2791 	if (bp->caps & MACB_CAPS_JUMBO)  in macb_init_hw()
2795 	if (bp->dev->flags & IFF_PROMISC)  in macb_init_hw()
2797 	else if (macb_is_gem(bp) && bp->dev->features & NETIF_F_RXCSUM)  in macb_init_hw()
2799 	if (!(bp->dev->flags & IFF_BROADCAST))  in macb_init_hw()
2803 	if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)  in macb_init_hw()
2804 		gem_writel(bp, JML, bp->jumbo_max_len);  in macb_init_hw()
2805 	bp->rx_frm_len_mask = MACB_RX_FRMLEN_MASK;  in macb_init_hw()
2806 	if (bp->caps & MACB_CAPS_JUMBO)  in macb_init_hw()
2807 		bp->rx_frm_len_mask = MACB_RX_JFRMLEN_MASK;  in macb_init_hw()
2812 	if (bp->rx_watermark)  in macb_init_hw()
2813 		gem_writel(bp, PBUFRXCUT, (bp->rx_watermark | GEM_BIT(ENCUTTHRU)));  in macb_init_hw()
2872 /* Add multicast addresses to the internal multicast-hash table. */
2884 		bitnr = hash_get_index(ha->addr);  in macb_sethashtable()
2900 	if (dev->flags & IFF_PROMISC) {  in macb_set_rx_mode()
2912 		if (macb_is_gem(bp) && dev->features & NETIF_F_RXCSUM)  in macb_set_rx_mode()
2916 	if (dev->flags & IFF_ALLMULTI) {  in macb_set_rx_mode()
2918 		macb_or_gem_writel(bp, HRB, -1);  in macb_set_rx_mode()
2919 		macb_or_gem_writel(bp, HRT, -1);  in macb_set_rx_mode()
2925 	} else if (dev->flags & (~IFF_ALLMULTI)) {  in macb_set_rx_mode()
2937 	size_t bufsz = dev->mtu + ETH_HLEN + ETH_FCS_LEN + NET_IP_ALIGN;  in macb_open()
2943 	netdev_dbg(bp->dev, "open\n");  in macb_open()
2945 	err = pm_runtime_resume_and_get(&bp->pdev->dev);  in macb_open()
2959 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_open()
2960 		napi_enable(&queue->napi_rx);  in macb_open()
2961 		napi_enable(&queue->napi_tx);  in macb_open()
2966 	err = phy_power_on(bp->sgmii_phy);  in macb_open()
2976 	if (bp->ptp_info)  in macb_open()
2977 		bp->ptp_info->ptp_init(dev);  in macb_open()
2982 	phy_power_off(bp->sgmii_phy);  in macb_open()
2986 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_open()
2987 		napi_disable(&queue->napi_rx);  in macb_open()
2988 		napi_disable(&queue->napi_tx);  in macb_open()
2992 	pm_runtime_put_sync(&bp->pdev->dev);  in macb_open()
3005 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in macb_close()
3006 		napi_disable(&queue->napi_rx);  in macb_close()
3007 		napi_disable(&queue->napi_tx);  in macb_close()
3010 	phylink_stop(bp->phylink);  in macb_close()
3011 	phylink_disconnect_phy(bp->phylink);  in macb_close()
3013 	phy_power_off(bp->sgmii_phy);  in macb_close()
3015 	spin_lock_irqsave(&bp->lock, flags);  in macb_close()
3018 	spin_unlock_irqrestore(&bp->lock, flags);  in macb_close()
3022 	if (bp->ptp_info)  in macb_close()
3023 		bp->ptp_info->ptp_remove(dev);  in macb_close()
3025 	pm_runtime_put(&bp->pdev->dev);  in macb_close()
3033 		return -EBUSY;  in macb_change_mtu()
3035 	dev->mtu = new_mtu;  in macb_change_mtu()
3058 	u32 *p = &bp->hw_stats.gem.tx_octets_31_0;  in gem_update_stats()
3062 		u64 val = bp->macb_reg_readl(bp, offset);  in gem_update_stats()
3064 		bp->ethtool_stats[i] += val;  in gem_update_stats()
3069 			val = bp->macb_reg_readl(bp, offset + 4);  in gem_update_stats()
3070 			bp->ethtool_stats[i] += ((u64)val) << 32;  in gem_update_stats()
3076 	for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue)  in gem_update_stats()
3077 		for (i = 0, stat = &queue->stats.first; i < QUEUE_STATS_LEN; ++i, ++stat)  in gem_update_stats()
3078 			bp->ethtool_stats[idx++] = *stat;  in gem_update_stats()
3083 	struct gem_stats *hwstat = &bp->hw_stats.gem;  in gem_get_stats()
3084 	struct net_device_stats *nstat = &bp->dev->stats;  in gem_get_stats()
3086 	if (!netif_running(bp->dev))  in gem_get_stats()
3089 	spin_lock_irq(&bp->stats_lock);  in gem_get_stats()
3092 	nstat->rx_errors = (hwstat->rx_frame_check_sequence_errors +  in gem_get_stats()
3093 			    hwstat->rx_alignment_errors +  in gem_get_stats()
3094 			    hwstat->rx_resource_errors +  in gem_get_stats()
3095 			    hwstat->rx_overruns +  in gem_get_stats()
3096 			    hwstat->rx_oversize_frames +  in gem_get_stats()
3097 			    hwstat->rx_jabbers +  in gem_get_stats()
3098 			    hwstat->rx_undersized_frames +  in gem_get_stats()
3099 			    hwstat->rx_length_field_frame_errors);  in gem_get_stats()
3100 	nstat->tx_errors = (hwstat->tx_late_collisions +  in gem_get_stats()
3101 			    hwstat->tx_excessive_collisions +  in gem_get_stats()
3102 			    hwstat->tx_underrun +  in gem_get_stats()
3103 			    hwstat->tx_carrier_sense_errors);  in gem_get_stats()
3104 	nstat->multicast = hwstat->rx_multicast_frames;  in gem_get_stats()
3105 	nstat->collisions = (hwstat->tx_single_collision_frames +  in gem_get_stats()
3106 			     hwstat->tx_multiple_collision_frames +  in gem_get_stats()
3107 			     hwstat->tx_excessive_collisions);  in gem_get_stats()
3108 	nstat->rx_length_errors = (hwstat->rx_oversize_frames +  in gem_get_stats()
3109 				   hwstat->rx_jabbers +  in gem_get_stats()
3110 				   hwstat->rx_undersized_frames +  in gem_get_stats()
3111 				   hwstat->rx_length_field_frame_errors);  in gem_get_stats()
3112 	nstat->rx_over_errors = hwstat->rx_resource_errors;  in gem_get_stats()
3113 	nstat->rx_crc_errors = hwstat->rx_frame_check_sequence_errors;  in gem_get_stats()
3114 	nstat->rx_frame_errors = hwstat->rx_alignment_errors;  in gem_get_stats()
3115 	nstat->rx_fifo_errors = hwstat->rx_overruns;  in gem_get_stats()
3116 	nstat->tx_aborted_errors = hwstat->tx_excessive_collisions;  in gem_get_stats()
3117 	nstat->tx_carrier_errors = hwstat->tx_carrier_sense_errors;  in gem_get_stats()
3118 	nstat->tx_fifo_errors = hwstat->tx_underrun;  in gem_get_stats()
3119 	spin_unlock_irq(&bp->stats_lock);  in gem_get_stats()
3129 	spin_lock_irq(&bp->stats_lock);  in gem_get_ethtool_stats()
3131 	memcpy(data, &bp->ethtool_stats, sizeof(u64)  in gem_get_ethtool_stats()
3133 	spin_unlock_irq(&bp->stats_lock);  in gem_get_ethtool_stats()
3142 		return GEM_STATS_LEN + bp->num_queues * QUEUE_STATS_LEN;  in gem_get_sset_count()
3144 		return -EOPNOTSUPP;  in gem_get_sset_count()
3162 		for (q = 0, queue = bp->queues; q < bp->num_queues; ++q, ++queue) {  in gem_get_ethtool_strings()
3176 	struct net_device_stats *nstat = &bp->dev->stats;  in macb_get_stats()
3177 	struct macb_stats *hwstat = &bp->hw_stats.macb;  in macb_get_stats()
3183 	spin_lock_irq(&bp->stats_lock);  in macb_get_stats()
3187 	nstat->rx_errors = (hwstat->rx_fcs_errors +  in macb_get_stats()
3188 			    hwstat->rx_align_errors +  in macb_get_stats()
3189 			    hwstat->rx_resource_errors +  in macb_get_stats()
3190 			    hwstat->rx_overruns +  in macb_get_stats()
3191 			    hwstat->rx_oversize_pkts +  in macb_get_stats()
3192 			    hwstat->rx_jabbers +  in macb_get_stats()
3193 			    hwstat->rx_undersize_pkts +  in macb_get_stats()
3194 			    hwstat->rx_length_mismatch);  in macb_get_stats()
3195 	nstat->tx_errors = (hwstat->tx_late_cols +  in macb_get_stats()
3196 			    hwstat->tx_excessive_cols +  in macb_get_stats()
3197 			    hwstat->tx_underruns +  in macb_get_stats()
3198 			    hwstat->tx_carrier_errors +  in macb_get_stats()
3199 			    hwstat->sqe_test_errors);  in macb_get_stats()
3200 	nstat->collisions = (hwstat->tx_single_cols +  in macb_get_stats()
3201 			     hwstat->tx_multiple_cols +  in macb_get_stats()
3202 			     hwstat->tx_excessive_cols);  in macb_get_stats()
3203 	nstat->rx_length_errors = (hwstat->rx_oversize_pkts +  in macb_get_stats()
3204 				   hwstat->rx_jabbers +  in macb_get_stats()
3205 				   hwstat->rx_undersize_pkts +  in macb_get_stats()
3206 				   hwstat->rx_length_mismatch);  in macb_get_stats()
3207 	nstat->rx_over_errors = hwstat->rx_resource_errors +  in macb_get_stats()
3208 				   hwstat->rx_overruns;  in macb_get_stats()
3209 	nstat->rx_crc_errors = hwstat->rx_fcs_errors;  in macb_get_stats()
3210 	nstat->rx_frame_errors = hwstat->rx_align_errors;  in macb_get_stats()
3211 	nstat->rx_fifo_errors = hwstat->rx_overruns;  in macb_get_stats()
3213 	nstat->tx_aborted_errors = hwstat->tx_excessive_cols;  in macb_get_stats()
3214 	nstat->tx_carrier_errors = hwstat->tx_carrier_errors;  in macb_get_stats()
3215 	nstat->tx_fifo_errors = hwstat->tx_underruns;  in macb_get_stats()
3217 	spin_unlock_irq(&bp->stats_lock);  in macb_get_stats()
3234 	regs->version = (macb_readl(bp, MID) & ((1 << MACB_REV_SIZE) - 1))  in macb_get_regs()
3237 	tail = macb_tx_ring_wrap(bp, bp->queues[0].tx_tail);  in macb_get_regs()
3238 	head = macb_tx_ring_wrap(bp, bp->queues[0].tx_head);  in macb_get_regs()
3251 	regs_buff[10] = macb_tx_dma(&bp->queues[0], tail);  in macb_get_regs()
3252 	regs_buff[11] = macb_tx_dma(&bp->queues[0], head);  in macb_get_regs()
3254 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))  in macb_get_regs()
3264 	if (bp->wol & MACB_WOL_HAS_MAGIC_PACKET) {  in macb_get_wol()
3265 		phylink_ethtool_get_wol(bp->phylink, wol);  in macb_get_wol()
3266 		wol->supported |= WAKE_MAGIC;  in macb_get_wol()
3268 		if (bp->wol & MACB_WOL_ENABLED)  in macb_get_wol()
3269 			wol->wolopts |= WAKE_MAGIC;  in macb_get_wol()
3279 	ret = phylink_ethtool_set_wol(bp->phylink, wol);  in macb_set_wol()
3283 	if (!ret || ret != -EOPNOTSUPP)  in macb_set_wol()
3286 	if (!(bp->wol & MACB_WOL_HAS_MAGIC_PACKET) ||  in macb_set_wol()
3287 	    (wol->wolopts & ~WAKE_MAGIC))  in macb_set_wol()
3288 		return -EOPNOTSUPP;  in macb_set_wol()
3290 	if (wol->wolopts & WAKE_MAGIC)  in macb_set_wol()
3291 		bp->wol |= MACB_WOL_ENABLED;  in macb_set_wol()
3293 		bp->wol &= ~MACB_WOL_ENABLED;  in macb_set_wol()
3295 	device_set_wakeup_enable(&bp->pdev->dev, bp->wol & MACB_WOL_ENABLED);  in macb_set_wol()
3305 	return phylink_ethtool_ksettings_get(bp->phylink, kset);  in macb_get_link_ksettings()
3313 	return phylink_ethtool_ksettings_set(bp->phylink, kset);  in macb_set_link_ksettings()
3323 	ring->rx_max_pending = MAX_RX_RING_SIZE;  in macb_get_ringparam()
3324 	ring->tx_max_pending = MAX_TX_RING_SIZE;  in macb_get_ringparam()
3326 	ring->rx_pending = bp->rx_ring_size;  in macb_get_ringparam()
3327 	ring->tx_pending = bp->tx_ring_size;  in macb_get_ringparam()
3339 	if ((ring->rx_mini_pending) || (ring->rx_jumbo_pending))  in macb_set_ringparam()
3340 		return -EINVAL;  in macb_set_ringparam()
3342 	new_rx_size = clamp_t(u32, ring->rx_pending,  in macb_set_ringparam()
3346 	new_tx_size = clamp_t(u32, ring->tx_pending,  in macb_set_ringparam()
3350 	if ((new_tx_size == bp->tx_ring_size) &&  in macb_set_ringparam()
3351 	    (new_rx_size == bp->rx_ring_size)) {  in macb_set_ringparam()
3356 	if (netif_running(bp->dev)) {  in macb_set_ringparam()
3358 		macb_close(bp->dev);  in macb_set_ringparam()
3361 	bp->rx_ring_size = new_rx_size;  in macb_set_ringparam()
3362 	bp->tx_ring_size = new_tx_size;  in macb_set_ringparam()
3365 		macb_open(bp->dev);  in macb_set_ringparam()
3376 	tsu_clk = devm_clk_get(&bp->pdev->dev, "tsu_clk");  in gem_get_tsu_rate()
3380 	else if (!IS_ERR(bp->pclk)) {  in gem_get_tsu_rate()
3381 		tsu_clk = bp->pclk;  in gem_get_tsu_rate()
3384 		return -ENOTSUPP;  in gem_get_tsu_rate()
3398 	if ((bp->hw_dma_cap & HW_DMA_CAP_PTP) == 0) {  in gem_get_ts_info()
3403 	info->so_timestamping =  in gem_get_ts_info()
3410 	info->tx_types =  in gem_get_ts_info()
3414 	info->rx_filters =  in gem_get_ts_info()
3418 	info->phc_index = bp->ptp_clock ? ptp_clock_index(bp->ptp_clock) : -1;  in gem_get_ts_info()
3439 	if (bp->ptp_info)  in macb_get_ts_info()
3440 		return bp->ptp_info->get_ts_info(netdev, info);  in macb_get_ts_info()
3447 	struct net_device *netdev = bp->dev;  in gem_enable_flow_filters()
3452 	if (!(netdev->features & NETIF_F_NTUPLE))  in gem_enable_flow_filters()
3457 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_enable_flow_filters()
3458 		struct ethtool_rx_flow_spec *fs = &item->fs;  in gem_enable_flow_filters()
3461 		if (fs->location >= num_t2_scr)  in gem_enable_flow_filters()
3464 		t2_scr = gem_readl_n(bp, SCRT2, fs->location);  in gem_enable_flow_filters()
3470 		tp4sp_m = &(fs->m_u.tcp_ip4_spec);  in gem_enable_flow_filters()
3472 		if (enable && (tp4sp_m->ip4src == 0xFFFFFFFF))  in gem_enable_flow_filters()
3477 		if (enable && (tp4sp_m->ip4dst == 0xFFFFFFFF))  in gem_enable_flow_filters()
3482 		if (enable && ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)))  in gem_enable_flow_filters()
3487 		gem_writel_n(bp, SCRT2, fs->location, t2_scr);  in gem_enable_flow_filters()
3494 	uint16_t index = fs->location;  in gem_prog_cmp_regs()
3503 	tp4sp_v = &(fs->h_u.tcp_ip4_spec);  in gem_prog_cmp_regs()
3504 	tp4sp_m = &(fs->m_u.tcp_ip4_spec);  in gem_prog_cmp_regs()
3507 	if (tp4sp_m->ip4src == 0xFFFFFFFF) {  in gem_prog_cmp_regs()
3508 		/* 1st compare reg - IP source address */  in gem_prog_cmp_regs()
3511 		w0 = tp4sp_v->ip4src;  in gem_prog_cmp_regs()
3512 		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */  in gem_prog_cmp_regs()
3521 	if (tp4sp_m->ip4dst == 0xFFFFFFFF) {  in gem_prog_cmp_regs()
3522 		/* 2nd compare reg - IP destination address */  in gem_prog_cmp_regs()
3525 		w0 = tp4sp_v->ip4dst;  in gem_prog_cmp_regs()
3526 		w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */  in gem_prog_cmp_regs()
3535 	if ((tp4sp_m->psrc == 0xFFFF) || (tp4sp_m->pdst == 0xFFFF)) {  in gem_prog_cmp_regs()
3536 		/* 3rd compare reg - source port, destination port */  in gem_prog_cmp_regs()
3540 		if (tp4sp_m->psrc == tp4sp_m->pdst) {  in gem_prog_cmp_regs()
3541 			w0 = GEM_BFINS(T2MASK, tp4sp_v->psrc, w0);  in gem_prog_cmp_regs()
3542 			w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);  in gem_prog_cmp_regs()
3543 			w1 = GEM_BFINS(T2DISMSK, 1, w1); /* 32-bit compare */  in gem_prog_cmp_regs()
3547 			w1 = GEM_BFINS(T2DISMSK, 0, w1); /* 16-bit compare */  in gem_prog_cmp_regs()
3549 			if (tp4sp_m->psrc == 0xFFFF) { /* src port */  in gem_prog_cmp_regs()
3550 				w0 = GEM_BFINS(T2CMP, tp4sp_v->psrc, w0);  in gem_prog_cmp_regs()
3553 				w0 = GEM_BFINS(T2CMP, tp4sp_v->pdst, w0);  in gem_prog_cmp_regs()
3563 	t2_scr = GEM_BFINS(QUEUE, (fs->ring_cookie) & 0xFF, t2_scr);  in gem_prog_cmp_regs()
3578 	struct ethtool_rx_flow_spec *fs = &cmd->fs;  in gem_add_flow_filter()
3581 	int ret = -EINVAL;  in gem_add_flow_filter()
3586 		return -ENOMEM;  in gem_add_flow_filter()
3587 	memcpy(&newfs->fs, fs, sizeof(newfs->fs));  in gem_add_flow_filter()
3591 			fs->flow_type, (int)fs->ring_cookie, fs->location,  in gem_add_flow_filter()
3592 			htonl(fs->h_u.tcp_ip4_spec.ip4src),  in gem_add_flow_filter()
3593 			htonl(fs->h_u.tcp_ip4_spec.ip4dst),  in gem_add_flow_filter()
3594 			be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),  in gem_add_flow_filter()
3595 			be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));  in gem_add_flow_filter()
3597 	spin_lock_irqsave(&bp->rx_fs_lock, flags);  in gem_add_flow_filter()
3600 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_add_flow_filter()
3601 		if (item->fs.location > newfs->fs.location) {  in gem_add_flow_filter()
3602 			list_add_tail(&newfs->list, &item->list);  in gem_add_flow_filter()
3605 		} else if (item->fs.location == fs->location) {  in gem_add_flow_filter()
3607 					fs->location);  in gem_add_flow_filter()
3608 			ret = -EBUSY;  in gem_add_flow_filter()
3613 		list_add_tail(&newfs->list, &bp->rx_fs_list.list);  in gem_add_flow_filter()
3616 	bp->rx_fs_list.count++;  in gem_add_flow_filter()
3620 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_add_flow_filter()
3624 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_add_flow_filter()
3637 	spin_lock_irqsave(&bp->rx_fs_lock, flags);  in gem_del_flow_filter()
3639 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_del_flow_filter()
3640 		if (item->fs.location == cmd->fs.location) {  in gem_del_flow_filter()
3642 			fs = &(item->fs);  in gem_del_flow_filter()
3645 					fs->flow_type, (int)fs->ring_cookie, fs->location,  in gem_del_flow_filter()
3646 					htonl(fs->h_u.tcp_ip4_spec.ip4src),  in gem_del_flow_filter()
3647 					htonl(fs->h_u.tcp_ip4_spec.ip4dst),  in gem_del_flow_filter()
3648 					be16_to_cpu(fs->h_u.tcp_ip4_spec.psrc),  in gem_del_flow_filter()
3649 					be16_to_cpu(fs->h_u.tcp_ip4_spec.pdst));  in gem_del_flow_filter()
3651 			gem_writel_n(bp, SCRT2, fs->location, 0);  in gem_del_flow_filter()
3653 			list_del(&item->list);  in gem_del_flow_filter()
3654 			bp->rx_fs_list.count--;  in gem_del_flow_filter()
3655 			spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_del_flow_filter()
3661 	spin_unlock_irqrestore(&bp->rx_fs_lock, flags);  in gem_del_flow_filter()
3662 	return -EINVAL;  in gem_del_flow_filter()
3671 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_get_flow_entry()
3672 		if (item->fs.location == cmd->fs.location) {  in gem_get_flow_entry()
3673 			memcpy(&cmd->fs, &item->fs, sizeof(cmd->fs));  in gem_get_flow_entry()
3677 	return -EINVAL;  in gem_get_flow_entry()
3687 	list_for_each_entry(item, &bp->rx_fs_list.list, list) {  in gem_get_all_flow_entries()
3688 		if (cnt == cmd->rule_cnt)  in gem_get_all_flow_entries()
3689 			return -EMSGSIZE;  in gem_get_all_flow_entries()
3690 		rule_locs[cnt] = item->fs.location;  in gem_get_all_flow_entries()
3693 	cmd->data = bp->max_tuples;  in gem_get_all_flow_entries()
3694 	cmd->rule_cnt = cnt;  in gem_get_all_flow_entries()
3705 	switch (cmd->cmd) {  in gem_get_rxnfc()
3707 		cmd->data = bp->num_queues;  in gem_get_rxnfc()
3710 		cmd->rule_cnt = bp->rx_fs_list.count;  in gem_get_rxnfc()
3720 			  "Command parameter %d is not supported\n", cmd->cmd);  in gem_get_rxnfc()
3721 		ret = -EOPNOTSUPP;  in gem_get_rxnfc()
3732 	switch (cmd->cmd) {  in gem_set_rxnfc()
3734 		if ((cmd->fs.location >= bp->max_tuples)  in gem_set_rxnfc()
3735 				|| (cmd->fs.ring_cookie >= bp->num_queues)) {  in gem_set_rxnfc()
3736 			ret = -EINVAL;  in gem_set_rxnfc()
3746 			  "Command parameter %d is not supported\n", cmd->cmd);  in gem_set_rxnfc()
3747 		ret = -EOPNOTSUPP;  in gem_set_rxnfc()
3789 		return -EINVAL;  in macb_ioctl()
3791 	if (bp->ptp_info) {  in macb_ioctl()
3794 			return bp->ptp_info->set_hwtst(dev, rq, cmd);  in macb_ioctl()
3796 			return bp->ptp_info->get_hwtst(dev, rq);  in macb_ioctl()
3800 	return phylink_mii_ioctl(bp->phylink, rq, cmd);  in macb_ioctl()
3823 	struct net_device *netdev = bp->dev;  in macb_set_rxcsum_feature()
3830 	if ((features & NETIF_F_RXCSUM) && !(netdev->flags & IFF_PROMISC))  in macb_set_rxcsum_feature()
3851 	netdev_features_t changed = features ^ netdev->features;  in macb_set_features()
3870 	struct net_device *netdev = bp->dev;  in macb_restore_features()
3871 	netdev_features_t features = netdev->features;  in macb_restore_features()
3881 	list_for_each_entry(item, &bp->rx_fs_list.list, list)  in macb_restore_features()
3882 		gem_prog_cmp_regs(bp, &item->fs);  in macb_restore_features()
3913 		bp->caps = dt_conf->caps;  in macb_configure_caps()
3915 	if (hw_is_gem(bp->regs, bp->native_io)) {  in macb_configure_caps()
3916 		bp->caps |= MACB_CAPS_MACB_IS_GEM;  in macb_configure_caps()
3920 			bp->caps |= MACB_CAPS_ISR_CLEAR_ON_WRITE;  in macb_configure_caps()
3922 			bp->caps |= MACB_CAPS_PCS;  in macb_configure_caps()
3925 			bp->caps |= MACB_CAPS_HIGH_SPEED;  in macb_configure_caps()
3928 			bp->caps |= MACB_CAPS_FIFO_MODE;  in macb_configure_caps()
3931 				dev_err(&bp->pdev->dev,  in macb_configure_caps()
3935 				bp->hw_dma_cap |= HW_DMA_CAP_PTP;  in macb_configure_caps()
3936 				bp->ptp_info = &gem_ptp_info;  in macb_configure_caps()
3942 	dev_dbg(&bp->pdev->dev, "Cadence caps 0x%08x\n", bp->caps);  in macb_configure_caps()
3988 	pdata = dev_get_platdata(&pdev->dev);  in macb_clk_init()
3990 		*pclk = pdata->pclk;  in macb_clk_init()
3991 		*hclk = pdata->hclk;  in macb_clk_init()
3993 		*pclk = devm_clk_get(&pdev->dev, "pclk");  in macb_clk_init()
3994 		*hclk = devm_clk_get(&pdev->dev, "hclk");  in macb_clk_init()
3998 		return dev_err_probe(&pdev->dev,  in macb_clk_init()
3999 				     IS_ERR(*pclk) ? PTR_ERR(*pclk) : -ENODEV,  in macb_clk_init()
4003 		return dev_err_probe(&pdev->dev,  in macb_clk_init()
4004 				     IS_ERR(*hclk) ? PTR_ERR(*hclk) : -ENODEV,  in macb_clk_init()
4007 	*tx_clk = devm_clk_get_optional(&pdev->dev, "tx_clk");  in macb_clk_init()
4011 	*rx_clk = devm_clk_get_optional(&pdev->dev, "rx_clk");  in macb_clk_init()
4015 	*tsu_clk = devm_clk_get_optional(&pdev->dev, "tsu_clk");  in macb_clk_init()
4021 		dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);  in macb_clk_init()
4027 		dev_err(&pdev->dev, "failed to enable hclk (%d)\n", err);  in macb_clk_init()
4033 		dev_err(&pdev->dev, "failed to enable tx_clk (%d)\n", err);  in macb_clk_init()
4039 		dev_err(&pdev->dev, "failed to enable rx_clk (%d)\n", err);  in macb_clk_init()
4045 		dev_err(&pdev->dev, "failed to enable tsu_clk (%d)\n", err);  in macb_clk_init()
4075 	bp->tx_ring_size = DEFAULT_TX_RING_SIZE;  in macb_init()
4076 	bp->rx_ring_size = DEFAULT_RX_RING_SIZE;  in macb_init()
4083 		if (!(bp->queue_mask & (1 << hw_q)))  in macb_init()
4086 		queue = &bp->queues[q];  in macb_init()
4087 		queue->bp = bp;  in macb_init()
4088 		spin_lock_init(&queue->tx_ptr_lock);  in macb_init()
4089 		netif_napi_add(dev, &queue->napi_rx, macb_rx_poll);  in macb_init()
4090 		netif_napi_add(dev, &queue->napi_tx, macb_tx_poll);  in macb_init()
4092 			queue->ISR  = GEM_ISR(hw_q - 1);  in macb_init()
4093 			queue->IER  = GEM_IER(hw_q - 1);  in macb_init()
4094 			queue->IDR  = GEM_IDR(hw_q - 1);  in macb_init()
4095 			queue->IMR  = GEM_IMR(hw_q - 1);  in macb_init()
4096 			queue->TBQP = GEM_TBQP(hw_q - 1);  in macb_init()
4097 			queue->RBQP = GEM_RBQP(hw_q - 1);  in macb_init()
4098 			queue->RBQS = GEM_RBQS(hw_q - 1);  in macb_init()
4100 			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_init()
4101 				queue->TBQPH = GEM_TBQPH(hw_q - 1);  in macb_init()
4102 				queue->RBQPH = GEM_RBQPH(hw_q - 1);  in macb_init()
4107 			queue->ISR  = MACB_ISR;  in macb_init()
4108 			queue->IER  = MACB_IER;  in macb_init()
4109 			queue->IDR  = MACB_IDR;  in macb_init()
4110 			queue->IMR  = MACB_IMR;  in macb_init()
4111 			queue->TBQP = MACB_TBQP;  in macb_init()
4112 			queue->RBQP = MACB_RBQP;  in macb_init()
4114 			if (bp->hw_dma_cap & HW_DMA_CAP_64B) {  in macb_init()
4115 				queue->TBQPH = MACB_TBQPH;  in macb_init()
4116 				queue->RBQPH = MACB_RBQPH;  in macb_init()
4126 		queue->irq = platform_get_irq(pdev, q);  in macb_init()
4127 		err = devm_request_irq(&pdev->dev, queue->irq, macb_interrupt,  in macb_init()
4128 				       IRQF_SHARED, dev->name, queue);  in macb_init()
4130 			dev_err(&pdev->dev,  in macb_init()
4132 				queue->irq, err);  in macb_init()
4136 		INIT_WORK(&queue->tx_error_task, macb_tx_error_task);  in macb_init()
4140 	dev->netdev_ops = &macb_netdev_ops;  in macb_init()
4144 		bp->macbgem_ops.mog_alloc_rx_buffers = gem_alloc_rx_buffers;  in macb_init()
4145 		bp->macbgem_ops.mog_free_rx_buffers = gem_free_rx_buffers;  in macb_init()
4146 		bp->macbgem_ops.mog_init_rings = gem_init_rings;  in macb_init()
4147 		bp->macbgem_ops.mog_rx = gem_rx;  in macb_init()
4148 		dev->ethtool_ops = &gem_ethtool_ops;  in macb_init()
4150 		bp->macbgem_ops.mog_alloc_rx_buffers = macb_alloc_rx_buffers;  in macb_init()
4151 		bp->macbgem_ops.mog_free_rx_buffers = macb_free_rx_buffers;  in macb_init()
4152 		bp->macbgem_ops.mog_init_rings = macb_init_rings;  in macb_init()
4153 		bp->macbgem_ops.mog_rx = macb_rx;  in macb_init()
4154 		dev->ethtool_ops = &macb_ethtool_ops;  in macb_init()
4157 	dev->priv_flags |= IFF_LIVE_ADDR_CHANGE;  in macb_init()
4160 	dev->hw_features = NETIF_F_SG;  in macb_init()
4164 		dev->hw_features |= MACB_NETIF_LSO;  in macb_init()
4167 	if (macb_is_gem(bp) && !(bp->caps & MACB_CAPS_FIFO_MODE))  in macb_init()
4168 		dev->hw_features |= NETIF_F_HW_CSUM | NETIF_F_RXCSUM;  in macb_init()
4169 	if (bp->caps & MACB_CAPS_SG_DISABLED)  in macb_init()
4170 		dev->hw_features &= ~NETIF_F_SG;  in macb_init()
4171 	dev->features = dev->hw_features;  in macb_init()
4175 	 * each 4-tuple define requires 1 T2 screener reg + 3 compare regs  in macb_init()
4178 	bp->max_tuples = min((GEM_BFEXT(SCR2CMP, reg) / 3),  in macb_init()
4180 	INIT_LIST_HEAD(&bp->rx_fs_list.list);  in macb_init()
4181 	if (bp->max_tuples > 0) {  in macb_init()
4189 			dev->hw_features |= NETIF_F_NTUPLE;  in macb_init()
4191 			bp->rx_fs_list.count = 0;  in macb_init()
4192 			spin_lock_init(&bp->rx_fs_lock);  in macb_init()
4194 			bp->max_tuples = 0;  in macb_init()
4197 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED)) {  in macb_init()
4199 		if (phy_interface_mode_is_rgmii(bp->phy_interface))  in macb_init()
4200 			val = bp->usrio->rgmii;  in macb_init()
4201 		else if (bp->phy_interface == PHY_INTERFACE_MODE_RMII &&  in macb_init()
4202 			 (bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))  in macb_init()
4203 			val = bp->usrio->rmii;  in macb_init()
4204 		else if (!(bp->caps & MACB_CAPS_USRIO_DEFAULT_IS_MII_GMII))  in macb_init()
4205 			val = bp->usrio->mii;  in macb_init()
4207 		if (bp->caps & MACB_CAPS_USRIO_HAS_CLKEN)  in macb_init()
4208 			val |= bp->usrio->refclk;  in macb_init()
4216 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII)  in macb_init()
4238 static int at91ether_alloc_coherent(struct macb *lp)  in at91ether_alloc_coherent()  argument
4240 	struct macb_queue *q = &lp->queues[0];  in at91ether_alloc_coherent()
4242 	q->rx_ring = dma_alloc_coherent(&lp->pdev->dev,  in at91ether_alloc_coherent()
4244 					  macb_dma_desc_get_size(lp)),  in at91ether_alloc_coherent()
4245 					 &q->rx_ring_dma, GFP_KERNEL);  in at91ether_alloc_coherent()
4246 	if (!q->rx_ring)  in at91ether_alloc_coherent()
4247 		return -ENOMEM;  in at91ether_alloc_coherent()
4249 	q->rx_buffers = dma_alloc_coherent(&lp->pdev->dev,  in at91ether_alloc_coherent()
4252 					    &q->rx_buffers_dma, GFP_KERNEL);  in at91ether_alloc_coherent()
4253 	if (!q->rx_buffers) {  in at91ether_alloc_coherent()
4254 		dma_free_coherent(&lp->pdev->dev,  in at91ether_alloc_coherent()
4256 				  macb_dma_desc_get_size(lp),  in at91ether_alloc_coherent()
4257 				  q->rx_ring, q->rx_ring_dma);  in at91ether_alloc_coherent()
4258 		q->rx_ring = NULL;  in at91ether_alloc_coherent()
4259 		return -ENOMEM;  in at91ether_alloc_coherent()
4265 static void at91ether_free_coherent(struct macb *lp)  in at91ether_free_coherent()  argument
4267 	struct macb_queue *q = &lp->queues[0];  in at91ether_free_coherent()
4269 	if (q->rx_ring) {  in at91ether_free_coherent()
4270 		dma_free_coherent(&lp->pdev->dev,  in at91ether_free_coherent()
4272 				  macb_dma_desc_get_size(lp),  in at91ether_free_coherent()
4273 				  q->rx_ring, q->rx_ring_dma);  in at91ether_free_coherent()
4274 		q->rx_ring = NULL;  in at91ether_free_coherent()
4277 	if (q->rx_buffers) {  in at91ether_free_coherent()
4278 		dma_free_coherent(&lp->pdev->dev,  in at91ether_free_coherent()
4281 				  q->rx_buffers, q->rx_buffers_dma);  in at91ether_free_coherent()
4282 		q->rx_buffers = NULL;  in at91ether_free_coherent()
4287 static int at91ether_start(struct macb *lp)  in at91ether_start()  argument
4289 	struct macb_queue *q = &lp->queues[0];  in at91ether_start()
4295 	ret = at91ether_alloc_coherent(lp);  in at91ether_start()
4299 	addr = q->rx_buffers_dma;  in at91ether_start()
4302 		macb_set_addr(lp, desc, addr);  in at91ether_start()
4303 		desc->ctrl = 0;  in at91ether_start()
4308 	desc->addr |= MACB_BIT(RX_WRAP);  in at91ether_start()
4311 	q->rx_tail = 0;  in at91ether_start()
4314 	macb_writel(lp, RBQP, q->rx_ring_dma);  in at91ether_start()
4317 	ctl = macb_readl(lp, NCR);  in at91ether_start()
4318 	macb_writel(lp, NCR, ctl | MACB_BIT(RE) | MACB_BIT(TE));  in at91ether_start()
4321 	macb_writel(lp, IER, MACB_BIT(RCOMP)	|  in at91ether_start()
4332 static void at91ether_stop(struct macb *lp)  in at91ether_stop()  argument
4337 	macb_writel(lp, IDR, MACB_BIT(RCOMP)	|  in at91ether_stop()
4346 	ctl = macb_readl(lp, NCR);  in at91ether_stop()
4347 	macb_writel(lp, NCR, ctl & ~(MACB_BIT(TE) | MACB_BIT(RE)));  in at91ether_stop()
4350 	at91ether_free_coherent(lp);  in at91ether_stop()
4356 	struct macb *lp = netdev_priv(dev);  in at91ether_open()  local
4360 	ret = pm_runtime_resume_and_get(&lp->pdev->dev);  in at91ether_open()
4365 	ctl = macb_readl(lp, NCR);  in at91ether_open()
4366 	macb_writel(lp, NCR, ctl | MACB_BIT(CLRSTAT));  in at91ether_open()
4368 	macb_set_hwaddr(lp);  in at91ether_open()
4370 	ret = at91ether_start(lp);  in at91ether_open()
4374 	ret = macb_phylink_connect(lp);  in at91ether_open()
4383 	at91ether_stop(lp);  in at91ether_open()
4385 	pm_runtime_put_sync(&lp->pdev->dev);  in at91ether_open()
4392 	struct macb *lp = netdev_priv(dev);  in at91ether_close()  local
4396 	phylink_stop(lp->phylink);  in at91ether_close()
4397 	phylink_disconnect_phy(lp->phylink);  in at91ether_close()
4399 	at91ether_stop(lp);  in at91ether_close()
4401 	return pm_runtime_put(&lp->pdev->dev);  in at91ether_close()
4408 	struct macb *lp = netdev_priv(dev);  in at91ether_start_xmit()  local
4410 	if (macb_readl(lp, TSR) & MACB_BIT(RM9200_BNQ)) {  in at91ether_start_xmit()
4416 		lp->rm9200_txq[desc].skb = skb;  in at91ether_start_xmit()
4417 		lp->rm9200_txq[desc].size = skb->len;  in at91ether_start_xmit()
4418 		lp->rm9200_txq[desc].mapping = dma_map_single(&lp->pdev->dev, skb->data,  in at91ether_start_xmit()
4419 							      skb->len, DMA_TO_DEVICE);  in at91ether_start_xmit()
4420 		if (dma_mapping_error(&lp->pdev->dev, lp->rm9200_txq[desc].mapping)) {  in at91ether_start_xmit()
4422 			dev->stats.tx_dropped++;  in at91ether_start_xmit()
4428 		macb_writel(lp, TAR, lp->rm9200_txq[desc].mapping);  in at91ether_start_xmit()
4430 		macb_writel(lp, TCR, skb->len);  in at91ether_start_xmit()
4445 	struct macb *lp = netdev_priv(dev);  in at91ether_rx()  local
4446 	struct macb_queue *q = &lp->queues[0];  in at91ether_rx()
4452 	desc = macb_rx_desc(q, q->rx_tail);  in at91ether_rx()
4453 	while (desc->addr & MACB_BIT(RX_USED)) {  in at91ether_rx()
4454 		p_recv = q->rx_buffers + q->rx_tail * AT91ETHER_MAX_RBUFF_SZ;  in at91ether_rx()
4455 		pktlen = MACB_BF(RX_FRMLEN, desc->ctrl);  in at91ether_rx()
4461 			skb->protocol = eth_type_trans(skb, dev);  in at91ether_rx()
4462 			dev->stats.rx_packets++;  in at91ether_rx()
4463 			dev->stats.rx_bytes += pktlen;  in at91ether_rx()
4466 			dev->stats.rx_dropped++;  in at91ether_rx()
4469 		if (desc->ctrl & MACB_BIT(RX_MHASH_MATCH))  in at91ether_rx()
4470 			dev->stats.multicast++;  in at91ether_rx()
4473 		desc->addr &= ~MACB_BIT(RX_USED);  in at91ether_rx()
4476 		if (q->rx_tail == AT91ETHER_MAX_RX_DESCR - 1)  in at91ether_rx()
4477 			q->rx_tail = 0;  in at91ether_rx()
4479 			q->rx_tail++;  in at91ether_rx()
4481 		desc = macb_rx_desc(q, q->rx_tail);  in at91ether_rx()
4489 	struct macb *lp = netdev_priv(dev);  in at91ether_interrupt()  local
4496 	intstatus = macb_readl(lp, ISR);  in at91ether_interrupt()
4506 			dev->stats.tx_errors++;  in at91ether_interrupt()
4509 		if (lp->rm9200_txq[desc].skb) {  in at91ether_interrupt()
4510 			dev_consume_skb_irq(lp->rm9200_txq[desc].skb);  in at91ether_interrupt()
4511 			lp->rm9200_txq[desc].skb = NULL;  in at91ether_interrupt()
4512 			dma_unmap_single(&lp->pdev->dev, lp->rm9200_txq[desc].mapping,  in at91ether_interrupt()
4513 					 lp->rm9200_txq[desc].size, DMA_TO_DEVICE);  in at91ether_interrupt()
4514 			dev->stats.tx_packets++;  in at91ether_interrupt()
4515 			dev->stats.tx_bytes += lp->rm9200_txq[desc].size;  in at91ether_interrupt()
4520 	/* Work-around for EMAC Errata section 41.3.1 */  in at91ether_interrupt()
4522 		ctl = macb_readl(lp, NCR);  in at91ether_interrupt()
4523 		macb_writel(lp, NCR, ctl & ~MACB_BIT(RE));  in at91ether_interrupt()
4525 		macb_writel(lp, NCR, ctl | MACB_BIT(RE));  in at91ether_interrupt()
4540 	at91ether_interrupt(dev->irq, dev);  in at91ether_poll_controller()
4570 	*pclk = devm_clk_get(&pdev->dev, "ether_clk");  in at91ether_clk_init()
4576 		dev_err(&pdev->dev, "failed to enable pclk (%d)\n", err);  in at91ether_clk_init()
4589 	bp->queues[0].bp = bp;  in at91ether_init()
4591 	dev->netdev_ops = &at91ether_netdev_ops;  in at91ether_init()
4592 	dev->ethtool_ops = &macb_ethtool_ops;  in at91ether_init()
4594 	err = devm_request_irq(&pdev->dev, dev->irq, at91ether_interrupt,  in at91ether_init()
4595 			       0, dev->name, dev);  in at91ether_init()
4609 	return mgmt->rate;  in fu540_macb_tx_recalc_rate()
4612 static long fu540_macb_tx_round_rate(struct clk_hw *hw, unsigned long rate,  in fu540_macb_tx_round_rate()  argument
4615 	if (WARN_ON(rate < 2500000))  in fu540_macb_tx_round_rate()
4617 	else if (rate == 2500000)  in fu540_macb_tx_round_rate()
4619 	else if (WARN_ON(rate < 13750000))  in fu540_macb_tx_round_rate()
4621 	else if (WARN_ON(rate < 25000000))  in fu540_macb_tx_round_rate()
4623 	else if (rate == 25000000)  in fu540_macb_tx_round_rate()
4625 	else if (WARN_ON(rate < 75000000))  in fu540_macb_tx_round_rate()
4627 	else if (WARN_ON(rate < 125000000))  in fu540_macb_tx_round_rate()
4629 	else if (rate == 125000000)  in fu540_macb_tx_round_rate()
4632 	WARN_ON(rate > 125000000);  in fu540_macb_tx_round_rate()
4637 static int fu540_macb_tx_set_rate(struct clk_hw *hw, unsigned long rate,  in fu540_macb_tx_set_rate()  argument
4640 	rate = fu540_macb_tx_round_rate(hw, rate, &parent_rate);  in fu540_macb_tx_set_rate()
4641 	if (rate != 125000000)  in fu540_macb_tx_set_rate()
4642 		iowrite32(1, mgmt->reg);  in fu540_macb_tx_set_rate()
4644 		iowrite32(0, mgmt->reg);  in fu540_macb_tx_set_rate()
4645 	mgmt->rate = rate;  in fu540_macb_tx_set_rate()
4667 	mgmt = devm_kzalloc(&pdev->dev, sizeof(*mgmt), GFP_KERNEL);  in fu540_c000_clk_init()
4669 		err = -ENOMEM;  in fu540_c000_clk_init()
4673 	init.name = "sifive-gemgxl-mgmt";  in fu540_c000_clk_init()
4678 	mgmt->rate = 0;  in fu540_c000_clk_init()
4679 	mgmt->hw.init = &init;  in fu540_c000_clk_init()
4681 	*tx_clk = devm_clk_register(&pdev->dev, &mgmt->hw);  in fu540_c000_clk_init()
4689 		dev_err(&pdev->dev, "failed to enable tx_clk (%u)\n", err);  in fu540_c000_clk_init()
4693 		dev_info(&pdev->dev, "Registered clk switch '%s'\n", init.name);  in fu540_c000_clk_init()
4706 	mgmt->reg = devm_platform_ioremap_resource(pdev, 1);  in fu540_c000_init()
4707 	if (IS_ERR(mgmt->reg))  in fu540_c000_init()
4708 		return PTR_ERR(mgmt->reg);  in fu540_c000_init()
4719 	if (bp->phy_interface == PHY_INTERFACE_MODE_SGMII) {  in init_reset_optional()
4721 		bp->sgmii_phy = devm_phy_optional_get(&pdev->dev, NULL);  in init_reset_optional()
4723 		if (IS_ERR(bp->sgmii_phy))  in init_reset_optional()
4724 			return dev_err_probe(&pdev->dev, PTR_ERR(bp->sgmii_phy),  in init_reset_optional()
4727 		ret = phy_init(bp->sgmii_phy);  in init_reset_optional()
4729 			return dev_err_probe(&pdev->dev, ret,  in init_reset_optional()
4736 			ret = of_property_read_u32_array(pdev->dev.of_node, "power-domains",  in init_reset_optional()
4739 				dev_err(&pdev->dev, "Failed to read power management information\n");  in init_reset_optional()
4754 	ret = device_reset_optional(&pdev->dev);  in init_reset_optional()
4756 		phy_exit(bp->sgmii_phy);  in init_reset_optional()
4757 		return dev_err_probe(&pdev->dev, ret, "failed to reset controller");  in init_reset_optional()
4764 		phy_exit(bp->sgmii_phy);  in init_reset_optional()
4920 	{ .compatible = "cdns,at91sam9260-macb", .data = &at91sam9260_config },
4922 	{ .compatible = "cdns,np4-macb", .data = &np4_config },
4923 	{ .compatible = "cdns,pc302-gem", .data = &pc302gem_config },
4925 	{ .compatible = "cdns,sam9x60-macb", .data = &at91sam9260_config },
4926 	{ .compatible = "atmel,sama5d2-gem", .data = &sama5d2_config },
4927 	{ .compatible = "atmel,sama5d29-gem", .data = &sama5d29_config },
4928 	{ .compatible = "atmel,sama5d3-gem", .data = &sama5d3_config },
4929 	{ .compatible = "atmel,sama5d3-macb", .data = &sama5d3macb_config },
4930 	{ .compatible = "atmel,sama5d4-gem", .data = &sama5d4_config },
4931 	{ .compatible = "cdns,at91rm9200-emac", .data = &emac_config },
4933 	{ .compatible = "cdns,zynqmp-gem", .data = &zynqmp_config}, /* deprecated */
4934 	{ .compatible = "cdns,zynq-gem", .data = &zynq_config }, /* deprecated */
4935 	{ .compatible = "sifive,fu540-c000-gem", .data = &fu540_c000_config },
4936 	{ .compatible = "microchip,mpfs-macb", .data = &mpfs_config },
4937 	{ .compatible = "microchip,sama7g5-gem", .data = &sama7g5_gem_config },
4938 	{ .compatible = "microchip,sama7g5-emac", .data = &sama7g5_emac_config },
4939 	{ .compatible = "xlnx,zynqmp-gem", .data = &zynqmp_config},
4940 	{ .compatible = "xlnx,zynq-gem", .data = &zynq_config },
4941 	{ .compatible = "xlnx,versal-gem", .data = &versal_config},
4963 			struct clk **) = macb_config->clk_init;  in macb_probe()
4964 	int (*init)(struct platform_device *) = macb_config->init;  in macb_probe()
4965 	struct device_node *np = pdev->dev.of_node;  in macb_probe()
4986 		if (match && match->data) {  in macb_probe()
4987 			macb_config = match->data;  in macb_probe()
4988 			clk_init = macb_config->clk_init;  in macb_probe()
4989 			init = macb_config->init;  in macb_probe()
4997 	pm_runtime_set_autosuspend_delay(&pdev->dev, MACB_PM_TIMEOUT);  in macb_probe()
4998 	pm_runtime_use_autosuspend(&pdev->dev);  in macb_probe()
4999 	pm_runtime_get_noresume(&pdev->dev);  in macb_probe()
5000 	pm_runtime_set_active(&pdev->dev);  in macb_probe()
5001 	pm_runtime_enable(&pdev->dev);  in macb_probe()
5007 		err = -ENOMEM;  in macb_probe()
5011 	dev->base_addr = regs->start;  in macb_probe()
5013 	SET_NETDEV_DEV(dev, &pdev->dev);  in macb_probe()
5016 	bp->pdev = pdev;  in macb_probe()
5017 	bp->dev = dev;  in macb_probe()
5018 	bp->regs = mem;  in macb_probe()
5019 	bp->native_io = native_io;  in macb_probe()
5021 		bp->macb_reg_readl = hw_readl_native;  in macb_probe()
5022 		bp->macb_reg_writel = hw_writel_native;  in macb_probe()
5024 		bp->macb_reg_readl = hw_readl;  in macb_probe()
5025 		bp->macb_reg_writel = hw_writel;  in macb_probe()
5027 	bp->num_queues = num_queues;  in macb_probe()
5028 	bp->queue_mask = queue_mask;  in macb_probe()
5030 		bp->dma_burst_length = macb_config->dma_burst_length;  in macb_probe()
5031 	bp->pclk = pclk;  in macb_probe()
5032 	bp->hclk = hclk;  in macb_probe()
5033 	bp->tx_clk = tx_clk;  in macb_probe()
5034 	bp->rx_clk = rx_clk;  in macb_probe()
5035 	bp->tsu_clk = tsu_clk;  in macb_probe()
5037 		bp->jumbo_max_len = macb_config->jumbo_max_len;  in macb_probe()
5039 	if (!hw_is_gem(bp->regs, bp->native_io))  in macb_probe()
5040 		bp->max_tx_length = MACB_MAX_TX_LEN;  in macb_probe()
5041 	else if (macb_config->max_tx_length)  in macb_probe()
5042 		bp->max_tx_length = macb_config->max_tx_length;  in macb_probe()
5044 		bp->max_tx_length = GEM_MAX_TX_LEN;  in macb_probe()
5046 	bp->wol = 0;  in macb_probe()
5047 	if (of_property_read_bool(np, "magic-packet"))  in macb_probe()
5048 		bp->wol |= MACB_WOL_HAS_MAGIC_PACKET;  in macb_probe()
5049 	device_set_wakeup_capable(&pdev->dev, bp->wol & MACB_WOL_HAS_MAGIC_PACKET);  in macb_probe()
5051 	bp->usrio = macb_config->usrio;  in macb_probe()
5057 		err = of_property_read_u32(bp->pdev->dev.of_node,  in macb_probe()
5058 					   "cdns,rx-watermark",  in macb_probe()
5059 					   &bp->rx_watermark);  in macb_probe()
5065 			wtrmrk_rst_val = (1 << (GEM_BFEXT(RX_PBUF_ADDR, gem_readl(bp, DCFG2)))) - 1;  in macb_probe()
5066 			if (bp->rx_watermark > wtrmrk_rst_val || !bp->rx_watermark) {  in macb_probe()
5067 				dev_info(&bp->pdev->dev, "Invalid watermark value\n");  in macb_probe()
5068 				bp->rx_watermark = 0;  in macb_probe()
5072 	spin_lock_init(&bp->lock);  in macb_probe()
5073 	spin_lock_init(&bp->stats_lock);  in macb_probe()
5080 		dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(44));  in macb_probe()
5081 		bp->hw_dma_cap |= HW_DMA_CAP_64B;  in macb_probe()
5086 	dev->irq = platform_get_irq(pdev, 0);  in macb_probe()
5087 	if (dev->irq < 0) {  in macb_probe()
5088 		err = dev->irq;  in macb_probe()
5092 	/* MTU range: 68 - 1500 or 10240 */  in macb_probe()
5093 	dev->min_mtu = GEM_MTU_MIN_SIZE;  in macb_probe()
5094 	if ((bp->caps & MACB_CAPS_JUMBO) && bp->jumbo_max_len)  in macb_probe()
5095 		dev->max_mtu = bp->jumbo_max_len - ETH_HLEN - ETH_FCS_LEN;  in macb_probe()
5097 		dev->max_mtu = ETH_DATA_LEN;  in macb_probe()
5099 	if (bp->caps & MACB_CAPS_BD_RD_PREFETCH) {  in macb_probe()
5102 			bp->rx_bd_rd_prefetch = (2 << (val - 1)) *  in macb_probe()
5107 			bp->tx_bd_rd_prefetch = (2 << (val - 1)) *  in macb_probe()
5111 	bp->rx_intr_mask = MACB_RX_INT_FLAGS;  in macb_probe()
5112 	if (bp->caps & MACB_CAPS_NEEDS_RSTONUBR)  in macb_probe()
5113 		bp->rx_intr_mask |= MACB_BIT(RXUBR);  in macb_probe()
5115 	err = of_get_ethdev_address(np, bp->dev);  in macb_probe()
5116 	if (err == -EPROBE_DEFER)  in macb_probe()
5124 		bp->phy_interface = PHY_INTERFACE_MODE_MII;  in macb_probe()
5126 		bp->phy_interface = interface;  in macb_probe()
5141 		dev_err(&pdev->dev, "Cannot register net device, aborting.\n");  in macb_probe()
5145 	tasklet_setup(&bp->hresp_err_tasklet, macb_hresp_error_task);  in macb_probe()
5149 		    dev->base_addr, dev->irq, dev->dev_addr);  in macb_probe()
5151 	pm_runtime_mark_last_busy(&bp->pdev->dev);  in macb_probe()
5152 	pm_runtime_put_autosuspend(&bp->pdev->dev);  in macb_probe()
5157 	mdiobus_unregister(bp->mii_bus);  in macb_probe()
5158 	mdiobus_free(bp->mii_bus);  in macb_probe()
5161 	phy_exit(bp->sgmii_phy);  in macb_probe()
5168 	pm_runtime_disable(&pdev->dev);  in macb_probe()
5169 	pm_runtime_set_suspended(&pdev->dev);  in macb_probe()
5170 	pm_runtime_dont_use_autosuspend(&pdev->dev);  in macb_probe()
5184 		phy_exit(bp->sgmii_phy);  in macb_remove()
5185 		mdiobus_unregister(bp->mii_bus);  in macb_remove()
5186 		mdiobus_free(bp->mii_bus);  in macb_remove()
5189 		tasklet_kill(&bp->hresp_err_tasklet);  in macb_remove()
5190 		pm_runtime_disable(&pdev->dev);  in macb_remove()
5191 		pm_runtime_dont_use_autosuspend(&pdev->dev);  in macb_remove()
5192 		if (!pm_runtime_suspended(&pdev->dev)) {  in macb_remove()
5193 			macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk,  in macb_remove()
5194 					  bp->rx_clk, bp->tsu_clk);  in macb_remove()
5195 			pm_runtime_set_suspended(&pdev->dev);  in macb_remove()
5197 		phylink_destroy(bp->phylink);  in macb_remove()
5213 	if (!device_may_wakeup(&bp->dev->dev))  in macb_suspend()
5214 		phy_exit(bp->sgmii_phy);  in macb_suspend()
5219 	if (bp->wol & MACB_WOL_ENABLED) {  in macb_suspend()
5220 		spin_lock_irqsave(&bp->lock, flags);  in macb_suspend()
5222 		macb_writel(bp, TSR, -1);  in macb_suspend()
5223 		macb_writel(bp, RSR, -1);  in macb_suspend()
5224 		for (q = 0, queue = bp->queues; q < bp->num_queues;  in macb_suspend()
5227 			queue_writel(queue, IDR, -1);  in macb_suspend()
5229 			if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_suspend()
5230 				queue_writel(queue, ISR, -1);  in macb_suspend()
5235 		devm_free_irq(dev, bp->queues[0].irq, bp->queues);  in macb_suspend()
5237 			err = devm_request_irq(dev, bp->queues[0].irq, gem_wol_interrupt,  in macb_suspend()
5238 					       IRQF_SHARED, netdev->name, bp->queues);  in macb_suspend()
5242 					bp->queues[0].irq, err);  in macb_suspend()
5243 				spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5246 			queue_writel(bp->queues, IER, GEM_BIT(WOL));  in macb_suspend()
5249 			err = devm_request_irq(dev, bp->queues[0].irq, macb_wol_interrupt,  in macb_suspend()
5250 					       IRQF_SHARED, netdev->name, bp->queues);  in macb_suspend()
5254 					bp->queues[0].irq, err);  in macb_suspend()
5255 				spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5258 			queue_writel(bp->queues, IER, MACB_BIT(WOL));  in macb_suspend()
5261 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5263 		enable_irq_wake(bp->queues[0].irq);  in macb_suspend()
5267 	for (q = 0, queue = bp->queues; q < bp->num_queues;  in macb_suspend()
5269 		napi_disable(&queue->napi_rx);  in macb_suspend()
5270 		napi_disable(&queue->napi_tx);  in macb_suspend()
5273 	if (!(bp->wol & MACB_WOL_ENABLED)) {  in macb_suspend()
5275 		phylink_stop(bp->phylink);  in macb_suspend()
5277 		spin_lock_irqsave(&bp->lock, flags);  in macb_suspend()
5279 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_suspend()
5282 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))  in macb_suspend()
5283 		bp->pm_data.usrio = macb_or_gem_readl(bp, USRIO);  in macb_suspend()
5285 	if (netdev->hw_features & NETIF_F_NTUPLE)  in macb_suspend()
5286 		bp->pm_data.scrt2 = gem_readl_n(bp, ETHT, SCRT2_ETHT);  in macb_suspend()
5288 	if (bp->ptp_info)  in macb_suspend()
5289 		bp->ptp_info->ptp_remove(netdev);  in macb_suspend()
5305 	if (!device_may_wakeup(&bp->dev->dev))  in macb_resume()
5306 		phy_init(bp->sgmii_phy);  in macb_resume()
5314 	if (bp->wol & MACB_WOL_ENABLED) {  in macb_resume()
5315 		spin_lock_irqsave(&bp->lock, flags);  in macb_resume()
5318 			queue_writel(bp->queues, IDR, GEM_BIT(WOL));  in macb_resume()
5321 			queue_writel(bp->queues, IDR, MACB_BIT(WOL));  in macb_resume()
5325 		queue_readl(bp->queues, ISR);  in macb_resume()
5326 		if (bp->caps & MACB_CAPS_ISR_CLEAR_ON_WRITE)  in macb_resume()
5327 			queue_writel(bp->queues, ISR, -1);  in macb_resume()
5329 		devm_free_irq(dev, bp->queues[0].irq, bp->queues);  in macb_resume()
5330 		err = devm_request_irq(dev, bp->queues[0].irq, macb_interrupt,  in macb_resume()
5331 				       IRQF_SHARED, netdev->name, bp->queues);  in macb_resume()
5335 				bp->queues[0].irq, err);  in macb_resume()
5336 			spin_unlock_irqrestore(&bp->lock, flags);  in macb_resume()
5339 		spin_unlock_irqrestore(&bp->lock, flags);  in macb_resume()
5341 		disable_irq_wake(bp->queues[0].irq);  in macb_resume()
5347 		phylink_stop(bp->phylink);  in macb_resume()
5351 	for (q = 0, queue = bp->queues; q < bp->num_queues;  in macb_resume()
5353 		napi_enable(&queue->napi_rx);  in macb_resume()
5354 		napi_enable(&queue->napi_tx);  in macb_resume()
5357 	if (netdev->hw_features & NETIF_F_NTUPLE)  in macb_resume()
5358 		gem_writel_n(bp, ETHT, SCRT2_ETHT, bp->pm_data.scrt2);  in macb_resume()
5360 	if (!(bp->caps & MACB_CAPS_USRIO_DISABLED))  in macb_resume()
5361 		macb_or_gem_writel(bp, USRIO, bp->pm_data.usrio);  in macb_resume()
5369 	phylink_start(bp->phylink);  in macb_resume()
5373 	if (bp->ptp_info)  in macb_resume()
5374 		bp->ptp_info->ptp_init(netdev);  in macb_resume()
5385 		macb_clks_disable(bp->pclk, bp->hclk, bp->tx_clk, bp->rx_clk, bp->tsu_clk);  in macb_runtime_suspend()
5386 	else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK))  in macb_runtime_suspend()
5387 		macb_clks_disable(NULL, NULL, NULL, NULL, bp->tsu_clk);  in macb_runtime_suspend()
5398 		clk_prepare_enable(bp->pclk);  in macb_runtime_resume()
5399 		clk_prepare_enable(bp->hclk);  in macb_runtime_resume()
5400 		clk_prepare_enable(bp->tx_clk);  in macb_runtime_resume()
5401 		clk_prepare_enable(bp->rx_clk);  in macb_runtime_resume()
5402 		clk_prepare_enable(bp->tsu_clk);  in macb_runtime_resume()
5403 	} else if (!(bp->caps & MACB_CAPS_NEED_TSUCLK)) {  in macb_runtime_resume()
5404 		clk_prepare_enable(bp->tsu_clk);  in macb_runtime_resume()