1 /* 2 * Driver for Marvell NETA network card for Armada XP and Armada 370 SoCs. 3 * 4 * Copyright (C) 2012 Marvell 5 * 6 * Rami Rosen <rosenr@marvell.com> 7 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> 8 * 9 * This file is licensed under the terms of the GNU General Public 10 * License version 2. This program is licensed "as is" without any 11 * warranty of any kind, whether express or implied. 12 */ 13 14 #include <linux/clk.h> 15 #include <linux/cpu.h> 16 #include <linux/etherdevice.h> 17 #include <linux/if_vlan.h> 18 #include <linux/inetdevice.h> 19 #include <linux/interrupt.h> 20 #include <linux/io.h> 21 #include <linux/kernel.h> 22 #include <linux/mbus.h> 23 #include <linux/module.h> 24 #include <linux/netdevice.h> 25 #include <linux/of.h> 26 #include <linux/of_address.h> 27 #include <linux/of_irq.h> 28 #include <linux/of_mdio.h> 29 #include <linux/of_net.h> 30 #include <linux/phy.h> 31 #include <linux/platform_device.h> 32 #include <linux/skbuff.h> 33 #include <net/hwbm.h> 34 #include "mvneta_bm.h" 35 #include <net/ip.h> 36 #include <net/ipv6.h> 37 #include <net/tso.h> 38 39 /* Registers */ 40 #define MVNETA_RXQ_CONFIG_REG(q) (0x1400 + ((q) << 2)) 41 #define MVNETA_RXQ_HW_BUF_ALLOC BIT(0) 42 #define MVNETA_RXQ_SHORT_POOL_ID_SHIFT 4 43 #define MVNETA_RXQ_SHORT_POOL_ID_MASK 0x30 44 #define MVNETA_RXQ_LONG_POOL_ID_SHIFT 6 45 #define MVNETA_RXQ_LONG_POOL_ID_MASK 0xc0 46 #define MVNETA_RXQ_PKT_OFFSET_ALL_MASK (0xf << 8) 47 #define MVNETA_RXQ_PKT_OFFSET_MASK(offs) ((offs) << 8) 48 #define MVNETA_RXQ_THRESHOLD_REG(q) (0x14c0 + ((q) << 2)) 49 #define MVNETA_RXQ_NON_OCCUPIED(v) ((v) << 16) 50 #define MVNETA_RXQ_BASE_ADDR_REG(q) (0x1480 + ((q) << 2)) 51 #define MVNETA_RXQ_SIZE_REG(q) (0x14a0 + ((q) << 2)) 52 #define MVNETA_RXQ_BUF_SIZE_SHIFT 19 53 #define MVNETA_RXQ_BUF_SIZE_MASK (0x1fff << 19) 54 #define MVNETA_RXQ_STATUS_REG(q) (0x14e0 + ((q) << 2)) 55 #define MVNETA_RXQ_OCCUPIED_ALL_MASK 0x3fff 56 #define MVNETA_RXQ_STATUS_UPDATE_REG(q) (0x1500 + ((q) << 2)) 57 #define MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT 16 58 #define MVNETA_RXQ_ADD_NON_OCCUPIED_MAX 255 59 #define MVNETA_PORT_POOL_BUFFER_SZ_REG(pool) (0x1700 + ((pool) << 2)) 60 #define MVNETA_PORT_POOL_BUFFER_SZ_SHIFT 3 61 #define MVNETA_PORT_POOL_BUFFER_SZ_MASK 0xfff8 62 #define MVNETA_PORT_RX_RESET 0x1cc0 63 #define MVNETA_PORT_RX_DMA_RESET BIT(0) 64 #define MVNETA_PHY_ADDR 0x2000 65 #define MVNETA_PHY_ADDR_MASK 0x1f 66 #define MVNETA_MBUS_RETRY 0x2010 67 #define MVNETA_UNIT_INTR_CAUSE 0x2080 68 #define MVNETA_UNIT_CONTROL 0x20B0 69 #define MVNETA_PHY_POLLING_ENABLE BIT(1) 70 #define MVNETA_WIN_BASE(w) (0x2200 + ((w) << 3)) 71 #define MVNETA_WIN_SIZE(w) (0x2204 + ((w) << 3)) 72 #define MVNETA_WIN_REMAP(w) (0x2280 + ((w) << 2)) 73 #define MVNETA_BASE_ADDR_ENABLE 0x2290 74 #define MVNETA_ACCESS_PROTECT_ENABLE 0x2294 75 #define MVNETA_PORT_CONFIG 0x2400 76 #define MVNETA_UNI_PROMISC_MODE BIT(0) 77 #define MVNETA_DEF_RXQ(q) ((q) << 1) 78 #define MVNETA_DEF_RXQ_ARP(q) ((q) << 4) 79 #define MVNETA_TX_UNSET_ERR_SUM BIT(12) 80 #define MVNETA_DEF_RXQ_TCP(q) ((q) << 16) 81 #define MVNETA_DEF_RXQ_UDP(q) ((q) << 19) 82 #define MVNETA_DEF_RXQ_BPDU(q) ((q) << 22) 83 #define MVNETA_RX_CSUM_WITH_PSEUDO_HDR BIT(25) 84 #define MVNETA_PORT_CONFIG_DEFL_VALUE(q) (MVNETA_DEF_RXQ(q) | \ 85 MVNETA_DEF_RXQ_ARP(q) | \ 86 MVNETA_DEF_RXQ_TCP(q) | \ 87 MVNETA_DEF_RXQ_UDP(q) | \ 88 MVNETA_DEF_RXQ_BPDU(q) | \ 89 MVNETA_TX_UNSET_ERR_SUM | \ 90 MVNETA_RX_CSUM_WITH_PSEUDO_HDR) 91 #define MVNETA_PORT_CONFIG_EXTEND 0x2404 92 #define MVNETA_MAC_ADDR_LOW 0x2414 93 #define MVNETA_MAC_ADDR_HIGH 0x2418 94 #define MVNETA_SDMA_CONFIG 0x241c 95 #define MVNETA_SDMA_BRST_SIZE_16 4 96 #define MVNETA_RX_BRST_SZ_MASK(burst) ((burst) << 1) 97 #define MVNETA_RX_NO_DATA_SWAP BIT(4) 98 #define MVNETA_TX_NO_DATA_SWAP BIT(5) 99 #define MVNETA_DESC_SWAP BIT(6) 100 #define MVNETA_TX_BRST_SZ_MASK(burst) ((burst) << 22) 101 #define MVNETA_PORT_STATUS 0x2444 102 #define MVNETA_TX_IN_PRGRS BIT(1) 103 #define MVNETA_TX_FIFO_EMPTY BIT(8) 104 #define MVNETA_RX_MIN_FRAME_SIZE 0x247c 105 #define MVNETA_SERDES_CFG 0x24A0 106 #define MVNETA_SGMII_SERDES_PROTO 0x0cc7 107 #define MVNETA_QSGMII_SERDES_PROTO 0x0667 108 #define MVNETA_TYPE_PRIO 0x24bc 109 #define MVNETA_FORCE_UNI BIT(21) 110 #define MVNETA_TXQ_CMD_1 0x24e4 111 #define MVNETA_TXQ_CMD 0x2448 112 #define MVNETA_TXQ_DISABLE_SHIFT 8 113 #define MVNETA_TXQ_ENABLE_MASK 0x000000ff 114 #define MVNETA_RX_DISCARD_FRAME_COUNT 0x2484 115 #define MVNETA_OVERRUN_FRAME_COUNT 0x2488 116 #define MVNETA_GMAC_CLOCK_DIVIDER 0x24f4 117 #define MVNETA_GMAC_1MS_CLOCK_ENABLE BIT(31) 118 #define MVNETA_ACC_MODE 0x2500 119 #define MVNETA_BM_ADDRESS 0x2504 120 #define MVNETA_CPU_MAP(cpu) (0x2540 + ((cpu) << 2)) 121 #define MVNETA_CPU_RXQ_ACCESS_ALL_MASK 0x000000ff 122 #define MVNETA_CPU_TXQ_ACCESS_ALL_MASK 0x0000ff00 123 #define MVNETA_CPU_RXQ_ACCESS(rxq) BIT(rxq) 124 #define MVNETA_CPU_TXQ_ACCESS(txq) BIT(txq + 8) 125 #define MVNETA_RXQ_TIME_COAL_REG(q) (0x2580 + ((q) << 2)) 126 127 /* Exception Interrupt Port/Queue Cause register 128 * 129 * Their behavior depend of the mapping done using the PCPX2Q 130 * registers. For a given CPU if the bit associated to a queue is not 131 * set, then for the register a read from this CPU will always return 132 * 0 and a write won't do anything 133 */ 134 135 #define MVNETA_INTR_NEW_CAUSE 0x25a0 136 #define MVNETA_INTR_NEW_MASK 0x25a4 137 138 /* bits 0..7 = TXQ SENT, one bit per queue. 139 * bits 8..15 = RXQ OCCUP, one bit per queue. 140 * bits 16..23 = RXQ FREE, one bit per queue. 141 * bit 29 = OLD_REG_SUM, see old reg ? 142 * bit 30 = TX_ERR_SUM, one bit for 4 ports 143 * bit 31 = MISC_SUM, one bit for 4 ports 144 */ 145 #define MVNETA_TX_INTR_MASK(nr_txqs) (((1 << nr_txqs) - 1) << 0) 146 #define MVNETA_TX_INTR_MASK_ALL (0xff << 0) 147 #define MVNETA_RX_INTR_MASK(nr_rxqs) (((1 << nr_rxqs) - 1) << 8) 148 #define MVNETA_RX_INTR_MASK_ALL (0xff << 8) 149 #define MVNETA_MISCINTR_INTR_MASK BIT(31) 150 151 #define MVNETA_INTR_OLD_CAUSE 0x25a8 152 #define MVNETA_INTR_OLD_MASK 0x25ac 153 154 /* Data Path Port/Queue Cause Register */ 155 #define MVNETA_INTR_MISC_CAUSE 0x25b0 156 #define MVNETA_INTR_MISC_MASK 0x25b4 157 158 #define MVNETA_CAUSE_PHY_STATUS_CHANGE BIT(0) 159 #define MVNETA_CAUSE_LINK_CHANGE BIT(1) 160 #define MVNETA_CAUSE_PTP BIT(4) 161 162 #define MVNETA_CAUSE_INTERNAL_ADDR_ERR BIT(7) 163 #define MVNETA_CAUSE_RX_OVERRUN BIT(8) 164 #define MVNETA_CAUSE_RX_CRC_ERROR BIT(9) 165 #define MVNETA_CAUSE_RX_LARGE_PKT BIT(10) 166 #define MVNETA_CAUSE_TX_UNDERUN BIT(11) 167 #define MVNETA_CAUSE_PRBS_ERR BIT(12) 168 #define MVNETA_CAUSE_PSC_SYNC_CHANGE BIT(13) 169 #define MVNETA_CAUSE_SERDES_SYNC_ERR BIT(14) 170 171 #define MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT 16 172 #define MVNETA_CAUSE_BMU_ALLOC_ERR_ALL_MASK (0xF << MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT) 173 #define MVNETA_CAUSE_BMU_ALLOC_ERR_MASK(pool) (1 << (MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT + (pool))) 174 175 #define MVNETA_CAUSE_TXQ_ERROR_SHIFT 24 176 #define MVNETA_CAUSE_TXQ_ERROR_ALL_MASK (0xFF << MVNETA_CAUSE_TXQ_ERROR_SHIFT) 177 #define MVNETA_CAUSE_TXQ_ERROR_MASK(q) (1 << (MVNETA_CAUSE_TXQ_ERROR_SHIFT + (q))) 178 179 #define MVNETA_INTR_ENABLE 0x25b8 180 #define MVNETA_TXQ_INTR_ENABLE_ALL_MASK 0x0000ff00 181 #define MVNETA_RXQ_INTR_ENABLE_ALL_MASK 0x000000ff 182 183 #define MVNETA_RXQ_CMD 0x2680 184 #define MVNETA_RXQ_DISABLE_SHIFT 8 185 #define MVNETA_RXQ_ENABLE_MASK 0x000000ff 186 #define MVETH_TXQ_TOKEN_COUNT_REG(q) (0x2700 + ((q) << 4)) 187 #define MVETH_TXQ_TOKEN_CFG_REG(q) (0x2704 + ((q) << 4)) 188 #define MVNETA_GMAC_CTRL_0 0x2c00 189 #define MVNETA_GMAC_MAX_RX_SIZE_SHIFT 2 190 #define MVNETA_GMAC_MAX_RX_SIZE_MASK 0x7ffc 191 #define MVNETA_GMAC0_PORT_ENABLE BIT(0) 192 #define MVNETA_GMAC_CTRL_2 0x2c08 193 #define MVNETA_GMAC2_INBAND_AN_ENABLE BIT(0) 194 #define MVNETA_GMAC2_PCS_ENABLE BIT(3) 195 #define MVNETA_GMAC2_PORT_RGMII BIT(4) 196 #define MVNETA_GMAC2_PORT_RESET BIT(6) 197 #define MVNETA_GMAC_STATUS 0x2c10 198 #define MVNETA_GMAC_LINK_UP BIT(0) 199 #define MVNETA_GMAC_SPEED_1000 BIT(1) 200 #define MVNETA_GMAC_SPEED_100 BIT(2) 201 #define MVNETA_GMAC_FULL_DUPLEX BIT(3) 202 #define MVNETA_GMAC_RX_FLOW_CTRL_ENABLE BIT(4) 203 #define MVNETA_GMAC_TX_FLOW_CTRL_ENABLE BIT(5) 204 #define MVNETA_GMAC_RX_FLOW_CTRL_ACTIVE BIT(6) 205 #define MVNETA_GMAC_TX_FLOW_CTRL_ACTIVE BIT(7) 206 #define MVNETA_GMAC_AUTONEG_CONFIG 0x2c0c 207 #define MVNETA_GMAC_FORCE_LINK_DOWN BIT(0) 208 #define MVNETA_GMAC_FORCE_LINK_PASS BIT(1) 209 #define MVNETA_GMAC_INBAND_AN_ENABLE BIT(2) 210 #define MVNETA_GMAC_CONFIG_MII_SPEED BIT(5) 211 #define MVNETA_GMAC_CONFIG_GMII_SPEED BIT(6) 212 #define MVNETA_GMAC_AN_SPEED_EN BIT(7) 213 #define MVNETA_GMAC_AN_FLOW_CTRL_EN BIT(11) 214 #define MVNETA_GMAC_CONFIG_FULL_DUPLEX BIT(12) 215 #define MVNETA_GMAC_AN_DUPLEX_EN BIT(13) 216 #define MVNETA_MIB_COUNTERS_BASE 0x3000 217 #define MVNETA_MIB_LATE_COLLISION 0x7c 218 #define MVNETA_DA_FILT_SPEC_MCAST 0x3400 219 #define MVNETA_DA_FILT_OTH_MCAST 0x3500 220 #define MVNETA_DA_FILT_UCAST_BASE 0x3600 221 #define MVNETA_TXQ_BASE_ADDR_REG(q) (0x3c00 + ((q) << 2)) 222 #define MVNETA_TXQ_SIZE_REG(q) (0x3c20 + ((q) << 2)) 223 #define MVNETA_TXQ_SENT_THRESH_ALL_MASK 0x3fff0000 224 #define MVNETA_TXQ_SENT_THRESH_MASK(coal) ((coal) << 16) 225 #define MVNETA_TXQ_UPDATE_REG(q) (0x3c60 + ((q) << 2)) 226 #define MVNETA_TXQ_DEC_SENT_SHIFT 16 227 #define MVNETA_TXQ_STATUS_REG(q) (0x3c40 + ((q) << 2)) 228 #define MVNETA_TXQ_SENT_DESC_SHIFT 16 229 #define MVNETA_TXQ_SENT_DESC_MASK 0x3fff0000 230 #define MVNETA_PORT_TX_RESET 0x3cf0 231 #define MVNETA_PORT_TX_DMA_RESET BIT(0) 232 #define MVNETA_TX_MTU 0x3e0c 233 #define MVNETA_TX_TOKEN_SIZE 0x3e14 234 #define MVNETA_TX_TOKEN_SIZE_MAX 0xffffffff 235 #define MVNETA_TXQ_TOKEN_SIZE_REG(q) (0x3e40 + ((q) << 2)) 236 #define MVNETA_TXQ_TOKEN_SIZE_MAX 0x7fffffff 237 238 #define MVNETA_CAUSE_TXQ_SENT_DESC_ALL_MASK 0xff 239 240 /* Descriptor ring Macros */ 241 #define MVNETA_QUEUE_NEXT_DESC(q, index) \ 242 (((index) < (q)->last_desc) ? ((index) + 1) : 0) 243 244 /* Various constants */ 245 246 /* Coalescing */ 247 #define MVNETA_TXDONE_COAL_PKTS 0 /* interrupt per packet */ 248 #define MVNETA_RX_COAL_PKTS 32 249 #define MVNETA_RX_COAL_USEC 100 250 251 /* The two bytes Marvell header. Either contains a special value used 252 * by Marvell switches when a specific hardware mode is enabled (not 253 * supported by this driver) or is filled automatically by zeroes on 254 * the RX side. Those two bytes being at the front of the Ethernet 255 * header, they allow to have the IP header aligned on a 4 bytes 256 * boundary automatically: the hardware skips those two bytes on its 257 * own. 258 */ 259 #define MVNETA_MH_SIZE 2 260 261 #define MVNETA_VLAN_TAG_LEN 4 262 263 #define MVNETA_TX_CSUM_DEF_SIZE 1600 264 #define MVNETA_TX_CSUM_MAX_SIZE 9800 265 #define MVNETA_ACC_MODE_EXT1 1 266 #define MVNETA_ACC_MODE_EXT2 2 267 268 #define MVNETA_MAX_DECODE_WIN 6 269 270 /* Timeout constants */ 271 #define MVNETA_TX_DISABLE_TIMEOUT_MSEC 1000 272 #define MVNETA_RX_DISABLE_TIMEOUT_MSEC 1000 273 #define MVNETA_TX_FIFO_EMPTY_TIMEOUT 10000 274 275 #define MVNETA_TX_MTU_MAX 0x3ffff 276 277 /* The RSS lookup table actually has 256 entries but we do not use 278 * them yet 279 */ 280 #define MVNETA_RSS_LU_TABLE_SIZE 1 281 282 /* TSO header size */ 283 #define TSO_HEADER_SIZE 128 284 285 /* Max number of Rx descriptors */ 286 #define MVNETA_MAX_RXD 128 287 288 /* Max number of Tx descriptors */ 289 #define MVNETA_MAX_TXD 532 290 291 /* Max number of allowed TCP segments for software TSO */ 292 #define MVNETA_MAX_TSO_SEGS 100 293 294 #define MVNETA_MAX_SKB_DESCS (MVNETA_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS) 295 296 /* descriptor aligned size */ 297 #define MVNETA_DESC_ALIGNED_SIZE 32 298 299 #define MVNETA_RX_PKT_SIZE(mtu) \ 300 ALIGN((mtu) + MVNETA_MH_SIZE + MVNETA_VLAN_TAG_LEN + \ 301 ETH_HLEN + ETH_FCS_LEN, \ 302 cache_line_size()) 303 304 #define IS_TSO_HEADER(txq, addr) \ 305 ((addr >= txq->tso_hdrs_phys) && \ 306 (addr < txq->tso_hdrs_phys + txq->size * TSO_HEADER_SIZE)) 307 308 #define MVNETA_RX_GET_BM_POOL_ID(rxd) \ 309 (((rxd)->status & MVNETA_RXD_BM_POOL_MASK) >> MVNETA_RXD_BM_POOL_SHIFT) 310 311 struct mvneta_statistic { 312 unsigned short offset; 313 unsigned short type; 314 const char name[ETH_GSTRING_LEN]; 315 }; 316 317 #define T_REG_32 32 318 #define T_REG_64 64 319 320 static const struct mvneta_statistic mvneta_statistics[] = { 321 { 0x3000, T_REG_64, "good_octets_received", }, 322 { 0x3010, T_REG_32, "good_frames_received", }, 323 { 0x3008, T_REG_32, "bad_octets_received", }, 324 { 0x3014, T_REG_32, "bad_frames_received", }, 325 { 0x3018, T_REG_32, "broadcast_frames_received", }, 326 { 0x301c, T_REG_32, "multicast_frames_received", }, 327 { 0x3050, T_REG_32, "unrec_mac_control_received", }, 328 { 0x3058, T_REG_32, "good_fc_received", }, 329 { 0x305c, T_REG_32, "bad_fc_received", }, 330 { 0x3060, T_REG_32, "undersize_received", }, 331 { 0x3064, T_REG_32, "fragments_received", }, 332 { 0x3068, T_REG_32, "oversize_received", }, 333 { 0x306c, T_REG_32, "jabber_received", }, 334 { 0x3070, T_REG_32, "mac_receive_error", }, 335 { 0x3074, T_REG_32, "bad_crc_event", }, 336 { 0x3078, T_REG_32, "collision", }, 337 { 0x307c, T_REG_32, "late_collision", }, 338 { 0x2484, T_REG_32, "rx_discard", }, 339 { 0x2488, T_REG_32, "rx_overrun", }, 340 { 0x3020, T_REG_32, "frames_64_octets", }, 341 { 0x3024, T_REG_32, "frames_65_to_127_octets", }, 342 { 0x3028, T_REG_32, "frames_128_to_255_octets", }, 343 { 0x302c, T_REG_32, "frames_256_to_511_octets", }, 344 { 0x3030, T_REG_32, "frames_512_to_1023_octets", }, 345 { 0x3034, T_REG_32, "frames_1024_to_max_octets", }, 346 { 0x3038, T_REG_64, "good_octets_sent", }, 347 { 0x3040, T_REG_32, "good_frames_sent", }, 348 { 0x3044, T_REG_32, "excessive_collision", }, 349 { 0x3048, T_REG_32, "multicast_frames_sent", }, 350 { 0x304c, T_REG_32, "broadcast_frames_sent", }, 351 { 0x3054, T_REG_32, "fc_sent", }, 352 { 0x300c, T_REG_32, "internal_mac_transmit_err", }, 353 }; 354 355 struct mvneta_pcpu_stats { 356 struct u64_stats_sync syncp; 357 u64 rx_packets; 358 u64 rx_bytes; 359 u64 tx_packets; 360 u64 tx_bytes; 361 }; 362 363 struct mvneta_pcpu_port { 364 /* Pointer to the shared port */ 365 struct mvneta_port *pp; 366 367 /* Pointer to the CPU-local NAPI struct */ 368 struct napi_struct napi; 369 370 /* Cause of the previous interrupt */ 371 u32 cause_rx_tx; 372 }; 373 374 struct mvneta_port { 375 u8 id; 376 struct mvneta_pcpu_port __percpu *ports; 377 struct mvneta_pcpu_stats __percpu *stats; 378 379 int pkt_size; 380 unsigned int frag_size; 381 void __iomem *base; 382 struct mvneta_rx_queue *rxqs; 383 struct mvneta_tx_queue *txqs; 384 struct net_device *dev; 385 struct notifier_block cpu_notifier; 386 int rxq_def; 387 /* Protect the access to the percpu interrupt registers, 388 * ensuring that the configuration remains coherent. 389 */ 390 spinlock_t lock; 391 bool is_stopped; 392 393 /* Core clock */ 394 struct clk *clk; 395 /* AXI clock */ 396 struct clk *clk_bus; 397 u8 mcast_count[256]; 398 u16 tx_ring_size; 399 u16 rx_ring_size; 400 401 struct mii_bus *mii_bus; 402 struct phy_device *phy_dev; 403 phy_interface_t phy_interface; 404 struct device_node *phy_node; 405 unsigned int link; 406 unsigned int duplex; 407 unsigned int speed; 408 unsigned int tx_csum_limit; 409 unsigned int use_inband_status:1; 410 411 struct mvneta_bm *bm_priv; 412 struct mvneta_bm_pool *pool_long; 413 struct mvneta_bm_pool *pool_short; 414 int bm_win_id; 415 416 u64 ethtool_stats[ARRAY_SIZE(mvneta_statistics)]; 417 418 u32 indir[MVNETA_RSS_LU_TABLE_SIZE]; 419 }; 420 421 /* The mvneta_tx_desc and mvneta_rx_desc structures describe the 422 * layout of the transmit and reception DMA descriptors, and their 423 * layout is therefore defined by the hardware design 424 */ 425 426 #define MVNETA_TX_L3_OFF_SHIFT 0 427 #define MVNETA_TX_IP_HLEN_SHIFT 8 428 #define MVNETA_TX_L4_UDP BIT(16) 429 #define MVNETA_TX_L3_IP6 BIT(17) 430 #define MVNETA_TXD_IP_CSUM BIT(18) 431 #define MVNETA_TXD_Z_PAD BIT(19) 432 #define MVNETA_TXD_L_DESC BIT(20) 433 #define MVNETA_TXD_F_DESC BIT(21) 434 #define MVNETA_TXD_FLZ_DESC (MVNETA_TXD_Z_PAD | \ 435 MVNETA_TXD_L_DESC | \ 436 MVNETA_TXD_F_DESC) 437 #define MVNETA_TX_L4_CSUM_FULL BIT(30) 438 #define MVNETA_TX_L4_CSUM_NOT BIT(31) 439 440 #define MVNETA_RXD_ERR_CRC 0x0 441 #define MVNETA_RXD_BM_POOL_SHIFT 13 442 #define MVNETA_RXD_BM_POOL_MASK (BIT(13) | BIT(14)) 443 #define MVNETA_RXD_ERR_SUMMARY BIT(16) 444 #define MVNETA_RXD_ERR_OVERRUN BIT(17) 445 #define MVNETA_RXD_ERR_LEN BIT(18) 446 #define MVNETA_RXD_ERR_RESOURCE (BIT(17) | BIT(18)) 447 #define MVNETA_RXD_ERR_CODE_MASK (BIT(17) | BIT(18)) 448 #define MVNETA_RXD_L3_IP4 BIT(25) 449 #define MVNETA_RXD_FIRST_LAST_DESC (BIT(26) | BIT(27)) 450 #define MVNETA_RXD_L4_CSUM_OK BIT(30) 451 452 #if defined(__LITTLE_ENDIAN) 453 struct mvneta_tx_desc { 454 u32 command; /* Options used by HW for packet transmitting.*/ 455 u16 reserverd1; /* csum_l4 (for future use) */ 456 u16 data_size; /* Data size of transmitted packet in bytes */ 457 u32 buf_phys_addr; /* Physical addr of transmitted buffer */ 458 u32 reserved2; /* hw_cmd - (for future use, PMT) */ 459 u32 reserved3[4]; /* Reserved - (for future use) */ 460 }; 461 462 struct mvneta_rx_desc { 463 u32 status; /* Info about received packet */ 464 u16 reserved1; /* pnc_info - (for future use, PnC) */ 465 u16 data_size; /* Size of received packet in bytes */ 466 467 u32 buf_phys_addr; /* Physical address of the buffer */ 468 u32 reserved2; /* pnc_flow_id (for future use, PnC) */ 469 470 u32 buf_cookie; /* cookie for access to RX buffer in rx path */ 471 u16 reserved3; /* prefetch_cmd, for future use */ 472 u16 reserved4; /* csum_l4 - (for future use, PnC) */ 473 474 u32 reserved5; /* pnc_extra PnC (for future use, PnC) */ 475 u32 reserved6; /* hw_cmd (for future use, PnC and HWF) */ 476 }; 477 #else 478 struct mvneta_tx_desc { 479 u16 data_size; /* Data size of transmitted packet in bytes */ 480 u16 reserverd1; /* csum_l4 (for future use) */ 481 u32 command; /* Options used by HW for packet transmitting.*/ 482 u32 reserved2; /* hw_cmd - (for future use, PMT) */ 483 u32 buf_phys_addr; /* Physical addr of transmitted buffer */ 484 u32 reserved3[4]; /* Reserved - (for future use) */ 485 }; 486 487 struct mvneta_rx_desc { 488 u16 data_size; /* Size of received packet in bytes */ 489 u16 reserved1; /* pnc_info - (for future use, PnC) */ 490 u32 status; /* Info about received packet */ 491 492 u32 reserved2; /* pnc_flow_id (for future use, PnC) */ 493 u32 buf_phys_addr; /* Physical address of the buffer */ 494 495 u16 reserved4; /* csum_l4 - (for future use, PnC) */ 496 u16 reserved3; /* prefetch_cmd, for future use */ 497 u32 buf_cookie; /* cookie for access to RX buffer in rx path */ 498 499 u32 reserved5; /* pnc_extra PnC (for future use, PnC) */ 500 u32 reserved6; /* hw_cmd (for future use, PnC and HWF) */ 501 }; 502 #endif 503 504 struct mvneta_tx_queue { 505 /* Number of this TX queue, in the range 0-7 */ 506 u8 id; 507 508 /* Number of TX DMA descriptors in the descriptor ring */ 509 int size; 510 511 /* Number of currently used TX DMA descriptor in the 512 * descriptor ring 513 */ 514 int count; 515 int tx_stop_threshold; 516 int tx_wake_threshold; 517 518 /* Array of transmitted skb */ 519 struct sk_buff **tx_skb; 520 521 /* Index of last TX DMA descriptor that was inserted */ 522 int txq_put_index; 523 524 /* Index of the TX DMA descriptor to be cleaned up */ 525 int txq_get_index; 526 527 u32 done_pkts_coal; 528 529 /* Virtual address of the TX DMA descriptors array */ 530 struct mvneta_tx_desc *descs; 531 532 /* DMA address of the TX DMA descriptors array */ 533 dma_addr_t descs_phys; 534 535 /* Index of the last TX DMA descriptor */ 536 int last_desc; 537 538 /* Index of the next TX DMA descriptor to process */ 539 int next_desc_to_proc; 540 541 /* DMA buffers for TSO headers */ 542 char *tso_hdrs; 543 544 /* DMA address of TSO headers */ 545 dma_addr_t tso_hdrs_phys; 546 547 /* Affinity mask for CPUs*/ 548 cpumask_t affinity_mask; 549 }; 550 551 struct mvneta_rx_queue { 552 /* rx queue number, in the range 0-7 */ 553 u8 id; 554 555 /* num of rx descriptors in the rx descriptor ring */ 556 int size; 557 558 /* counter of times when mvneta_refill() failed */ 559 int missed; 560 561 u32 pkts_coal; 562 u32 time_coal; 563 564 /* Virtual address of the RX DMA descriptors array */ 565 struct mvneta_rx_desc *descs; 566 567 /* DMA address of the RX DMA descriptors array */ 568 dma_addr_t descs_phys; 569 570 /* Index of the last RX DMA descriptor */ 571 int last_desc; 572 573 /* Index of the next RX DMA descriptor to process */ 574 int next_desc_to_proc; 575 }; 576 577 /* The hardware supports eight (8) rx queues, but we are only allowing 578 * the first one to be used. Therefore, let's just allocate one queue. 579 */ 580 static int rxq_number = 8; 581 static int txq_number = 8; 582 583 static int rxq_def; 584 585 static int rx_copybreak __read_mostly = 256; 586 587 /* HW BM need that each port be identify by a unique ID */ 588 static int global_port_id; 589 590 #define MVNETA_DRIVER_NAME "mvneta" 591 #define MVNETA_DRIVER_VERSION "1.0" 592 593 /* Utility/helper methods */ 594 595 /* Write helper method */ 596 static void mvreg_write(struct mvneta_port *pp, u32 offset, u32 data) 597 { 598 writel(data, pp->base + offset); 599 } 600 601 /* Read helper method */ 602 static u32 mvreg_read(struct mvneta_port *pp, u32 offset) 603 { 604 return readl(pp->base + offset); 605 } 606 607 /* Increment txq get counter */ 608 static void mvneta_txq_inc_get(struct mvneta_tx_queue *txq) 609 { 610 txq->txq_get_index++; 611 if (txq->txq_get_index == txq->size) 612 txq->txq_get_index = 0; 613 } 614 615 /* Increment txq put counter */ 616 static void mvneta_txq_inc_put(struct mvneta_tx_queue *txq) 617 { 618 txq->txq_put_index++; 619 if (txq->txq_put_index == txq->size) 620 txq->txq_put_index = 0; 621 } 622 623 624 /* Clear all MIB counters */ 625 static void mvneta_mib_counters_clear(struct mvneta_port *pp) 626 { 627 int i; 628 u32 dummy; 629 630 /* Perform dummy reads from MIB counters */ 631 for (i = 0; i < MVNETA_MIB_LATE_COLLISION; i += 4) 632 dummy = mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i)); 633 dummy = mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT); 634 dummy = mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT); 635 } 636 637 /* Get System Network Statistics */ 638 struct rtnl_link_stats64 *mvneta_get_stats64(struct net_device *dev, 639 struct rtnl_link_stats64 *stats) 640 { 641 struct mvneta_port *pp = netdev_priv(dev); 642 unsigned int start; 643 int cpu; 644 645 for_each_possible_cpu(cpu) { 646 struct mvneta_pcpu_stats *cpu_stats; 647 u64 rx_packets; 648 u64 rx_bytes; 649 u64 tx_packets; 650 u64 tx_bytes; 651 652 cpu_stats = per_cpu_ptr(pp->stats, cpu); 653 do { 654 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); 655 rx_packets = cpu_stats->rx_packets; 656 rx_bytes = cpu_stats->rx_bytes; 657 tx_packets = cpu_stats->tx_packets; 658 tx_bytes = cpu_stats->tx_bytes; 659 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); 660 661 stats->rx_packets += rx_packets; 662 stats->rx_bytes += rx_bytes; 663 stats->tx_packets += tx_packets; 664 stats->tx_bytes += tx_bytes; 665 } 666 667 stats->rx_errors = dev->stats.rx_errors; 668 stats->rx_dropped = dev->stats.rx_dropped; 669 670 stats->tx_dropped = dev->stats.tx_dropped; 671 672 return stats; 673 } 674 675 /* Rx descriptors helper methods */ 676 677 /* Checks whether the RX descriptor having this status is both the first 678 * and the last descriptor for the RX packet. Each RX packet is currently 679 * received through a single RX descriptor, so not having each RX 680 * descriptor with its first and last bits set is an error 681 */ 682 static int mvneta_rxq_desc_is_first_last(u32 status) 683 { 684 return (status & MVNETA_RXD_FIRST_LAST_DESC) == 685 MVNETA_RXD_FIRST_LAST_DESC; 686 } 687 688 /* Add number of descriptors ready to receive new packets */ 689 static void mvneta_rxq_non_occup_desc_add(struct mvneta_port *pp, 690 struct mvneta_rx_queue *rxq, 691 int ndescs) 692 { 693 /* Only MVNETA_RXQ_ADD_NON_OCCUPIED_MAX (255) descriptors can 694 * be added at once 695 */ 696 while (ndescs > MVNETA_RXQ_ADD_NON_OCCUPIED_MAX) { 697 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), 698 (MVNETA_RXQ_ADD_NON_OCCUPIED_MAX << 699 MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT)); 700 ndescs -= MVNETA_RXQ_ADD_NON_OCCUPIED_MAX; 701 } 702 703 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), 704 (ndescs << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT)); 705 } 706 707 /* Get number of RX descriptors occupied by received packets */ 708 static int mvneta_rxq_busy_desc_num_get(struct mvneta_port *pp, 709 struct mvneta_rx_queue *rxq) 710 { 711 u32 val; 712 713 val = mvreg_read(pp, MVNETA_RXQ_STATUS_REG(rxq->id)); 714 return val & MVNETA_RXQ_OCCUPIED_ALL_MASK; 715 } 716 717 /* Update num of rx desc called upon return from rx path or 718 * from mvneta_rxq_drop_pkts(). 719 */ 720 static void mvneta_rxq_desc_num_update(struct mvneta_port *pp, 721 struct mvneta_rx_queue *rxq, 722 int rx_done, int rx_filled) 723 { 724 u32 val; 725 726 if ((rx_done <= 0xff) && (rx_filled <= 0xff)) { 727 val = rx_done | 728 (rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT); 729 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val); 730 return; 731 } 732 733 /* Only 255 descriptors can be added at once */ 734 while ((rx_done > 0) || (rx_filled > 0)) { 735 if (rx_done <= 0xff) { 736 val = rx_done; 737 rx_done = 0; 738 } else { 739 val = 0xff; 740 rx_done -= 0xff; 741 } 742 if (rx_filled <= 0xff) { 743 val |= rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT; 744 rx_filled = 0; 745 } else { 746 val |= 0xff << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT; 747 rx_filled -= 0xff; 748 } 749 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val); 750 } 751 } 752 753 /* Get pointer to next RX descriptor to be processed by SW */ 754 static struct mvneta_rx_desc * 755 mvneta_rxq_next_desc_get(struct mvneta_rx_queue *rxq) 756 { 757 int rx_desc = rxq->next_desc_to_proc; 758 759 rxq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(rxq, rx_desc); 760 prefetch(rxq->descs + rxq->next_desc_to_proc); 761 return rxq->descs + rx_desc; 762 } 763 764 /* Change maximum receive size of the port. */ 765 static void mvneta_max_rx_size_set(struct mvneta_port *pp, int max_rx_size) 766 { 767 u32 val; 768 769 val = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 770 val &= ~MVNETA_GMAC_MAX_RX_SIZE_MASK; 771 val |= ((max_rx_size - MVNETA_MH_SIZE) / 2) << 772 MVNETA_GMAC_MAX_RX_SIZE_SHIFT; 773 mvreg_write(pp, MVNETA_GMAC_CTRL_0, val); 774 } 775 776 777 /* Set rx queue offset */ 778 static void mvneta_rxq_offset_set(struct mvneta_port *pp, 779 struct mvneta_rx_queue *rxq, 780 int offset) 781 { 782 u32 val; 783 784 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 785 val &= ~MVNETA_RXQ_PKT_OFFSET_ALL_MASK; 786 787 /* Offset is in */ 788 val |= MVNETA_RXQ_PKT_OFFSET_MASK(offset >> 3); 789 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 790 } 791 792 793 /* Tx descriptors helper methods */ 794 795 /* Update HW with number of TX descriptors to be sent */ 796 static void mvneta_txq_pend_desc_add(struct mvneta_port *pp, 797 struct mvneta_tx_queue *txq, 798 int pend_desc) 799 { 800 u32 val; 801 802 /* Only 255 descriptors can be added at once ; Assume caller 803 * process TX desriptors in quanta less than 256 804 */ 805 val = pend_desc; 806 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val); 807 } 808 809 /* Get pointer to next TX descriptor to be processed (send) by HW */ 810 static struct mvneta_tx_desc * 811 mvneta_txq_next_desc_get(struct mvneta_tx_queue *txq) 812 { 813 int tx_desc = txq->next_desc_to_proc; 814 815 txq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(txq, tx_desc); 816 return txq->descs + tx_desc; 817 } 818 819 /* Release the last allocated TX descriptor. Useful to handle DMA 820 * mapping failures in the TX path. 821 */ 822 static void mvneta_txq_desc_put(struct mvneta_tx_queue *txq) 823 { 824 if (txq->next_desc_to_proc == 0) 825 txq->next_desc_to_proc = txq->last_desc - 1; 826 else 827 txq->next_desc_to_proc--; 828 } 829 830 /* Set rxq buf size */ 831 static void mvneta_rxq_buf_size_set(struct mvneta_port *pp, 832 struct mvneta_rx_queue *rxq, 833 int buf_size) 834 { 835 u32 val; 836 837 val = mvreg_read(pp, MVNETA_RXQ_SIZE_REG(rxq->id)); 838 839 val &= ~MVNETA_RXQ_BUF_SIZE_MASK; 840 val |= ((buf_size >> 3) << MVNETA_RXQ_BUF_SIZE_SHIFT); 841 842 mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), val); 843 } 844 845 /* Disable buffer management (BM) */ 846 static void mvneta_rxq_bm_disable(struct mvneta_port *pp, 847 struct mvneta_rx_queue *rxq) 848 { 849 u32 val; 850 851 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 852 val &= ~MVNETA_RXQ_HW_BUF_ALLOC; 853 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 854 } 855 856 /* Enable buffer management (BM) */ 857 static void mvneta_rxq_bm_enable(struct mvneta_port *pp, 858 struct mvneta_rx_queue *rxq) 859 { 860 u32 val; 861 862 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 863 val |= MVNETA_RXQ_HW_BUF_ALLOC; 864 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 865 } 866 867 /* Notify HW about port's assignment of pool for bigger packets */ 868 static void mvneta_rxq_long_pool_set(struct mvneta_port *pp, 869 struct mvneta_rx_queue *rxq) 870 { 871 u32 val; 872 873 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 874 val &= ~MVNETA_RXQ_LONG_POOL_ID_MASK; 875 val |= (pp->pool_long->id << MVNETA_RXQ_LONG_POOL_ID_SHIFT); 876 877 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 878 } 879 880 /* Notify HW about port's assignment of pool for smaller packets */ 881 static void mvneta_rxq_short_pool_set(struct mvneta_port *pp, 882 struct mvneta_rx_queue *rxq) 883 { 884 u32 val; 885 886 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 887 val &= ~MVNETA_RXQ_SHORT_POOL_ID_MASK; 888 val |= (pp->pool_short->id << MVNETA_RXQ_SHORT_POOL_ID_SHIFT); 889 890 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 891 } 892 893 /* Set port's receive buffer size for assigned BM pool */ 894 static inline void mvneta_bm_pool_bufsize_set(struct mvneta_port *pp, 895 int buf_size, 896 u8 pool_id) 897 { 898 u32 val; 899 900 if (!IS_ALIGNED(buf_size, 8)) { 901 dev_warn(pp->dev->dev.parent, 902 "illegal buf_size value %d, round to %d\n", 903 buf_size, ALIGN(buf_size, 8)); 904 buf_size = ALIGN(buf_size, 8); 905 } 906 907 val = mvreg_read(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id)); 908 val |= buf_size & MVNETA_PORT_POOL_BUFFER_SZ_MASK; 909 mvreg_write(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id), val); 910 } 911 912 /* Configure MBUS window in order to enable access BM internal SRAM */ 913 static int mvneta_mbus_io_win_set(struct mvneta_port *pp, u32 base, u32 wsize, 914 u8 target, u8 attr) 915 { 916 u32 win_enable, win_protect; 917 int i; 918 919 win_enable = mvreg_read(pp, MVNETA_BASE_ADDR_ENABLE); 920 921 if (pp->bm_win_id < 0) { 922 /* Find first not occupied window */ 923 for (i = 0; i < MVNETA_MAX_DECODE_WIN; i++) { 924 if (win_enable & (1 << i)) { 925 pp->bm_win_id = i; 926 break; 927 } 928 } 929 if (i == MVNETA_MAX_DECODE_WIN) 930 return -ENOMEM; 931 } else { 932 i = pp->bm_win_id; 933 } 934 935 mvreg_write(pp, MVNETA_WIN_BASE(i), 0); 936 mvreg_write(pp, MVNETA_WIN_SIZE(i), 0); 937 938 if (i < 4) 939 mvreg_write(pp, MVNETA_WIN_REMAP(i), 0); 940 941 mvreg_write(pp, MVNETA_WIN_BASE(i), (base & 0xffff0000) | 942 (attr << 8) | target); 943 944 mvreg_write(pp, MVNETA_WIN_SIZE(i), (wsize - 1) & 0xffff0000); 945 946 win_protect = mvreg_read(pp, MVNETA_ACCESS_PROTECT_ENABLE); 947 win_protect |= 3 << (2 * i); 948 mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect); 949 950 win_enable &= ~(1 << i); 951 mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable); 952 953 return 0; 954 } 955 956 /* Assign and initialize pools for port. In case of fail 957 * buffer manager will remain disabled for current port. 958 */ 959 static int mvneta_bm_port_init(struct platform_device *pdev, 960 struct mvneta_port *pp) 961 { 962 struct device_node *dn = pdev->dev.of_node; 963 u32 long_pool_id, short_pool_id, wsize; 964 u8 target, attr; 965 int err; 966 967 /* Get BM window information */ 968 err = mvebu_mbus_get_io_win_info(pp->bm_priv->bppi_phys_addr, &wsize, 969 &target, &attr); 970 if (err < 0) 971 return err; 972 973 pp->bm_win_id = -1; 974 975 /* Open NETA -> BM window */ 976 err = mvneta_mbus_io_win_set(pp, pp->bm_priv->bppi_phys_addr, wsize, 977 target, attr); 978 if (err < 0) { 979 netdev_info(pp->dev, "fail to configure mbus window to BM\n"); 980 return err; 981 } 982 983 if (of_property_read_u32(dn, "bm,pool-long", &long_pool_id)) { 984 netdev_info(pp->dev, "missing long pool id\n"); 985 return -EINVAL; 986 } 987 988 /* Create port's long pool depending on mtu */ 989 pp->pool_long = mvneta_bm_pool_use(pp->bm_priv, long_pool_id, 990 MVNETA_BM_LONG, pp->id, 991 MVNETA_RX_PKT_SIZE(pp->dev->mtu)); 992 if (!pp->pool_long) { 993 netdev_info(pp->dev, "fail to obtain long pool for port\n"); 994 return -ENOMEM; 995 } 996 997 pp->pool_long->port_map |= 1 << pp->id; 998 999 mvneta_bm_pool_bufsize_set(pp, pp->pool_long->buf_size, 1000 pp->pool_long->id); 1001 1002 /* If short pool id is not defined, assume using single pool */ 1003 if (of_property_read_u32(dn, "bm,pool-short", &short_pool_id)) 1004 short_pool_id = long_pool_id; 1005 1006 /* Create port's short pool */ 1007 pp->pool_short = mvneta_bm_pool_use(pp->bm_priv, short_pool_id, 1008 MVNETA_BM_SHORT, pp->id, 1009 MVNETA_BM_SHORT_PKT_SIZE); 1010 if (!pp->pool_short) { 1011 netdev_info(pp->dev, "fail to obtain short pool for port\n"); 1012 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 1013 return -ENOMEM; 1014 } 1015 1016 if (short_pool_id != long_pool_id) { 1017 pp->pool_short->port_map |= 1 << pp->id; 1018 mvneta_bm_pool_bufsize_set(pp, pp->pool_short->buf_size, 1019 pp->pool_short->id); 1020 } 1021 1022 return 0; 1023 } 1024 1025 /* Update settings of a pool for bigger packets */ 1026 static void mvneta_bm_update_mtu(struct mvneta_port *pp, int mtu) 1027 { 1028 struct mvneta_bm_pool *bm_pool = pp->pool_long; 1029 struct hwbm_pool *hwbm_pool = &bm_pool->hwbm_pool; 1030 int num; 1031 1032 /* Release all buffers from long pool */ 1033 mvneta_bm_bufs_free(pp->bm_priv, bm_pool, 1 << pp->id); 1034 if (hwbm_pool->buf_num) { 1035 WARN(1, "cannot free all buffers in pool %d\n", 1036 bm_pool->id); 1037 goto bm_mtu_err; 1038 } 1039 1040 bm_pool->pkt_size = MVNETA_RX_PKT_SIZE(mtu); 1041 bm_pool->buf_size = MVNETA_RX_BUF_SIZE(bm_pool->pkt_size); 1042 hwbm_pool->frag_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1043 SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(bm_pool->pkt_size)); 1044 1045 /* Fill entire long pool */ 1046 num = hwbm_pool_add(hwbm_pool, hwbm_pool->size, GFP_ATOMIC); 1047 if (num != hwbm_pool->size) { 1048 WARN(1, "pool %d: %d of %d allocated\n", 1049 bm_pool->id, num, hwbm_pool->size); 1050 goto bm_mtu_err; 1051 } 1052 mvneta_bm_pool_bufsize_set(pp, bm_pool->buf_size, bm_pool->id); 1053 1054 return; 1055 1056 bm_mtu_err: 1057 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 1058 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 1 << pp->id); 1059 1060 pp->bm_priv = NULL; 1061 mvreg_write(pp, MVNETA_ACC_MODE, MVNETA_ACC_MODE_EXT1); 1062 netdev_info(pp->dev, "fail to update MTU, fall back to software BM\n"); 1063 } 1064 1065 /* Start the Ethernet port RX and TX activity */ 1066 static void mvneta_port_up(struct mvneta_port *pp) 1067 { 1068 int queue; 1069 u32 q_map; 1070 1071 /* Enable all initialized TXs. */ 1072 q_map = 0; 1073 for (queue = 0; queue < txq_number; queue++) { 1074 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 1075 if (txq->descs != NULL) 1076 q_map |= (1 << queue); 1077 } 1078 mvreg_write(pp, MVNETA_TXQ_CMD, q_map); 1079 1080 /* Enable all initialized RXQs. */ 1081 for (queue = 0; queue < rxq_number; queue++) { 1082 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 1083 1084 if (rxq->descs != NULL) 1085 q_map |= (1 << queue); 1086 } 1087 mvreg_write(pp, MVNETA_RXQ_CMD, q_map); 1088 } 1089 1090 /* Stop the Ethernet port activity */ 1091 static void mvneta_port_down(struct mvneta_port *pp) 1092 { 1093 u32 val; 1094 int count; 1095 1096 /* Stop Rx port activity. Check port Rx activity. */ 1097 val = mvreg_read(pp, MVNETA_RXQ_CMD) & MVNETA_RXQ_ENABLE_MASK; 1098 1099 /* Issue stop command for active channels only */ 1100 if (val != 0) 1101 mvreg_write(pp, MVNETA_RXQ_CMD, 1102 val << MVNETA_RXQ_DISABLE_SHIFT); 1103 1104 /* Wait for all Rx activity to terminate. */ 1105 count = 0; 1106 do { 1107 if (count++ >= MVNETA_RX_DISABLE_TIMEOUT_MSEC) { 1108 netdev_warn(pp->dev, 1109 "TIMEOUT for RX stopped ! rx_queue_cmd: 0x%08x\n", 1110 val); 1111 break; 1112 } 1113 mdelay(1); 1114 1115 val = mvreg_read(pp, MVNETA_RXQ_CMD); 1116 } while (val & MVNETA_RXQ_ENABLE_MASK); 1117 1118 /* Stop Tx port activity. Check port Tx activity. Issue stop 1119 * command for active channels only 1120 */ 1121 val = (mvreg_read(pp, MVNETA_TXQ_CMD)) & MVNETA_TXQ_ENABLE_MASK; 1122 1123 if (val != 0) 1124 mvreg_write(pp, MVNETA_TXQ_CMD, 1125 (val << MVNETA_TXQ_DISABLE_SHIFT)); 1126 1127 /* Wait for all Tx activity to terminate. */ 1128 count = 0; 1129 do { 1130 if (count++ >= MVNETA_TX_DISABLE_TIMEOUT_MSEC) { 1131 netdev_warn(pp->dev, 1132 "TIMEOUT for TX stopped status=0x%08x\n", 1133 val); 1134 break; 1135 } 1136 mdelay(1); 1137 1138 /* Check TX Command reg that all Txqs are stopped */ 1139 val = mvreg_read(pp, MVNETA_TXQ_CMD); 1140 1141 } while (val & MVNETA_TXQ_ENABLE_MASK); 1142 1143 /* Double check to verify that TX FIFO is empty */ 1144 count = 0; 1145 do { 1146 if (count++ >= MVNETA_TX_FIFO_EMPTY_TIMEOUT) { 1147 netdev_warn(pp->dev, 1148 "TX FIFO empty timeout status=0x%08x\n", 1149 val); 1150 break; 1151 } 1152 mdelay(1); 1153 1154 val = mvreg_read(pp, MVNETA_PORT_STATUS); 1155 } while (!(val & MVNETA_TX_FIFO_EMPTY) && 1156 (val & MVNETA_TX_IN_PRGRS)); 1157 1158 udelay(200); 1159 } 1160 1161 /* Enable the port by setting the port enable bit of the MAC control register */ 1162 static void mvneta_port_enable(struct mvneta_port *pp) 1163 { 1164 u32 val; 1165 1166 /* Enable port */ 1167 val = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 1168 val |= MVNETA_GMAC0_PORT_ENABLE; 1169 mvreg_write(pp, MVNETA_GMAC_CTRL_0, val); 1170 } 1171 1172 /* Disable the port and wait for about 200 usec before retuning */ 1173 static void mvneta_port_disable(struct mvneta_port *pp) 1174 { 1175 u32 val; 1176 1177 /* Reset the Enable bit in the Serial Control Register */ 1178 val = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 1179 val &= ~MVNETA_GMAC0_PORT_ENABLE; 1180 mvreg_write(pp, MVNETA_GMAC_CTRL_0, val); 1181 1182 udelay(200); 1183 } 1184 1185 /* Multicast tables methods */ 1186 1187 /* Set all entries in Unicast MAC Table; queue==-1 means reject all */ 1188 static void mvneta_set_ucast_table(struct mvneta_port *pp, int queue) 1189 { 1190 int offset; 1191 u32 val; 1192 1193 if (queue == -1) { 1194 val = 0; 1195 } else { 1196 val = 0x1 | (queue << 1); 1197 val |= (val << 24) | (val << 16) | (val << 8); 1198 } 1199 1200 for (offset = 0; offset <= 0xc; offset += 4) 1201 mvreg_write(pp, MVNETA_DA_FILT_UCAST_BASE + offset, val); 1202 } 1203 1204 /* Set all entries in Special Multicast MAC Table; queue==-1 means reject all */ 1205 static void mvneta_set_special_mcast_table(struct mvneta_port *pp, int queue) 1206 { 1207 int offset; 1208 u32 val; 1209 1210 if (queue == -1) { 1211 val = 0; 1212 } else { 1213 val = 0x1 | (queue << 1); 1214 val |= (val << 24) | (val << 16) | (val << 8); 1215 } 1216 1217 for (offset = 0; offset <= 0xfc; offset += 4) 1218 mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + offset, val); 1219 1220 } 1221 1222 /* Set all entries in Other Multicast MAC Table. queue==-1 means reject all */ 1223 static void mvneta_set_other_mcast_table(struct mvneta_port *pp, int queue) 1224 { 1225 int offset; 1226 u32 val; 1227 1228 if (queue == -1) { 1229 memset(pp->mcast_count, 0, sizeof(pp->mcast_count)); 1230 val = 0; 1231 } else { 1232 memset(pp->mcast_count, 1, sizeof(pp->mcast_count)); 1233 val = 0x1 | (queue << 1); 1234 val |= (val << 24) | (val << 16) | (val << 8); 1235 } 1236 1237 for (offset = 0; offset <= 0xfc; offset += 4) 1238 mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + offset, val); 1239 } 1240 1241 static void mvneta_set_autoneg(struct mvneta_port *pp, int enable) 1242 { 1243 u32 val; 1244 1245 if (enable) { 1246 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 1247 val &= ~(MVNETA_GMAC_FORCE_LINK_PASS | 1248 MVNETA_GMAC_FORCE_LINK_DOWN | 1249 MVNETA_GMAC_AN_FLOW_CTRL_EN); 1250 val |= MVNETA_GMAC_INBAND_AN_ENABLE | 1251 MVNETA_GMAC_AN_SPEED_EN | 1252 MVNETA_GMAC_AN_DUPLEX_EN; 1253 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); 1254 1255 val = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER); 1256 val |= MVNETA_GMAC_1MS_CLOCK_ENABLE; 1257 mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, val); 1258 1259 val = mvreg_read(pp, MVNETA_GMAC_CTRL_2); 1260 val |= MVNETA_GMAC2_INBAND_AN_ENABLE; 1261 mvreg_write(pp, MVNETA_GMAC_CTRL_2, val); 1262 } else { 1263 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 1264 val &= ~(MVNETA_GMAC_INBAND_AN_ENABLE | 1265 MVNETA_GMAC_AN_SPEED_EN | 1266 MVNETA_GMAC_AN_DUPLEX_EN); 1267 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); 1268 1269 val = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER); 1270 val &= ~MVNETA_GMAC_1MS_CLOCK_ENABLE; 1271 mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, val); 1272 1273 val = mvreg_read(pp, MVNETA_GMAC_CTRL_2); 1274 val &= ~MVNETA_GMAC2_INBAND_AN_ENABLE; 1275 mvreg_write(pp, MVNETA_GMAC_CTRL_2, val); 1276 } 1277 } 1278 1279 static void mvneta_percpu_unmask_interrupt(void *arg) 1280 { 1281 struct mvneta_port *pp = arg; 1282 1283 /* All the queue are unmasked, but actually only the ones 1284 * mapped to this CPU will be unmasked 1285 */ 1286 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 1287 MVNETA_RX_INTR_MASK_ALL | 1288 MVNETA_TX_INTR_MASK_ALL | 1289 MVNETA_MISCINTR_INTR_MASK); 1290 } 1291 1292 static void mvneta_percpu_mask_interrupt(void *arg) 1293 { 1294 struct mvneta_port *pp = arg; 1295 1296 /* All the queue are masked, but actually only the ones 1297 * mapped to this CPU will be masked 1298 */ 1299 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0); 1300 mvreg_write(pp, MVNETA_INTR_OLD_MASK, 0); 1301 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 0); 1302 } 1303 1304 static void mvneta_percpu_clear_intr_cause(void *arg) 1305 { 1306 struct mvneta_port *pp = arg; 1307 1308 /* All the queue are cleared, but actually only the ones 1309 * mapped to this CPU will be cleared 1310 */ 1311 mvreg_write(pp, MVNETA_INTR_NEW_CAUSE, 0); 1312 mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0); 1313 mvreg_write(pp, MVNETA_INTR_OLD_CAUSE, 0); 1314 } 1315 1316 /* This method sets defaults to the NETA port: 1317 * Clears interrupt Cause and Mask registers. 1318 * Clears all MAC tables. 1319 * Sets defaults to all registers. 1320 * Resets RX and TX descriptor rings. 1321 * Resets PHY. 1322 * This method can be called after mvneta_port_down() to return the port 1323 * settings to defaults. 1324 */ 1325 static void mvneta_defaults_set(struct mvneta_port *pp) 1326 { 1327 int cpu; 1328 int queue; 1329 u32 val; 1330 int max_cpu = num_present_cpus(); 1331 1332 /* Clear all Cause registers */ 1333 on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true); 1334 1335 /* Mask all interrupts */ 1336 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 1337 mvreg_write(pp, MVNETA_INTR_ENABLE, 0); 1338 1339 /* Enable MBUS Retry bit16 */ 1340 mvreg_write(pp, MVNETA_MBUS_RETRY, 0x20); 1341 1342 /* Set CPU queue access map. CPUs are assigned to the RX and 1343 * TX queues modulo their number. If there is only one TX 1344 * queue then it is assigned to the CPU associated to the 1345 * default RX queue. 1346 */ 1347 for_each_present_cpu(cpu) { 1348 int rxq_map = 0, txq_map = 0; 1349 int rxq, txq; 1350 1351 for (rxq = 0; rxq < rxq_number; rxq++) 1352 if ((rxq % max_cpu) == cpu) 1353 rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq); 1354 1355 for (txq = 0; txq < txq_number; txq++) 1356 if ((txq % max_cpu) == cpu) 1357 txq_map |= MVNETA_CPU_TXQ_ACCESS(txq); 1358 1359 /* With only one TX queue we configure a special case 1360 * which will allow to get all the irq on a single 1361 * CPU 1362 */ 1363 if (txq_number == 1) 1364 txq_map = (cpu == pp->rxq_def) ? 1365 MVNETA_CPU_TXQ_ACCESS(1) : 0; 1366 1367 mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map); 1368 } 1369 1370 /* Reset RX and TX DMAs */ 1371 mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET); 1372 mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET); 1373 1374 /* Disable Legacy WRR, Disable EJP, Release from reset */ 1375 mvreg_write(pp, MVNETA_TXQ_CMD_1, 0); 1376 for (queue = 0; queue < txq_number; queue++) { 1377 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(queue), 0); 1378 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(queue), 0); 1379 } 1380 1381 mvreg_write(pp, MVNETA_PORT_TX_RESET, 0); 1382 mvreg_write(pp, MVNETA_PORT_RX_RESET, 0); 1383 1384 /* Set Port Acceleration Mode */ 1385 if (pp->bm_priv) 1386 /* HW buffer management + legacy parser */ 1387 val = MVNETA_ACC_MODE_EXT2; 1388 else 1389 /* SW buffer management + legacy parser */ 1390 val = MVNETA_ACC_MODE_EXT1; 1391 mvreg_write(pp, MVNETA_ACC_MODE, val); 1392 1393 if (pp->bm_priv) 1394 mvreg_write(pp, MVNETA_BM_ADDRESS, pp->bm_priv->bppi_phys_addr); 1395 1396 /* Update val of portCfg register accordingly with all RxQueue types */ 1397 val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def); 1398 mvreg_write(pp, MVNETA_PORT_CONFIG, val); 1399 1400 val = 0; 1401 mvreg_write(pp, MVNETA_PORT_CONFIG_EXTEND, val); 1402 mvreg_write(pp, MVNETA_RX_MIN_FRAME_SIZE, 64); 1403 1404 /* Build PORT_SDMA_CONFIG_REG */ 1405 val = 0; 1406 1407 /* Default burst size */ 1408 val |= MVNETA_TX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16); 1409 val |= MVNETA_RX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16); 1410 val |= MVNETA_RX_NO_DATA_SWAP | MVNETA_TX_NO_DATA_SWAP; 1411 1412 #if defined(__BIG_ENDIAN) 1413 val |= MVNETA_DESC_SWAP; 1414 #endif 1415 1416 /* Assign port SDMA configuration */ 1417 mvreg_write(pp, MVNETA_SDMA_CONFIG, val); 1418 1419 /* Disable PHY polling in hardware, since we're using the 1420 * kernel phylib to do this. 1421 */ 1422 val = mvreg_read(pp, MVNETA_UNIT_CONTROL); 1423 val &= ~MVNETA_PHY_POLLING_ENABLE; 1424 mvreg_write(pp, MVNETA_UNIT_CONTROL, val); 1425 1426 mvneta_set_autoneg(pp, pp->use_inband_status); 1427 mvneta_set_ucast_table(pp, -1); 1428 mvneta_set_special_mcast_table(pp, -1); 1429 mvneta_set_other_mcast_table(pp, -1); 1430 1431 /* Set port interrupt enable register - default enable all */ 1432 mvreg_write(pp, MVNETA_INTR_ENABLE, 1433 (MVNETA_RXQ_INTR_ENABLE_ALL_MASK 1434 | MVNETA_TXQ_INTR_ENABLE_ALL_MASK)); 1435 1436 mvneta_mib_counters_clear(pp); 1437 } 1438 1439 /* Set max sizes for tx queues */ 1440 static void mvneta_txq_max_tx_size_set(struct mvneta_port *pp, int max_tx_size) 1441 1442 { 1443 u32 val, size, mtu; 1444 int queue; 1445 1446 mtu = max_tx_size * 8; 1447 if (mtu > MVNETA_TX_MTU_MAX) 1448 mtu = MVNETA_TX_MTU_MAX; 1449 1450 /* Set MTU */ 1451 val = mvreg_read(pp, MVNETA_TX_MTU); 1452 val &= ~MVNETA_TX_MTU_MAX; 1453 val |= mtu; 1454 mvreg_write(pp, MVNETA_TX_MTU, val); 1455 1456 /* TX token size and all TXQs token size must be larger that MTU */ 1457 val = mvreg_read(pp, MVNETA_TX_TOKEN_SIZE); 1458 1459 size = val & MVNETA_TX_TOKEN_SIZE_MAX; 1460 if (size < mtu) { 1461 size = mtu; 1462 val &= ~MVNETA_TX_TOKEN_SIZE_MAX; 1463 val |= size; 1464 mvreg_write(pp, MVNETA_TX_TOKEN_SIZE, val); 1465 } 1466 for (queue = 0; queue < txq_number; queue++) { 1467 val = mvreg_read(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue)); 1468 1469 size = val & MVNETA_TXQ_TOKEN_SIZE_MAX; 1470 if (size < mtu) { 1471 size = mtu; 1472 val &= ~MVNETA_TXQ_TOKEN_SIZE_MAX; 1473 val |= size; 1474 mvreg_write(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue), val); 1475 } 1476 } 1477 } 1478 1479 /* Set unicast address */ 1480 static void mvneta_set_ucast_addr(struct mvneta_port *pp, u8 last_nibble, 1481 int queue) 1482 { 1483 unsigned int unicast_reg; 1484 unsigned int tbl_offset; 1485 unsigned int reg_offset; 1486 1487 /* Locate the Unicast table entry */ 1488 last_nibble = (0xf & last_nibble); 1489 1490 /* offset from unicast tbl base */ 1491 tbl_offset = (last_nibble / 4) * 4; 1492 1493 /* offset within the above reg */ 1494 reg_offset = last_nibble % 4; 1495 1496 unicast_reg = mvreg_read(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset)); 1497 1498 if (queue == -1) { 1499 /* Clear accepts frame bit at specified unicast DA tbl entry */ 1500 unicast_reg &= ~(0xff << (8 * reg_offset)); 1501 } else { 1502 unicast_reg &= ~(0xff << (8 * reg_offset)); 1503 unicast_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset)); 1504 } 1505 1506 mvreg_write(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset), unicast_reg); 1507 } 1508 1509 /* Set mac address */ 1510 static void mvneta_mac_addr_set(struct mvneta_port *pp, unsigned char *addr, 1511 int queue) 1512 { 1513 unsigned int mac_h; 1514 unsigned int mac_l; 1515 1516 if (queue != -1) { 1517 mac_l = (addr[4] << 8) | (addr[5]); 1518 mac_h = (addr[0] << 24) | (addr[1] << 16) | 1519 (addr[2] << 8) | (addr[3] << 0); 1520 1521 mvreg_write(pp, MVNETA_MAC_ADDR_LOW, mac_l); 1522 mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, mac_h); 1523 } 1524 1525 /* Accept frames of this address */ 1526 mvneta_set_ucast_addr(pp, addr[5], queue); 1527 } 1528 1529 /* Set the number of packets that will be received before RX interrupt 1530 * will be generated by HW. 1531 */ 1532 static void mvneta_rx_pkts_coal_set(struct mvneta_port *pp, 1533 struct mvneta_rx_queue *rxq, u32 value) 1534 { 1535 mvreg_write(pp, MVNETA_RXQ_THRESHOLD_REG(rxq->id), 1536 value | MVNETA_RXQ_NON_OCCUPIED(0)); 1537 rxq->pkts_coal = value; 1538 } 1539 1540 /* Set the time delay in usec before RX interrupt will be generated by 1541 * HW. 1542 */ 1543 static void mvneta_rx_time_coal_set(struct mvneta_port *pp, 1544 struct mvneta_rx_queue *rxq, u32 value) 1545 { 1546 u32 val; 1547 unsigned long clk_rate; 1548 1549 clk_rate = clk_get_rate(pp->clk); 1550 val = (clk_rate / 1000000) * value; 1551 1552 mvreg_write(pp, MVNETA_RXQ_TIME_COAL_REG(rxq->id), val); 1553 rxq->time_coal = value; 1554 } 1555 1556 /* Set threshold for TX_DONE pkts coalescing */ 1557 static void mvneta_tx_done_pkts_coal_set(struct mvneta_port *pp, 1558 struct mvneta_tx_queue *txq, u32 value) 1559 { 1560 u32 val; 1561 1562 val = mvreg_read(pp, MVNETA_TXQ_SIZE_REG(txq->id)); 1563 1564 val &= ~MVNETA_TXQ_SENT_THRESH_ALL_MASK; 1565 val |= MVNETA_TXQ_SENT_THRESH_MASK(value); 1566 1567 mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), val); 1568 1569 txq->done_pkts_coal = value; 1570 } 1571 1572 /* Handle rx descriptor fill by setting buf_cookie and buf_phys_addr */ 1573 static void mvneta_rx_desc_fill(struct mvneta_rx_desc *rx_desc, 1574 u32 phys_addr, u32 cookie) 1575 { 1576 rx_desc->buf_cookie = cookie; 1577 rx_desc->buf_phys_addr = phys_addr; 1578 } 1579 1580 /* Decrement sent descriptors counter */ 1581 static void mvneta_txq_sent_desc_dec(struct mvneta_port *pp, 1582 struct mvneta_tx_queue *txq, 1583 int sent_desc) 1584 { 1585 u32 val; 1586 1587 /* Only 255 TX descriptors can be updated at once */ 1588 while (sent_desc > 0xff) { 1589 val = 0xff << MVNETA_TXQ_DEC_SENT_SHIFT; 1590 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val); 1591 sent_desc = sent_desc - 0xff; 1592 } 1593 1594 val = sent_desc << MVNETA_TXQ_DEC_SENT_SHIFT; 1595 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val); 1596 } 1597 1598 /* Get number of TX descriptors already sent by HW */ 1599 static int mvneta_txq_sent_desc_num_get(struct mvneta_port *pp, 1600 struct mvneta_tx_queue *txq) 1601 { 1602 u32 val; 1603 int sent_desc; 1604 1605 val = mvreg_read(pp, MVNETA_TXQ_STATUS_REG(txq->id)); 1606 sent_desc = (val & MVNETA_TXQ_SENT_DESC_MASK) >> 1607 MVNETA_TXQ_SENT_DESC_SHIFT; 1608 1609 return sent_desc; 1610 } 1611 1612 /* Get number of sent descriptors and decrement counter. 1613 * The number of sent descriptors is returned. 1614 */ 1615 static int mvneta_txq_sent_desc_proc(struct mvneta_port *pp, 1616 struct mvneta_tx_queue *txq) 1617 { 1618 int sent_desc; 1619 1620 /* Get number of sent descriptors */ 1621 sent_desc = mvneta_txq_sent_desc_num_get(pp, txq); 1622 1623 /* Decrement sent descriptors counter */ 1624 if (sent_desc) 1625 mvneta_txq_sent_desc_dec(pp, txq, sent_desc); 1626 1627 return sent_desc; 1628 } 1629 1630 /* Set TXQ descriptors fields relevant for CSUM calculation */ 1631 static u32 mvneta_txq_desc_csum(int l3_offs, int l3_proto, 1632 int ip_hdr_len, int l4_proto) 1633 { 1634 u32 command; 1635 1636 /* Fields: L3_offset, IP_hdrlen, L3_type, G_IPv4_chk, 1637 * G_L4_chk, L4_type; required only for checksum 1638 * calculation 1639 */ 1640 command = l3_offs << MVNETA_TX_L3_OFF_SHIFT; 1641 command |= ip_hdr_len << MVNETA_TX_IP_HLEN_SHIFT; 1642 1643 if (l3_proto == htons(ETH_P_IP)) 1644 command |= MVNETA_TXD_IP_CSUM; 1645 else 1646 command |= MVNETA_TX_L3_IP6; 1647 1648 if (l4_proto == IPPROTO_TCP) 1649 command |= MVNETA_TX_L4_CSUM_FULL; 1650 else if (l4_proto == IPPROTO_UDP) 1651 command |= MVNETA_TX_L4_UDP | MVNETA_TX_L4_CSUM_FULL; 1652 else 1653 command |= MVNETA_TX_L4_CSUM_NOT; 1654 1655 return command; 1656 } 1657 1658 1659 /* Display more error info */ 1660 static void mvneta_rx_error(struct mvneta_port *pp, 1661 struct mvneta_rx_desc *rx_desc) 1662 { 1663 u32 status = rx_desc->status; 1664 1665 if (!mvneta_rxq_desc_is_first_last(status)) { 1666 netdev_err(pp->dev, 1667 "bad rx status %08x (buffer oversize), size=%d\n", 1668 status, rx_desc->data_size); 1669 return; 1670 } 1671 1672 switch (status & MVNETA_RXD_ERR_CODE_MASK) { 1673 case MVNETA_RXD_ERR_CRC: 1674 netdev_err(pp->dev, "bad rx status %08x (crc error), size=%d\n", 1675 status, rx_desc->data_size); 1676 break; 1677 case MVNETA_RXD_ERR_OVERRUN: 1678 netdev_err(pp->dev, "bad rx status %08x (overrun error), size=%d\n", 1679 status, rx_desc->data_size); 1680 break; 1681 case MVNETA_RXD_ERR_LEN: 1682 netdev_err(pp->dev, "bad rx status %08x (max frame length error), size=%d\n", 1683 status, rx_desc->data_size); 1684 break; 1685 case MVNETA_RXD_ERR_RESOURCE: 1686 netdev_err(pp->dev, "bad rx status %08x (resource error), size=%d\n", 1687 status, rx_desc->data_size); 1688 break; 1689 } 1690 } 1691 1692 /* Handle RX checksum offload based on the descriptor's status */ 1693 static void mvneta_rx_csum(struct mvneta_port *pp, u32 status, 1694 struct sk_buff *skb) 1695 { 1696 if ((status & MVNETA_RXD_L3_IP4) && 1697 (status & MVNETA_RXD_L4_CSUM_OK)) { 1698 skb->csum = 0; 1699 skb->ip_summed = CHECKSUM_UNNECESSARY; 1700 return; 1701 } 1702 1703 skb->ip_summed = CHECKSUM_NONE; 1704 } 1705 1706 /* Return tx queue pointer (find last set bit) according to <cause> returned 1707 * form tx_done reg. <cause> must not be null. The return value is always a 1708 * valid queue for matching the first one found in <cause>. 1709 */ 1710 static struct mvneta_tx_queue *mvneta_tx_done_policy(struct mvneta_port *pp, 1711 u32 cause) 1712 { 1713 int queue = fls(cause) - 1; 1714 1715 return &pp->txqs[queue]; 1716 } 1717 1718 /* Free tx queue skbuffs */ 1719 static void mvneta_txq_bufs_free(struct mvneta_port *pp, 1720 struct mvneta_tx_queue *txq, int num) 1721 { 1722 int i; 1723 1724 for (i = 0; i < num; i++) { 1725 struct mvneta_tx_desc *tx_desc = txq->descs + 1726 txq->txq_get_index; 1727 struct sk_buff *skb = txq->tx_skb[txq->txq_get_index]; 1728 1729 mvneta_txq_inc_get(txq); 1730 1731 if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr)) 1732 dma_unmap_single(pp->dev->dev.parent, 1733 tx_desc->buf_phys_addr, 1734 tx_desc->data_size, DMA_TO_DEVICE); 1735 if (!skb) 1736 continue; 1737 dev_kfree_skb_any(skb); 1738 } 1739 } 1740 1741 /* Handle end of transmission */ 1742 static void mvneta_txq_done(struct mvneta_port *pp, 1743 struct mvneta_tx_queue *txq) 1744 { 1745 struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id); 1746 int tx_done; 1747 1748 tx_done = mvneta_txq_sent_desc_proc(pp, txq); 1749 if (!tx_done) 1750 return; 1751 1752 mvneta_txq_bufs_free(pp, txq, tx_done); 1753 1754 txq->count -= tx_done; 1755 1756 if (netif_tx_queue_stopped(nq)) { 1757 if (txq->count <= txq->tx_wake_threshold) 1758 netif_tx_wake_queue(nq); 1759 } 1760 } 1761 1762 void *mvneta_frag_alloc(unsigned int frag_size) 1763 { 1764 if (likely(frag_size <= PAGE_SIZE)) 1765 return netdev_alloc_frag(frag_size); 1766 else 1767 return kmalloc(frag_size, GFP_ATOMIC); 1768 } 1769 EXPORT_SYMBOL_GPL(mvneta_frag_alloc); 1770 1771 void mvneta_frag_free(unsigned int frag_size, void *data) 1772 { 1773 if (likely(frag_size <= PAGE_SIZE)) 1774 skb_free_frag(data); 1775 else 1776 kfree(data); 1777 } 1778 EXPORT_SYMBOL_GPL(mvneta_frag_free); 1779 1780 /* Refill processing for SW buffer management */ 1781 static int mvneta_rx_refill(struct mvneta_port *pp, 1782 struct mvneta_rx_desc *rx_desc) 1783 1784 { 1785 dma_addr_t phys_addr; 1786 void *data; 1787 1788 data = mvneta_frag_alloc(pp->frag_size); 1789 if (!data) 1790 return -ENOMEM; 1791 1792 phys_addr = dma_map_single(pp->dev->dev.parent, data, 1793 MVNETA_RX_BUF_SIZE(pp->pkt_size), 1794 DMA_FROM_DEVICE); 1795 if (unlikely(dma_mapping_error(pp->dev->dev.parent, phys_addr))) { 1796 mvneta_frag_free(pp->frag_size, data); 1797 return -ENOMEM; 1798 } 1799 1800 mvneta_rx_desc_fill(rx_desc, phys_addr, (u32)data); 1801 return 0; 1802 } 1803 1804 /* Handle tx checksum */ 1805 static u32 mvneta_skb_tx_csum(struct mvneta_port *pp, struct sk_buff *skb) 1806 { 1807 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1808 int ip_hdr_len = 0; 1809 __be16 l3_proto = vlan_get_protocol(skb); 1810 u8 l4_proto; 1811 1812 if (l3_proto == htons(ETH_P_IP)) { 1813 struct iphdr *ip4h = ip_hdr(skb); 1814 1815 /* Calculate IPv4 checksum and L4 checksum */ 1816 ip_hdr_len = ip4h->ihl; 1817 l4_proto = ip4h->protocol; 1818 } else if (l3_proto == htons(ETH_P_IPV6)) { 1819 struct ipv6hdr *ip6h = ipv6_hdr(skb); 1820 1821 /* Read l4_protocol from one of IPv6 extra headers */ 1822 if (skb_network_header_len(skb) > 0) 1823 ip_hdr_len = (skb_network_header_len(skb) >> 2); 1824 l4_proto = ip6h->nexthdr; 1825 } else 1826 return MVNETA_TX_L4_CSUM_NOT; 1827 1828 return mvneta_txq_desc_csum(skb_network_offset(skb), 1829 l3_proto, ip_hdr_len, l4_proto); 1830 } 1831 1832 return MVNETA_TX_L4_CSUM_NOT; 1833 } 1834 1835 /* Drop packets received by the RXQ and free buffers */ 1836 static void mvneta_rxq_drop_pkts(struct mvneta_port *pp, 1837 struct mvneta_rx_queue *rxq) 1838 { 1839 int rx_done, i; 1840 1841 rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); 1842 if (rx_done) 1843 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); 1844 1845 if (pp->bm_priv) { 1846 for (i = 0; i < rx_done; i++) { 1847 struct mvneta_rx_desc *rx_desc = 1848 mvneta_rxq_next_desc_get(rxq); 1849 u8 pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc); 1850 struct mvneta_bm_pool *bm_pool; 1851 1852 bm_pool = &pp->bm_priv->bm_pools[pool_id]; 1853 /* Return dropped buffer to the pool */ 1854 mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool, 1855 rx_desc->buf_phys_addr); 1856 } 1857 return; 1858 } 1859 1860 for (i = 0; i < rxq->size; i++) { 1861 struct mvneta_rx_desc *rx_desc = rxq->descs + i; 1862 void *data = (void *)rx_desc->buf_cookie; 1863 1864 dma_unmap_single(pp->dev->dev.parent, rx_desc->buf_phys_addr, 1865 MVNETA_RX_BUF_SIZE(pp->pkt_size), DMA_FROM_DEVICE); 1866 mvneta_frag_free(pp->frag_size, data); 1867 } 1868 } 1869 1870 /* Main rx processing when using software buffer management */ 1871 static int mvneta_rx_swbm(struct mvneta_port *pp, int rx_todo, 1872 struct mvneta_rx_queue *rxq) 1873 { 1874 struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports); 1875 struct net_device *dev = pp->dev; 1876 int rx_done; 1877 u32 rcvd_pkts = 0; 1878 u32 rcvd_bytes = 0; 1879 1880 /* Get number of received packets */ 1881 rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); 1882 1883 if (rx_todo > rx_done) 1884 rx_todo = rx_done; 1885 1886 rx_done = 0; 1887 1888 /* Fairness NAPI loop */ 1889 while (rx_done < rx_todo) { 1890 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); 1891 struct sk_buff *skb; 1892 unsigned char *data; 1893 dma_addr_t phys_addr; 1894 u32 rx_status, frag_size; 1895 int rx_bytes, err; 1896 1897 rx_done++; 1898 rx_status = rx_desc->status; 1899 rx_bytes = rx_desc->data_size - (ETH_FCS_LEN + MVNETA_MH_SIZE); 1900 data = (unsigned char *)rx_desc->buf_cookie; 1901 phys_addr = rx_desc->buf_phys_addr; 1902 1903 if (!mvneta_rxq_desc_is_first_last(rx_status) || 1904 (rx_status & MVNETA_RXD_ERR_SUMMARY)) { 1905 err_drop_frame: 1906 dev->stats.rx_errors++; 1907 mvneta_rx_error(pp, rx_desc); 1908 /* leave the descriptor untouched */ 1909 continue; 1910 } 1911 1912 if (rx_bytes <= rx_copybreak) { 1913 /* better copy a small frame and not unmap the DMA region */ 1914 skb = netdev_alloc_skb_ip_align(dev, rx_bytes); 1915 if (unlikely(!skb)) 1916 goto err_drop_frame; 1917 1918 dma_sync_single_range_for_cpu(dev->dev.parent, 1919 rx_desc->buf_phys_addr, 1920 MVNETA_MH_SIZE + NET_SKB_PAD, 1921 rx_bytes, 1922 DMA_FROM_DEVICE); 1923 memcpy(skb_put(skb, rx_bytes), 1924 data + MVNETA_MH_SIZE + NET_SKB_PAD, 1925 rx_bytes); 1926 1927 skb->protocol = eth_type_trans(skb, dev); 1928 mvneta_rx_csum(pp, rx_status, skb); 1929 napi_gro_receive(&port->napi, skb); 1930 1931 rcvd_pkts++; 1932 rcvd_bytes += rx_bytes; 1933 1934 /* leave the descriptor and buffer untouched */ 1935 continue; 1936 } 1937 1938 /* Refill processing */ 1939 err = mvneta_rx_refill(pp, rx_desc); 1940 if (err) { 1941 netdev_err(dev, "Linux processing - Can't refill\n"); 1942 rxq->missed++; 1943 goto err_drop_frame; 1944 } 1945 1946 frag_size = pp->frag_size; 1947 1948 skb = build_skb(data, frag_size > PAGE_SIZE ? 0 : frag_size); 1949 1950 /* After refill old buffer has to be unmapped regardless 1951 * the skb is successfully built or not. 1952 */ 1953 dma_unmap_single(dev->dev.parent, phys_addr, 1954 MVNETA_RX_BUF_SIZE(pp->pkt_size), 1955 DMA_FROM_DEVICE); 1956 1957 if (!skb) 1958 goto err_drop_frame; 1959 1960 rcvd_pkts++; 1961 rcvd_bytes += rx_bytes; 1962 1963 /* Linux processing */ 1964 skb_reserve(skb, MVNETA_MH_SIZE + NET_SKB_PAD); 1965 skb_put(skb, rx_bytes); 1966 1967 skb->protocol = eth_type_trans(skb, dev); 1968 1969 mvneta_rx_csum(pp, rx_status, skb); 1970 1971 napi_gro_receive(&port->napi, skb); 1972 } 1973 1974 if (rcvd_pkts) { 1975 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 1976 1977 u64_stats_update_begin(&stats->syncp); 1978 stats->rx_packets += rcvd_pkts; 1979 stats->rx_bytes += rcvd_bytes; 1980 u64_stats_update_end(&stats->syncp); 1981 } 1982 1983 /* Update rxq management counters */ 1984 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); 1985 1986 return rx_done; 1987 } 1988 1989 /* Main rx processing when using hardware buffer management */ 1990 static int mvneta_rx_hwbm(struct mvneta_port *pp, int rx_todo, 1991 struct mvneta_rx_queue *rxq) 1992 { 1993 struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports); 1994 struct net_device *dev = pp->dev; 1995 int rx_done; 1996 u32 rcvd_pkts = 0; 1997 u32 rcvd_bytes = 0; 1998 1999 /* Get number of received packets */ 2000 rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); 2001 2002 if (rx_todo > rx_done) 2003 rx_todo = rx_done; 2004 2005 rx_done = 0; 2006 2007 /* Fairness NAPI loop */ 2008 while (rx_done < rx_todo) { 2009 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); 2010 struct mvneta_bm_pool *bm_pool = NULL; 2011 struct sk_buff *skb; 2012 unsigned char *data; 2013 dma_addr_t phys_addr; 2014 u32 rx_status, frag_size; 2015 int rx_bytes, err; 2016 u8 pool_id; 2017 2018 rx_done++; 2019 rx_status = rx_desc->status; 2020 rx_bytes = rx_desc->data_size - (ETH_FCS_LEN + MVNETA_MH_SIZE); 2021 data = (unsigned char *)rx_desc->buf_cookie; 2022 phys_addr = rx_desc->buf_phys_addr; 2023 pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc); 2024 bm_pool = &pp->bm_priv->bm_pools[pool_id]; 2025 2026 if (!mvneta_rxq_desc_is_first_last(rx_status) || 2027 (rx_status & MVNETA_RXD_ERR_SUMMARY)) { 2028 err_drop_frame_ret_pool: 2029 /* Return the buffer to the pool */ 2030 mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool, 2031 rx_desc->buf_phys_addr); 2032 err_drop_frame: 2033 dev->stats.rx_errors++; 2034 mvneta_rx_error(pp, rx_desc); 2035 /* leave the descriptor untouched */ 2036 continue; 2037 } 2038 2039 if (rx_bytes <= rx_copybreak) { 2040 /* better copy a small frame and not unmap the DMA region */ 2041 skb = netdev_alloc_skb_ip_align(dev, rx_bytes); 2042 if (unlikely(!skb)) 2043 goto err_drop_frame_ret_pool; 2044 2045 dma_sync_single_range_for_cpu(dev->dev.parent, 2046 rx_desc->buf_phys_addr, 2047 MVNETA_MH_SIZE + NET_SKB_PAD, 2048 rx_bytes, 2049 DMA_FROM_DEVICE); 2050 memcpy(skb_put(skb, rx_bytes), 2051 data + MVNETA_MH_SIZE + NET_SKB_PAD, 2052 rx_bytes); 2053 2054 skb->protocol = eth_type_trans(skb, dev); 2055 mvneta_rx_csum(pp, rx_status, skb); 2056 napi_gro_receive(&port->napi, skb); 2057 2058 rcvd_pkts++; 2059 rcvd_bytes += rx_bytes; 2060 2061 /* Return the buffer to the pool */ 2062 mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool, 2063 rx_desc->buf_phys_addr); 2064 2065 /* leave the descriptor and buffer untouched */ 2066 continue; 2067 } 2068 2069 /* Refill processing */ 2070 err = hwbm_pool_refill(&bm_pool->hwbm_pool, GFP_ATOMIC); 2071 if (err) { 2072 netdev_err(dev, "Linux processing - Can't refill\n"); 2073 rxq->missed++; 2074 goto err_drop_frame_ret_pool; 2075 } 2076 2077 frag_size = bm_pool->hwbm_pool.frag_size; 2078 2079 skb = build_skb(data, frag_size > PAGE_SIZE ? 0 : frag_size); 2080 2081 /* After refill old buffer has to be unmapped regardless 2082 * the skb is successfully built or not. 2083 */ 2084 dma_unmap_single(&pp->bm_priv->pdev->dev, phys_addr, 2085 bm_pool->buf_size, DMA_FROM_DEVICE); 2086 if (!skb) 2087 goto err_drop_frame; 2088 2089 rcvd_pkts++; 2090 rcvd_bytes += rx_bytes; 2091 2092 /* Linux processing */ 2093 skb_reserve(skb, MVNETA_MH_SIZE + NET_SKB_PAD); 2094 skb_put(skb, rx_bytes); 2095 2096 skb->protocol = eth_type_trans(skb, dev); 2097 2098 mvneta_rx_csum(pp, rx_status, skb); 2099 2100 napi_gro_receive(&port->napi, skb); 2101 } 2102 2103 if (rcvd_pkts) { 2104 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2105 2106 u64_stats_update_begin(&stats->syncp); 2107 stats->rx_packets += rcvd_pkts; 2108 stats->rx_bytes += rcvd_bytes; 2109 u64_stats_update_end(&stats->syncp); 2110 } 2111 2112 /* Update rxq management counters */ 2113 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); 2114 2115 return rx_done; 2116 } 2117 2118 static inline void 2119 mvneta_tso_put_hdr(struct sk_buff *skb, 2120 struct mvneta_port *pp, struct mvneta_tx_queue *txq) 2121 { 2122 struct mvneta_tx_desc *tx_desc; 2123 int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2124 2125 txq->tx_skb[txq->txq_put_index] = NULL; 2126 tx_desc = mvneta_txq_next_desc_get(txq); 2127 tx_desc->data_size = hdr_len; 2128 tx_desc->command = mvneta_skb_tx_csum(pp, skb); 2129 tx_desc->command |= MVNETA_TXD_F_DESC; 2130 tx_desc->buf_phys_addr = txq->tso_hdrs_phys + 2131 txq->txq_put_index * TSO_HEADER_SIZE; 2132 mvneta_txq_inc_put(txq); 2133 } 2134 2135 static inline int 2136 mvneta_tso_put_data(struct net_device *dev, struct mvneta_tx_queue *txq, 2137 struct sk_buff *skb, char *data, int size, 2138 bool last_tcp, bool is_last) 2139 { 2140 struct mvneta_tx_desc *tx_desc; 2141 2142 tx_desc = mvneta_txq_next_desc_get(txq); 2143 tx_desc->data_size = size; 2144 tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, data, 2145 size, DMA_TO_DEVICE); 2146 if (unlikely(dma_mapping_error(dev->dev.parent, 2147 tx_desc->buf_phys_addr))) { 2148 mvneta_txq_desc_put(txq); 2149 return -ENOMEM; 2150 } 2151 2152 tx_desc->command = 0; 2153 txq->tx_skb[txq->txq_put_index] = NULL; 2154 2155 if (last_tcp) { 2156 /* last descriptor in the TCP packet */ 2157 tx_desc->command = MVNETA_TXD_L_DESC; 2158 2159 /* last descriptor in SKB */ 2160 if (is_last) 2161 txq->tx_skb[txq->txq_put_index] = skb; 2162 } 2163 mvneta_txq_inc_put(txq); 2164 return 0; 2165 } 2166 2167 static int mvneta_tx_tso(struct sk_buff *skb, struct net_device *dev, 2168 struct mvneta_tx_queue *txq) 2169 { 2170 int total_len, data_left; 2171 int desc_count = 0; 2172 struct mvneta_port *pp = netdev_priv(dev); 2173 struct tso_t tso; 2174 int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2175 int i; 2176 2177 /* Count needed descriptors */ 2178 if ((txq->count + tso_count_descs(skb)) >= txq->size) 2179 return 0; 2180 2181 if (skb_headlen(skb) < (skb_transport_offset(skb) + tcp_hdrlen(skb))) { 2182 pr_info("*** Is this even possible???!?!?\n"); 2183 return 0; 2184 } 2185 2186 /* Initialize the TSO handler, and prepare the first payload */ 2187 tso_start(skb, &tso); 2188 2189 total_len = skb->len - hdr_len; 2190 while (total_len > 0) { 2191 char *hdr; 2192 2193 data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len); 2194 total_len -= data_left; 2195 desc_count++; 2196 2197 /* prepare packet headers: MAC + IP + TCP */ 2198 hdr = txq->tso_hdrs + txq->txq_put_index * TSO_HEADER_SIZE; 2199 tso_build_hdr(skb, hdr, &tso, data_left, total_len == 0); 2200 2201 mvneta_tso_put_hdr(skb, pp, txq); 2202 2203 while (data_left > 0) { 2204 int size; 2205 desc_count++; 2206 2207 size = min_t(int, tso.size, data_left); 2208 2209 if (mvneta_tso_put_data(dev, txq, skb, 2210 tso.data, size, 2211 size == data_left, 2212 total_len == 0)) 2213 goto err_release; 2214 data_left -= size; 2215 2216 tso_build_data(skb, &tso, size); 2217 } 2218 } 2219 2220 return desc_count; 2221 2222 err_release: 2223 /* Release all used data descriptors; header descriptors must not 2224 * be DMA-unmapped. 2225 */ 2226 for (i = desc_count - 1; i >= 0; i--) { 2227 struct mvneta_tx_desc *tx_desc = txq->descs + i; 2228 if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr)) 2229 dma_unmap_single(pp->dev->dev.parent, 2230 tx_desc->buf_phys_addr, 2231 tx_desc->data_size, 2232 DMA_TO_DEVICE); 2233 mvneta_txq_desc_put(txq); 2234 } 2235 return 0; 2236 } 2237 2238 /* Handle tx fragmentation processing */ 2239 static int mvneta_tx_frag_process(struct mvneta_port *pp, struct sk_buff *skb, 2240 struct mvneta_tx_queue *txq) 2241 { 2242 struct mvneta_tx_desc *tx_desc; 2243 int i, nr_frags = skb_shinfo(skb)->nr_frags; 2244 2245 for (i = 0; i < nr_frags; i++) { 2246 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2247 void *addr = page_address(frag->page.p) + frag->page_offset; 2248 2249 tx_desc = mvneta_txq_next_desc_get(txq); 2250 tx_desc->data_size = frag->size; 2251 2252 tx_desc->buf_phys_addr = 2253 dma_map_single(pp->dev->dev.parent, addr, 2254 tx_desc->data_size, DMA_TO_DEVICE); 2255 2256 if (dma_mapping_error(pp->dev->dev.parent, 2257 tx_desc->buf_phys_addr)) { 2258 mvneta_txq_desc_put(txq); 2259 goto error; 2260 } 2261 2262 if (i == nr_frags - 1) { 2263 /* Last descriptor */ 2264 tx_desc->command = MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD; 2265 txq->tx_skb[txq->txq_put_index] = skb; 2266 } else { 2267 /* Descriptor in the middle: Not First, Not Last */ 2268 tx_desc->command = 0; 2269 txq->tx_skb[txq->txq_put_index] = NULL; 2270 } 2271 mvneta_txq_inc_put(txq); 2272 } 2273 2274 return 0; 2275 2276 error: 2277 /* Release all descriptors that were used to map fragments of 2278 * this packet, as well as the corresponding DMA mappings 2279 */ 2280 for (i = i - 1; i >= 0; i--) { 2281 tx_desc = txq->descs + i; 2282 dma_unmap_single(pp->dev->dev.parent, 2283 tx_desc->buf_phys_addr, 2284 tx_desc->data_size, 2285 DMA_TO_DEVICE); 2286 mvneta_txq_desc_put(txq); 2287 } 2288 2289 return -ENOMEM; 2290 } 2291 2292 /* Main tx processing */ 2293 static int mvneta_tx(struct sk_buff *skb, struct net_device *dev) 2294 { 2295 struct mvneta_port *pp = netdev_priv(dev); 2296 u16 txq_id = skb_get_queue_mapping(skb); 2297 struct mvneta_tx_queue *txq = &pp->txqs[txq_id]; 2298 struct mvneta_tx_desc *tx_desc; 2299 int len = skb->len; 2300 int frags = 0; 2301 u32 tx_cmd; 2302 2303 if (!netif_running(dev)) 2304 goto out; 2305 2306 if (skb_is_gso(skb)) { 2307 frags = mvneta_tx_tso(skb, dev, txq); 2308 goto out; 2309 } 2310 2311 frags = skb_shinfo(skb)->nr_frags + 1; 2312 2313 /* Get a descriptor for the first part of the packet */ 2314 tx_desc = mvneta_txq_next_desc_get(txq); 2315 2316 tx_cmd = mvneta_skb_tx_csum(pp, skb); 2317 2318 tx_desc->data_size = skb_headlen(skb); 2319 2320 tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, skb->data, 2321 tx_desc->data_size, 2322 DMA_TO_DEVICE); 2323 if (unlikely(dma_mapping_error(dev->dev.parent, 2324 tx_desc->buf_phys_addr))) { 2325 mvneta_txq_desc_put(txq); 2326 frags = 0; 2327 goto out; 2328 } 2329 2330 if (frags == 1) { 2331 /* First and Last descriptor */ 2332 tx_cmd |= MVNETA_TXD_FLZ_DESC; 2333 tx_desc->command = tx_cmd; 2334 txq->tx_skb[txq->txq_put_index] = skb; 2335 mvneta_txq_inc_put(txq); 2336 } else { 2337 /* First but not Last */ 2338 tx_cmd |= MVNETA_TXD_F_DESC; 2339 txq->tx_skb[txq->txq_put_index] = NULL; 2340 mvneta_txq_inc_put(txq); 2341 tx_desc->command = tx_cmd; 2342 /* Continue with other skb fragments */ 2343 if (mvneta_tx_frag_process(pp, skb, txq)) { 2344 dma_unmap_single(dev->dev.parent, 2345 tx_desc->buf_phys_addr, 2346 tx_desc->data_size, 2347 DMA_TO_DEVICE); 2348 mvneta_txq_desc_put(txq); 2349 frags = 0; 2350 goto out; 2351 } 2352 } 2353 2354 out: 2355 if (frags > 0) { 2356 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2357 struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id); 2358 2359 txq->count += frags; 2360 mvneta_txq_pend_desc_add(pp, txq, frags); 2361 2362 if (txq->count >= txq->tx_stop_threshold) 2363 netif_tx_stop_queue(nq); 2364 2365 u64_stats_update_begin(&stats->syncp); 2366 stats->tx_packets++; 2367 stats->tx_bytes += len; 2368 u64_stats_update_end(&stats->syncp); 2369 } else { 2370 dev->stats.tx_dropped++; 2371 dev_kfree_skb_any(skb); 2372 } 2373 2374 return NETDEV_TX_OK; 2375 } 2376 2377 2378 /* Free tx resources, when resetting a port */ 2379 static void mvneta_txq_done_force(struct mvneta_port *pp, 2380 struct mvneta_tx_queue *txq) 2381 2382 { 2383 int tx_done = txq->count; 2384 2385 mvneta_txq_bufs_free(pp, txq, tx_done); 2386 2387 /* reset txq */ 2388 txq->count = 0; 2389 txq->txq_put_index = 0; 2390 txq->txq_get_index = 0; 2391 } 2392 2393 /* Handle tx done - called in softirq context. The <cause_tx_done> argument 2394 * must be a valid cause according to MVNETA_TXQ_INTR_MASK_ALL. 2395 */ 2396 static void mvneta_tx_done_gbe(struct mvneta_port *pp, u32 cause_tx_done) 2397 { 2398 struct mvneta_tx_queue *txq; 2399 struct netdev_queue *nq; 2400 2401 while (cause_tx_done) { 2402 txq = mvneta_tx_done_policy(pp, cause_tx_done); 2403 2404 nq = netdev_get_tx_queue(pp->dev, txq->id); 2405 __netif_tx_lock(nq, smp_processor_id()); 2406 2407 if (txq->count) 2408 mvneta_txq_done(pp, txq); 2409 2410 __netif_tx_unlock(nq); 2411 cause_tx_done &= ~((1 << txq->id)); 2412 } 2413 } 2414 2415 /* Compute crc8 of the specified address, using a unique algorithm , 2416 * according to hw spec, different than generic crc8 algorithm 2417 */ 2418 static int mvneta_addr_crc(unsigned char *addr) 2419 { 2420 int crc = 0; 2421 int i; 2422 2423 for (i = 0; i < ETH_ALEN; i++) { 2424 int j; 2425 2426 crc = (crc ^ addr[i]) << 8; 2427 for (j = 7; j >= 0; j--) { 2428 if (crc & (0x100 << j)) 2429 crc ^= 0x107 << j; 2430 } 2431 } 2432 2433 return crc; 2434 } 2435 2436 /* This method controls the net device special MAC multicast support. 2437 * The Special Multicast Table for MAC addresses supports MAC of the form 2438 * 0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF). 2439 * The MAC DA[7:0] bits are used as a pointer to the Special Multicast 2440 * Table entries in the DA-Filter table. This method set the Special 2441 * Multicast Table appropriate entry. 2442 */ 2443 static void mvneta_set_special_mcast_addr(struct mvneta_port *pp, 2444 unsigned char last_byte, 2445 int queue) 2446 { 2447 unsigned int smc_table_reg; 2448 unsigned int tbl_offset; 2449 unsigned int reg_offset; 2450 2451 /* Register offset from SMC table base */ 2452 tbl_offset = (last_byte / 4); 2453 /* Entry offset within the above reg */ 2454 reg_offset = last_byte % 4; 2455 2456 smc_table_reg = mvreg_read(pp, (MVNETA_DA_FILT_SPEC_MCAST 2457 + tbl_offset * 4)); 2458 2459 if (queue == -1) 2460 smc_table_reg &= ~(0xff << (8 * reg_offset)); 2461 else { 2462 smc_table_reg &= ~(0xff << (8 * reg_offset)); 2463 smc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset)); 2464 } 2465 2466 mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + tbl_offset * 4, 2467 smc_table_reg); 2468 } 2469 2470 /* This method controls the network device Other MAC multicast support. 2471 * The Other Multicast Table is used for multicast of another type. 2472 * A CRC-8 is used as an index to the Other Multicast Table entries 2473 * in the DA-Filter table. 2474 * The method gets the CRC-8 value from the calling routine and 2475 * sets the Other Multicast Table appropriate entry according to the 2476 * specified CRC-8 . 2477 */ 2478 static void mvneta_set_other_mcast_addr(struct mvneta_port *pp, 2479 unsigned char crc8, 2480 int queue) 2481 { 2482 unsigned int omc_table_reg; 2483 unsigned int tbl_offset; 2484 unsigned int reg_offset; 2485 2486 tbl_offset = (crc8 / 4) * 4; /* Register offset from OMC table base */ 2487 reg_offset = crc8 % 4; /* Entry offset within the above reg */ 2488 2489 omc_table_reg = mvreg_read(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset); 2490 2491 if (queue == -1) { 2492 /* Clear accepts frame bit at specified Other DA table entry */ 2493 omc_table_reg &= ~(0xff << (8 * reg_offset)); 2494 } else { 2495 omc_table_reg &= ~(0xff << (8 * reg_offset)); 2496 omc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset)); 2497 } 2498 2499 mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset, omc_table_reg); 2500 } 2501 2502 /* The network device supports multicast using two tables: 2503 * 1) Special Multicast Table for MAC addresses of the form 2504 * 0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF). 2505 * The MAC DA[7:0] bits are used as a pointer to the Special Multicast 2506 * Table entries in the DA-Filter table. 2507 * 2) Other Multicast Table for multicast of another type. A CRC-8 value 2508 * is used as an index to the Other Multicast Table entries in the 2509 * DA-Filter table. 2510 */ 2511 static int mvneta_mcast_addr_set(struct mvneta_port *pp, unsigned char *p_addr, 2512 int queue) 2513 { 2514 unsigned char crc_result = 0; 2515 2516 if (memcmp(p_addr, "\x01\x00\x5e\x00\x00", 5) == 0) { 2517 mvneta_set_special_mcast_addr(pp, p_addr[5], queue); 2518 return 0; 2519 } 2520 2521 crc_result = mvneta_addr_crc(p_addr); 2522 if (queue == -1) { 2523 if (pp->mcast_count[crc_result] == 0) { 2524 netdev_info(pp->dev, "No valid Mcast for crc8=0x%02x\n", 2525 crc_result); 2526 return -EINVAL; 2527 } 2528 2529 pp->mcast_count[crc_result]--; 2530 if (pp->mcast_count[crc_result] != 0) { 2531 netdev_info(pp->dev, 2532 "After delete there are %d valid Mcast for crc8=0x%02x\n", 2533 pp->mcast_count[crc_result], crc_result); 2534 return -EINVAL; 2535 } 2536 } else 2537 pp->mcast_count[crc_result]++; 2538 2539 mvneta_set_other_mcast_addr(pp, crc_result, queue); 2540 2541 return 0; 2542 } 2543 2544 /* Configure Fitering mode of Ethernet port */ 2545 static void mvneta_rx_unicast_promisc_set(struct mvneta_port *pp, 2546 int is_promisc) 2547 { 2548 u32 port_cfg_reg, val; 2549 2550 port_cfg_reg = mvreg_read(pp, MVNETA_PORT_CONFIG); 2551 2552 val = mvreg_read(pp, MVNETA_TYPE_PRIO); 2553 2554 /* Set / Clear UPM bit in port configuration register */ 2555 if (is_promisc) { 2556 /* Accept all Unicast addresses */ 2557 port_cfg_reg |= MVNETA_UNI_PROMISC_MODE; 2558 val |= MVNETA_FORCE_UNI; 2559 mvreg_write(pp, MVNETA_MAC_ADDR_LOW, 0xffff); 2560 mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, 0xffffffff); 2561 } else { 2562 /* Reject all Unicast addresses */ 2563 port_cfg_reg &= ~MVNETA_UNI_PROMISC_MODE; 2564 val &= ~MVNETA_FORCE_UNI; 2565 } 2566 2567 mvreg_write(pp, MVNETA_PORT_CONFIG, port_cfg_reg); 2568 mvreg_write(pp, MVNETA_TYPE_PRIO, val); 2569 } 2570 2571 /* register unicast and multicast addresses */ 2572 static void mvneta_set_rx_mode(struct net_device *dev) 2573 { 2574 struct mvneta_port *pp = netdev_priv(dev); 2575 struct netdev_hw_addr *ha; 2576 2577 if (dev->flags & IFF_PROMISC) { 2578 /* Accept all: Multicast + Unicast */ 2579 mvneta_rx_unicast_promisc_set(pp, 1); 2580 mvneta_set_ucast_table(pp, pp->rxq_def); 2581 mvneta_set_special_mcast_table(pp, pp->rxq_def); 2582 mvneta_set_other_mcast_table(pp, pp->rxq_def); 2583 } else { 2584 /* Accept single Unicast */ 2585 mvneta_rx_unicast_promisc_set(pp, 0); 2586 mvneta_set_ucast_table(pp, -1); 2587 mvneta_mac_addr_set(pp, dev->dev_addr, pp->rxq_def); 2588 2589 if (dev->flags & IFF_ALLMULTI) { 2590 /* Accept all multicast */ 2591 mvneta_set_special_mcast_table(pp, pp->rxq_def); 2592 mvneta_set_other_mcast_table(pp, pp->rxq_def); 2593 } else { 2594 /* Accept only initialized multicast */ 2595 mvneta_set_special_mcast_table(pp, -1); 2596 mvneta_set_other_mcast_table(pp, -1); 2597 2598 if (!netdev_mc_empty(dev)) { 2599 netdev_for_each_mc_addr(ha, dev) { 2600 mvneta_mcast_addr_set(pp, ha->addr, 2601 pp->rxq_def); 2602 } 2603 } 2604 } 2605 } 2606 } 2607 2608 /* Interrupt handling - the callback for request_irq() */ 2609 static irqreturn_t mvneta_isr(int irq, void *dev_id) 2610 { 2611 struct mvneta_pcpu_port *port = (struct mvneta_pcpu_port *)dev_id; 2612 2613 disable_percpu_irq(port->pp->dev->irq); 2614 napi_schedule(&port->napi); 2615 2616 return IRQ_HANDLED; 2617 } 2618 2619 static int mvneta_fixed_link_update(struct mvneta_port *pp, 2620 struct phy_device *phy) 2621 { 2622 struct fixed_phy_status status; 2623 struct fixed_phy_status changed = {}; 2624 u32 gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS); 2625 2626 status.link = !!(gmac_stat & MVNETA_GMAC_LINK_UP); 2627 if (gmac_stat & MVNETA_GMAC_SPEED_1000) 2628 status.speed = SPEED_1000; 2629 else if (gmac_stat & MVNETA_GMAC_SPEED_100) 2630 status.speed = SPEED_100; 2631 else 2632 status.speed = SPEED_10; 2633 status.duplex = !!(gmac_stat & MVNETA_GMAC_FULL_DUPLEX); 2634 changed.link = 1; 2635 changed.speed = 1; 2636 changed.duplex = 1; 2637 fixed_phy_update_state(phy, &status, &changed); 2638 return 0; 2639 } 2640 2641 /* NAPI handler 2642 * Bits 0 - 7 of the causeRxTx register indicate that are transmitted 2643 * packets on the corresponding TXQ (Bit 0 is for TX queue 1). 2644 * Bits 8 -15 of the cause Rx Tx register indicate that are received 2645 * packets on the corresponding RXQ (Bit 8 is for RX queue 0). 2646 * Each CPU has its own causeRxTx register 2647 */ 2648 static int mvneta_poll(struct napi_struct *napi, int budget) 2649 { 2650 int rx_done = 0; 2651 u32 cause_rx_tx; 2652 int rx_queue; 2653 struct mvneta_port *pp = netdev_priv(napi->dev); 2654 struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports); 2655 2656 if (!netif_running(pp->dev)) { 2657 napi_complete(&port->napi); 2658 return rx_done; 2659 } 2660 2661 /* Read cause register */ 2662 cause_rx_tx = mvreg_read(pp, MVNETA_INTR_NEW_CAUSE); 2663 if (cause_rx_tx & MVNETA_MISCINTR_INTR_MASK) { 2664 u32 cause_misc = mvreg_read(pp, MVNETA_INTR_MISC_CAUSE); 2665 2666 mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0); 2667 if (pp->use_inband_status && (cause_misc & 2668 (MVNETA_CAUSE_PHY_STATUS_CHANGE | 2669 MVNETA_CAUSE_LINK_CHANGE | 2670 MVNETA_CAUSE_PSC_SYNC_CHANGE))) { 2671 mvneta_fixed_link_update(pp, pp->phy_dev); 2672 } 2673 } 2674 2675 /* Release Tx descriptors */ 2676 if (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL) { 2677 mvneta_tx_done_gbe(pp, (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL)); 2678 cause_rx_tx &= ~MVNETA_TX_INTR_MASK_ALL; 2679 } 2680 2681 /* For the case where the last mvneta_poll did not process all 2682 * RX packets 2683 */ 2684 rx_queue = fls(((cause_rx_tx >> 8) & 0xff)); 2685 2686 cause_rx_tx |= port->cause_rx_tx; 2687 2688 if (rx_queue) { 2689 rx_queue = rx_queue - 1; 2690 if (pp->bm_priv) 2691 rx_done = mvneta_rx_hwbm(pp, budget, &pp->rxqs[rx_queue]); 2692 else 2693 rx_done = mvneta_rx_swbm(pp, budget, &pp->rxqs[rx_queue]); 2694 } 2695 2696 budget -= rx_done; 2697 2698 if (budget > 0) { 2699 cause_rx_tx = 0; 2700 napi_complete(&port->napi); 2701 enable_percpu_irq(pp->dev->irq, 0); 2702 } 2703 2704 port->cause_rx_tx = cause_rx_tx; 2705 return rx_done; 2706 } 2707 2708 /* Handle rxq fill: allocates rxq skbs; called when initializing a port */ 2709 static int mvneta_rxq_fill(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, 2710 int num) 2711 { 2712 int i; 2713 2714 for (i = 0; i < num; i++) { 2715 memset(rxq->descs + i, 0, sizeof(struct mvneta_rx_desc)); 2716 if (mvneta_rx_refill(pp, rxq->descs + i) != 0) { 2717 netdev_err(pp->dev, "%s:rxq %d, %d of %d buffs filled\n", 2718 __func__, rxq->id, i, num); 2719 break; 2720 } 2721 } 2722 2723 /* Add this number of RX descriptors as non occupied (ready to 2724 * get packets) 2725 */ 2726 mvneta_rxq_non_occup_desc_add(pp, rxq, i); 2727 2728 return i; 2729 } 2730 2731 /* Free all packets pending transmit from all TXQs and reset TX port */ 2732 static void mvneta_tx_reset(struct mvneta_port *pp) 2733 { 2734 int queue; 2735 2736 /* free the skb's in the tx ring */ 2737 for (queue = 0; queue < txq_number; queue++) 2738 mvneta_txq_done_force(pp, &pp->txqs[queue]); 2739 2740 mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET); 2741 mvreg_write(pp, MVNETA_PORT_TX_RESET, 0); 2742 } 2743 2744 static void mvneta_rx_reset(struct mvneta_port *pp) 2745 { 2746 mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET); 2747 mvreg_write(pp, MVNETA_PORT_RX_RESET, 0); 2748 } 2749 2750 /* Rx/Tx queue initialization/cleanup methods */ 2751 2752 /* Create a specified RX queue */ 2753 static int mvneta_rxq_init(struct mvneta_port *pp, 2754 struct mvneta_rx_queue *rxq) 2755 2756 { 2757 rxq->size = pp->rx_ring_size; 2758 2759 /* Allocate memory for RX descriptors */ 2760 rxq->descs = dma_alloc_coherent(pp->dev->dev.parent, 2761 rxq->size * MVNETA_DESC_ALIGNED_SIZE, 2762 &rxq->descs_phys, GFP_KERNEL); 2763 if (rxq->descs == NULL) 2764 return -ENOMEM; 2765 2766 rxq->last_desc = rxq->size - 1; 2767 2768 /* Set Rx descriptors queue starting address */ 2769 mvreg_write(pp, MVNETA_RXQ_BASE_ADDR_REG(rxq->id), rxq->descs_phys); 2770 mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), rxq->size); 2771 2772 /* Set Offset */ 2773 mvneta_rxq_offset_set(pp, rxq, NET_SKB_PAD); 2774 2775 /* Set coalescing pkts and time */ 2776 mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal); 2777 mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal); 2778 2779 if (!pp->bm_priv) { 2780 /* Fill RXQ with buffers from RX pool */ 2781 mvneta_rxq_buf_size_set(pp, rxq, 2782 MVNETA_RX_BUF_SIZE(pp->pkt_size)); 2783 mvneta_rxq_bm_disable(pp, rxq); 2784 } else { 2785 mvneta_rxq_bm_enable(pp, rxq); 2786 mvneta_rxq_long_pool_set(pp, rxq); 2787 mvneta_rxq_short_pool_set(pp, rxq); 2788 } 2789 2790 mvneta_rxq_fill(pp, rxq, rxq->size); 2791 2792 return 0; 2793 } 2794 2795 /* Cleanup Rx queue */ 2796 static void mvneta_rxq_deinit(struct mvneta_port *pp, 2797 struct mvneta_rx_queue *rxq) 2798 { 2799 mvneta_rxq_drop_pkts(pp, rxq); 2800 2801 if (rxq->descs) 2802 dma_free_coherent(pp->dev->dev.parent, 2803 rxq->size * MVNETA_DESC_ALIGNED_SIZE, 2804 rxq->descs, 2805 rxq->descs_phys); 2806 2807 rxq->descs = NULL; 2808 rxq->last_desc = 0; 2809 rxq->next_desc_to_proc = 0; 2810 rxq->descs_phys = 0; 2811 } 2812 2813 /* Create and initialize a tx queue */ 2814 static int mvneta_txq_init(struct mvneta_port *pp, 2815 struct mvneta_tx_queue *txq) 2816 { 2817 int cpu; 2818 2819 txq->size = pp->tx_ring_size; 2820 2821 /* A queue must always have room for at least one skb. 2822 * Therefore, stop the queue when the free entries reaches 2823 * the maximum number of descriptors per skb. 2824 */ 2825 txq->tx_stop_threshold = txq->size - MVNETA_MAX_SKB_DESCS; 2826 txq->tx_wake_threshold = txq->tx_stop_threshold / 2; 2827 2828 2829 /* Allocate memory for TX descriptors */ 2830 txq->descs = dma_alloc_coherent(pp->dev->dev.parent, 2831 txq->size * MVNETA_DESC_ALIGNED_SIZE, 2832 &txq->descs_phys, GFP_KERNEL); 2833 if (txq->descs == NULL) 2834 return -ENOMEM; 2835 2836 txq->last_desc = txq->size - 1; 2837 2838 /* Set maximum bandwidth for enabled TXQs */ 2839 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0x03ffffff); 2840 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0x3fffffff); 2841 2842 /* Set Tx descriptors queue starting address */ 2843 mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), txq->descs_phys); 2844 mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), txq->size); 2845 2846 txq->tx_skb = kmalloc(txq->size * sizeof(*txq->tx_skb), GFP_KERNEL); 2847 if (txq->tx_skb == NULL) { 2848 dma_free_coherent(pp->dev->dev.parent, 2849 txq->size * MVNETA_DESC_ALIGNED_SIZE, 2850 txq->descs, txq->descs_phys); 2851 return -ENOMEM; 2852 } 2853 2854 /* Allocate DMA buffers for TSO MAC/IP/TCP headers */ 2855 txq->tso_hdrs = dma_alloc_coherent(pp->dev->dev.parent, 2856 txq->size * TSO_HEADER_SIZE, 2857 &txq->tso_hdrs_phys, GFP_KERNEL); 2858 if (txq->tso_hdrs == NULL) { 2859 kfree(txq->tx_skb); 2860 dma_free_coherent(pp->dev->dev.parent, 2861 txq->size * MVNETA_DESC_ALIGNED_SIZE, 2862 txq->descs, txq->descs_phys); 2863 return -ENOMEM; 2864 } 2865 mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal); 2866 2867 /* Setup XPS mapping */ 2868 if (txq_number > 1) 2869 cpu = txq->id % num_present_cpus(); 2870 else 2871 cpu = pp->rxq_def % num_present_cpus(); 2872 cpumask_set_cpu(cpu, &txq->affinity_mask); 2873 netif_set_xps_queue(pp->dev, &txq->affinity_mask, txq->id); 2874 2875 return 0; 2876 } 2877 2878 /* Free allocated resources when mvneta_txq_init() fails to allocate memory*/ 2879 static void mvneta_txq_deinit(struct mvneta_port *pp, 2880 struct mvneta_tx_queue *txq) 2881 { 2882 kfree(txq->tx_skb); 2883 2884 if (txq->tso_hdrs) 2885 dma_free_coherent(pp->dev->dev.parent, 2886 txq->size * TSO_HEADER_SIZE, 2887 txq->tso_hdrs, txq->tso_hdrs_phys); 2888 if (txq->descs) 2889 dma_free_coherent(pp->dev->dev.parent, 2890 txq->size * MVNETA_DESC_ALIGNED_SIZE, 2891 txq->descs, txq->descs_phys); 2892 2893 txq->descs = NULL; 2894 txq->last_desc = 0; 2895 txq->next_desc_to_proc = 0; 2896 txq->descs_phys = 0; 2897 2898 /* Set minimum bandwidth for disabled TXQs */ 2899 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0); 2900 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0); 2901 2902 /* Set Tx descriptors queue starting address and size */ 2903 mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), 0); 2904 mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), 0); 2905 } 2906 2907 /* Cleanup all Tx queues */ 2908 static void mvneta_cleanup_txqs(struct mvneta_port *pp) 2909 { 2910 int queue; 2911 2912 for (queue = 0; queue < txq_number; queue++) 2913 mvneta_txq_deinit(pp, &pp->txqs[queue]); 2914 } 2915 2916 /* Cleanup all Rx queues */ 2917 static void mvneta_cleanup_rxqs(struct mvneta_port *pp) 2918 { 2919 int queue; 2920 2921 for (queue = 0; queue < txq_number; queue++) 2922 mvneta_rxq_deinit(pp, &pp->rxqs[queue]); 2923 } 2924 2925 2926 /* Init all Rx queues */ 2927 static int mvneta_setup_rxqs(struct mvneta_port *pp) 2928 { 2929 int queue; 2930 2931 for (queue = 0; queue < rxq_number; queue++) { 2932 int err = mvneta_rxq_init(pp, &pp->rxqs[queue]); 2933 2934 if (err) { 2935 netdev_err(pp->dev, "%s: can't create rxq=%d\n", 2936 __func__, queue); 2937 mvneta_cleanup_rxqs(pp); 2938 return err; 2939 } 2940 } 2941 2942 return 0; 2943 } 2944 2945 /* Init all tx queues */ 2946 static int mvneta_setup_txqs(struct mvneta_port *pp) 2947 { 2948 int queue; 2949 2950 for (queue = 0; queue < txq_number; queue++) { 2951 int err = mvneta_txq_init(pp, &pp->txqs[queue]); 2952 if (err) { 2953 netdev_err(pp->dev, "%s: can't create txq=%d\n", 2954 __func__, queue); 2955 mvneta_cleanup_txqs(pp); 2956 return err; 2957 } 2958 } 2959 2960 return 0; 2961 } 2962 2963 static void mvneta_start_dev(struct mvneta_port *pp) 2964 { 2965 int cpu; 2966 2967 mvneta_max_rx_size_set(pp, pp->pkt_size); 2968 mvneta_txq_max_tx_size_set(pp, pp->pkt_size); 2969 2970 /* start the Rx/Tx activity */ 2971 mvneta_port_enable(pp); 2972 2973 /* Enable polling on the port */ 2974 for_each_online_cpu(cpu) { 2975 struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); 2976 2977 napi_enable(&port->napi); 2978 } 2979 2980 /* Unmask interrupts. It has to be done from each CPU */ 2981 on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true); 2982 2983 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 2984 MVNETA_CAUSE_PHY_STATUS_CHANGE | 2985 MVNETA_CAUSE_LINK_CHANGE | 2986 MVNETA_CAUSE_PSC_SYNC_CHANGE); 2987 2988 phy_start(pp->phy_dev); 2989 netif_tx_start_all_queues(pp->dev); 2990 } 2991 2992 static void mvneta_stop_dev(struct mvneta_port *pp) 2993 { 2994 unsigned int cpu; 2995 2996 phy_stop(pp->phy_dev); 2997 2998 for_each_online_cpu(cpu) { 2999 struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); 3000 3001 napi_disable(&port->napi); 3002 } 3003 3004 netif_carrier_off(pp->dev); 3005 3006 mvneta_port_down(pp); 3007 netif_tx_stop_all_queues(pp->dev); 3008 3009 /* Stop the port activity */ 3010 mvneta_port_disable(pp); 3011 3012 /* Clear all ethernet port interrupts */ 3013 on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true); 3014 3015 /* Mask all ethernet port interrupts */ 3016 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3017 3018 mvneta_tx_reset(pp); 3019 mvneta_rx_reset(pp); 3020 } 3021 3022 /* Return positive if MTU is valid */ 3023 static int mvneta_check_mtu_valid(struct net_device *dev, int mtu) 3024 { 3025 if (mtu < 68) { 3026 netdev_err(dev, "cannot change mtu to less than 68\n"); 3027 return -EINVAL; 3028 } 3029 3030 /* 9676 == 9700 - 20 and rounding to 8 */ 3031 if (mtu > 9676) { 3032 netdev_info(dev, "Illegal MTU value %d, round to 9676\n", mtu); 3033 mtu = 9676; 3034 } 3035 3036 if (!IS_ALIGNED(MVNETA_RX_PKT_SIZE(mtu), 8)) { 3037 netdev_info(dev, "Illegal MTU value %d, rounding to %d\n", 3038 mtu, ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8)); 3039 mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8); 3040 } 3041 3042 return mtu; 3043 } 3044 3045 static void mvneta_percpu_enable(void *arg) 3046 { 3047 struct mvneta_port *pp = arg; 3048 3049 enable_percpu_irq(pp->dev->irq, IRQ_TYPE_NONE); 3050 } 3051 3052 static void mvneta_percpu_disable(void *arg) 3053 { 3054 struct mvneta_port *pp = arg; 3055 3056 disable_percpu_irq(pp->dev->irq); 3057 } 3058 3059 /* Change the device mtu */ 3060 static int mvneta_change_mtu(struct net_device *dev, int mtu) 3061 { 3062 struct mvneta_port *pp = netdev_priv(dev); 3063 int ret; 3064 3065 mtu = mvneta_check_mtu_valid(dev, mtu); 3066 if (mtu < 0) 3067 return -EINVAL; 3068 3069 dev->mtu = mtu; 3070 3071 if (!netif_running(dev)) { 3072 if (pp->bm_priv) 3073 mvneta_bm_update_mtu(pp, mtu); 3074 3075 netdev_update_features(dev); 3076 return 0; 3077 } 3078 3079 /* The interface is running, so we have to force a 3080 * reallocation of the queues 3081 */ 3082 mvneta_stop_dev(pp); 3083 on_each_cpu(mvneta_percpu_disable, pp, true); 3084 3085 mvneta_cleanup_txqs(pp); 3086 mvneta_cleanup_rxqs(pp); 3087 3088 if (pp->bm_priv) 3089 mvneta_bm_update_mtu(pp, mtu); 3090 3091 pp->pkt_size = MVNETA_RX_PKT_SIZE(dev->mtu); 3092 pp->frag_size = SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(pp->pkt_size)) + 3093 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3094 3095 ret = mvneta_setup_rxqs(pp); 3096 if (ret) { 3097 netdev_err(dev, "unable to setup rxqs after MTU change\n"); 3098 return ret; 3099 } 3100 3101 ret = mvneta_setup_txqs(pp); 3102 if (ret) { 3103 netdev_err(dev, "unable to setup txqs after MTU change\n"); 3104 return ret; 3105 } 3106 3107 on_each_cpu(mvneta_percpu_enable, pp, true); 3108 mvneta_start_dev(pp); 3109 mvneta_port_up(pp); 3110 3111 netdev_update_features(dev); 3112 3113 return 0; 3114 } 3115 3116 static netdev_features_t mvneta_fix_features(struct net_device *dev, 3117 netdev_features_t features) 3118 { 3119 struct mvneta_port *pp = netdev_priv(dev); 3120 3121 if (pp->tx_csum_limit && dev->mtu > pp->tx_csum_limit) { 3122 features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO); 3123 netdev_info(dev, 3124 "Disable IP checksum for MTU greater than %dB\n", 3125 pp->tx_csum_limit); 3126 } 3127 3128 return features; 3129 } 3130 3131 /* Get mac address */ 3132 static void mvneta_get_mac_addr(struct mvneta_port *pp, unsigned char *addr) 3133 { 3134 u32 mac_addr_l, mac_addr_h; 3135 3136 mac_addr_l = mvreg_read(pp, MVNETA_MAC_ADDR_LOW); 3137 mac_addr_h = mvreg_read(pp, MVNETA_MAC_ADDR_HIGH); 3138 addr[0] = (mac_addr_h >> 24) & 0xFF; 3139 addr[1] = (mac_addr_h >> 16) & 0xFF; 3140 addr[2] = (mac_addr_h >> 8) & 0xFF; 3141 addr[3] = mac_addr_h & 0xFF; 3142 addr[4] = (mac_addr_l >> 8) & 0xFF; 3143 addr[5] = mac_addr_l & 0xFF; 3144 } 3145 3146 /* Handle setting mac address */ 3147 static int mvneta_set_mac_addr(struct net_device *dev, void *addr) 3148 { 3149 struct mvneta_port *pp = netdev_priv(dev); 3150 struct sockaddr *sockaddr = addr; 3151 int ret; 3152 3153 ret = eth_prepare_mac_addr_change(dev, addr); 3154 if (ret < 0) 3155 return ret; 3156 /* Remove previous address table entry */ 3157 mvneta_mac_addr_set(pp, dev->dev_addr, -1); 3158 3159 /* Set new addr in hw */ 3160 mvneta_mac_addr_set(pp, sockaddr->sa_data, pp->rxq_def); 3161 3162 eth_commit_mac_addr_change(dev, addr); 3163 return 0; 3164 } 3165 3166 static void mvneta_adjust_link(struct net_device *ndev) 3167 { 3168 struct mvneta_port *pp = netdev_priv(ndev); 3169 struct phy_device *phydev = pp->phy_dev; 3170 int status_change = 0; 3171 3172 if (phydev->link) { 3173 if ((pp->speed != phydev->speed) || 3174 (pp->duplex != phydev->duplex)) { 3175 u32 val; 3176 3177 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3178 val &= ~(MVNETA_GMAC_CONFIG_MII_SPEED | 3179 MVNETA_GMAC_CONFIG_GMII_SPEED | 3180 MVNETA_GMAC_CONFIG_FULL_DUPLEX); 3181 3182 if (phydev->duplex) 3183 val |= MVNETA_GMAC_CONFIG_FULL_DUPLEX; 3184 3185 if (phydev->speed == SPEED_1000) 3186 val |= MVNETA_GMAC_CONFIG_GMII_SPEED; 3187 else if (phydev->speed == SPEED_100) 3188 val |= MVNETA_GMAC_CONFIG_MII_SPEED; 3189 3190 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); 3191 3192 pp->duplex = phydev->duplex; 3193 pp->speed = phydev->speed; 3194 } 3195 } 3196 3197 if (phydev->link != pp->link) { 3198 if (!phydev->link) { 3199 pp->duplex = -1; 3200 pp->speed = 0; 3201 } 3202 3203 pp->link = phydev->link; 3204 status_change = 1; 3205 } 3206 3207 if (status_change) { 3208 if (phydev->link) { 3209 if (!pp->use_inband_status) { 3210 u32 val = mvreg_read(pp, 3211 MVNETA_GMAC_AUTONEG_CONFIG); 3212 val &= ~MVNETA_GMAC_FORCE_LINK_DOWN; 3213 val |= MVNETA_GMAC_FORCE_LINK_PASS; 3214 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, 3215 val); 3216 } 3217 mvneta_port_up(pp); 3218 } else { 3219 if (!pp->use_inband_status) { 3220 u32 val = mvreg_read(pp, 3221 MVNETA_GMAC_AUTONEG_CONFIG); 3222 val &= ~MVNETA_GMAC_FORCE_LINK_PASS; 3223 val |= MVNETA_GMAC_FORCE_LINK_DOWN; 3224 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, 3225 val); 3226 } 3227 mvneta_port_down(pp); 3228 } 3229 phy_print_status(phydev); 3230 } 3231 } 3232 3233 static int mvneta_mdio_probe(struct mvneta_port *pp) 3234 { 3235 struct phy_device *phy_dev; 3236 3237 phy_dev = of_phy_connect(pp->dev, pp->phy_node, mvneta_adjust_link, 0, 3238 pp->phy_interface); 3239 if (!phy_dev) { 3240 netdev_err(pp->dev, "could not find the PHY\n"); 3241 return -ENODEV; 3242 } 3243 3244 phy_dev->supported &= PHY_GBIT_FEATURES; 3245 phy_dev->advertising = phy_dev->supported; 3246 3247 pp->phy_dev = phy_dev; 3248 pp->link = 0; 3249 pp->duplex = 0; 3250 pp->speed = 0; 3251 3252 return 0; 3253 } 3254 3255 static void mvneta_mdio_remove(struct mvneta_port *pp) 3256 { 3257 phy_disconnect(pp->phy_dev); 3258 pp->phy_dev = NULL; 3259 } 3260 3261 /* Electing a CPU must be done in an atomic way: it should be done 3262 * after or before the removal/insertion of a CPU and this function is 3263 * not reentrant. 3264 */ 3265 static void mvneta_percpu_elect(struct mvneta_port *pp) 3266 { 3267 int elected_cpu = 0, max_cpu, cpu, i = 0; 3268 3269 /* Use the cpu associated to the rxq when it is online, in all 3270 * the other cases, use the cpu 0 which can't be offline. 3271 */ 3272 if (cpu_online(pp->rxq_def)) 3273 elected_cpu = pp->rxq_def; 3274 3275 max_cpu = num_present_cpus(); 3276 3277 for_each_online_cpu(cpu) { 3278 int rxq_map = 0, txq_map = 0; 3279 int rxq; 3280 3281 for (rxq = 0; rxq < rxq_number; rxq++) 3282 if ((rxq % max_cpu) == cpu) 3283 rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq); 3284 3285 if (cpu == elected_cpu) 3286 /* Map the default receive queue queue to the 3287 * elected CPU 3288 */ 3289 rxq_map |= MVNETA_CPU_RXQ_ACCESS(pp->rxq_def); 3290 3291 /* We update the TX queue map only if we have one 3292 * queue. In this case we associate the TX queue to 3293 * the CPU bound to the default RX queue 3294 */ 3295 if (txq_number == 1) 3296 txq_map = (cpu == elected_cpu) ? 3297 MVNETA_CPU_TXQ_ACCESS(1) : 0; 3298 else 3299 txq_map = mvreg_read(pp, MVNETA_CPU_MAP(cpu)) & 3300 MVNETA_CPU_TXQ_ACCESS_ALL_MASK; 3301 3302 mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map); 3303 3304 /* Update the interrupt mask on each CPU according the 3305 * new mapping 3306 */ 3307 smp_call_function_single(cpu, mvneta_percpu_unmask_interrupt, 3308 pp, true); 3309 i++; 3310 3311 } 3312 }; 3313 3314 static int mvneta_percpu_notifier(struct notifier_block *nfb, 3315 unsigned long action, void *hcpu) 3316 { 3317 struct mvneta_port *pp = container_of(nfb, struct mvneta_port, 3318 cpu_notifier); 3319 int cpu = (unsigned long)hcpu, other_cpu; 3320 struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); 3321 3322 switch (action) { 3323 case CPU_ONLINE: 3324 case CPU_ONLINE_FROZEN: 3325 case CPU_DOWN_FAILED: 3326 case CPU_DOWN_FAILED_FROZEN: 3327 spin_lock(&pp->lock); 3328 /* Configuring the driver for a new CPU while the 3329 * driver is stopping is racy, so just avoid it. 3330 */ 3331 if (pp->is_stopped) { 3332 spin_unlock(&pp->lock); 3333 break; 3334 } 3335 netif_tx_stop_all_queues(pp->dev); 3336 3337 /* We have to synchronise on tha napi of each CPU 3338 * except the one just being waked up 3339 */ 3340 for_each_online_cpu(other_cpu) { 3341 if (other_cpu != cpu) { 3342 struct mvneta_pcpu_port *other_port = 3343 per_cpu_ptr(pp->ports, other_cpu); 3344 3345 napi_synchronize(&other_port->napi); 3346 } 3347 } 3348 3349 /* Mask all ethernet port interrupts */ 3350 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3351 napi_enable(&port->napi); 3352 3353 3354 /* Enable per-CPU interrupts on the CPU that is 3355 * brought up. 3356 */ 3357 mvneta_percpu_enable(pp); 3358 3359 /* Enable per-CPU interrupt on the one CPU we care 3360 * about. 3361 */ 3362 mvneta_percpu_elect(pp); 3363 3364 /* Unmask all ethernet port interrupts */ 3365 on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true); 3366 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 3367 MVNETA_CAUSE_PHY_STATUS_CHANGE | 3368 MVNETA_CAUSE_LINK_CHANGE | 3369 MVNETA_CAUSE_PSC_SYNC_CHANGE); 3370 netif_tx_start_all_queues(pp->dev); 3371 spin_unlock(&pp->lock); 3372 break; 3373 case CPU_DOWN_PREPARE: 3374 case CPU_DOWN_PREPARE_FROZEN: 3375 netif_tx_stop_all_queues(pp->dev); 3376 /* Thanks to this lock we are sure that any pending 3377 * cpu election is done 3378 */ 3379 spin_lock(&pp->lock); 3380 /* Mask all ethernet port interrupts */ 3381 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3382 spin_unlock(&pp->lock); 3383 3384 napi_synchronize(&port->napi); 3385 napi_disable(&port->napi); 3386 /* Disable per-CPU interrupts on the CPU that is 3387 * brought down. 3388 */ 3389 mvneta_percpu_disable(pp); 3390 3391 break; 3392 case CPU_DEAD: 3393 case CPU_DEAD_FROZEN: 3394 /* Check if a new CPU must be elected now this on is down */ 3395 spin_lock(&pp->lock); 3396 mvneta_percpu_elect(pp); 3397 spin_unlock(&pp->lock); 3398 /* Unmask all ethernet port interrupts */ 3399 on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true); 3400 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 3401 MVNETA_CAUSE_PHY_STATUS_CHANGE | 3402 MVNETA_CAUSE_LINK_CHANGE | 3403 MVNETA_CAUSE_PSC_SYNC_CHANGE); 3404 netif_tx_start_all_queues(pp->dev); 3405 break; 3406 } 3407 3408 return NOTIFY_OK; 3409 } 3410 3411 static int mvneta_open(struct net_device *dev) 3412 { 3413 struct mvneta_port *pp = netdev_priv(dev); 3414 int ret; 3415 3416 pp->pkt_size = MVNETA_RX_PKT_SIZE(pp->dev->mtu); 3417 pp->frag_size = SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(pp->pkt_size)) + 3418 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); 3419 3420 ret = mvneta_setup_rxqs(pp); 3421 if (ret) 3422 return ret; 3423 3424 ret = mvneta_setup_txqs(pp); 3425 if (ret) 3426 goto err_cleanup_rxqs; 3427 3428 /* Connect to port interrupt line */ 3429 ret = request_percpu_irq(pp->dev->irq, mvneta_isr, 3430 MVNETA_DRIVER_NAME, pp->ports); 3431 if (ret) { 3432 netdev_err(pp->dev, "cannot request irq %d\n", pp->dev->irq); 3433 goto err_cleanup_txqs; 3434 } 3435 3436 /* Enable per-CPU interrupt on all the CPU to handle our RX 3437 * queue interrupts 3438 */ 3439 on_each_cpu(mvneta_percpu_enable, pp, true); 3440 3441 pp->is_stopped = false; 3442 /* Register a CPU notifier to handle the case where our CPU 3443 * might be taken offline. 3444 */ 3445 register_cpu_notifier(&pp->cpu_notifier); 3446 3447 /* In default link is down */ 3448 netif_carrier_off(pp->dev); 3449 3450 ret = mvneta_mdio_probe(pp); 3451 if (ret < 0) { 3452 netdev_err(dev, "cannot probe MDIO bus\n"); 3453 goto err_free_irq; 3454 } 3455 3456 mvneta_start_dev(pp); 3457 3458 return 0; 3459 3460 err_free_irq: 3461 unregister_cpu_notifier(&pp->cpu_notifier); 3462 on_each_cpu(mvneta_percpu_disable, pp, true); 3463 free_percpu_irq(pp->dev->irq, pp->ports); 3464 err_cleanup_txqs: 3465 mvneta_cleanup_txqs(pp); 3466 err_cleanup_rxqs: 3467 mvneta_cleanup_rxqs(pp); 3468 return ret; 3469 } 3470 3471 /* Stop the port, free port interrupt line */ 3472 static int mvneta_stop(struct net_device *dev) 3473 { 3474 struct mvneta_port *pp = netdev_priv(dev); 3475 3476 /* Inform that we are stopping so we don't want to setup the 3477 * driver for new CPUs in the notifiers. The code of the 3478 * notifier for CPU online is protected by the same spinlock, 3479 * so when we get the lock, the notifer work is done. 3480 */ 3481 spin_lock(&pp->lock); 3482 pp->is_stopped = true; 3483 spin_unlock(&pp->lock); 3484 3485 mvneta_stop_dev(pp); 3486 mvneta_mdio_remove(pp); 3487 unregister_cpu_notifier(&pp->cpu_notifier); 3488 on_each_cpu(mvneta_percpu_disable, pp, true); 3489 free_percpu_irq(dev->irq, pp->ports); 3490 mvneta_cleanup_rxqs(pp); 3491 mvneta_cleanup_txqs(pp); 3492 3493 return 0; 3494 } 3495 3496 static int mvneta_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 3497 { 3498 struct mvneta_port *pp = netdev_priv(dev); 3499 3500 if (!pp->phy_dev) 3501 return -ENOTSUPP; 3502 3503 return phy_mii_ioctl(pp->phy_dev, ifr, cmd); 3504 } 3505 3506 /* Ethtool methods */ 3507 3508 /* Get settings (phy address, speed) for ethtools */ 3509 int mvneta_ethtool_get_settings(struct net_device *dev, struct ethtool_cmd *cmd) 3510 { 3511 struct mvneta_port *pp = netdev_priv(dev); 3512 3513 if (!pp->phy_dev) 3514 return -ENODEV; 3515 3516 return phy_ethtool_gset(pp->phy_dev, cmd); 3517 } 3518 3519 /* Set settings (phy address, speed) for ethtools */ 3520 int mvneta_ethtool_set_settings(struct net_device *dev, struct ethtool_cmd *cmd) 3521 { 3522 struct mvneta_port *pp = netdev_priv(dev); 3523 struct phy_device *phydev = pp->phy_dev; 3524 3525 if (!phydev) 3526 return -ENODEV; 3527 3528 if ((cmd->autoneg == AUTONEG_ENABLE) != pp->use_inband_status) { 3529 u32 val; 3530 3531 mvneta_set_autoneg(pp, cmd->autoneg == AUTONEG_ENABLE); 3532 3533 if (cmd->autoneg == AUTONEG_DISABLE) { 3534 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3535 val &= ~(MVNETA_GMAC_CONFIG_MII_SPEED | 3536 MVNETA_GMAC_CONFIG_GMII_SPEED | 3537 MVNETA_GMAC_CONFIG_FULL_DUPLEX); 3538 3539 if (phydev->duplex) 3540 val |= MVNETA_GMAC_CONFIG_FULL_DUPLEX; 3541 3542 if (phydev->speed == SPEED_1000) 3543 val |= MVNETA_GMAC_CONFIG_GMII_SPEED; 3544 else if (phydev->speed == SPEED_100) 3545 val |= MVNETA_GMAC_CONFIG_MII_SPEED; 3546 3547 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); 3548 } 3549 3550 pp->use_inband_status = (cmd->autoneg == AUTONEG_ENABLE); 3551 netdev_info(pp->dev, "autoneg status set to %i\n", 3552 pp->use_inband_status); 3553 3554 if (netif_running(dev)) { 3555 mvneta_port_down(pp); 3556 mvneta_port_up(pp); 3557 } 3558 } 3559 3560 return phy_ethtool_sset(pp->phy_dev, cmd); 3561 } 3562 3563 /* Set interrupt coalescing for ethtools */ 3564 static int mvneta_ethtool_set_coalesce(struct net_device *dev, 3565 struct ethtool_coalesce *c) 3566 { 3567 struct mvneta_port *pp = netdev_priv(dev); 3568 int queue; 3569 3570 for (queue = 0; queue < rxq_number; queue++) { 3571 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 3572 rxq->time_coal = c->rx_coalesce_usecs; 3573 rxq->pkts_coal = c->rx_max_coalesced_frames; 3574 mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal); 3575 mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal); 3576 } 3577 3578 for (queue = 0; queue < txq_number; queue++) { 3579 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 3580 txq->done_pkts_coal = c->tx_max_coalesced_frames; 3581 mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal); 3582 } 3583 3584 return 0; 3585 } 3586 3587 /* get coalescing for ethtools */ 3588 static int mvneta_ethtool_get_coalesce(struct net_device *dev, 3589 struct ethtool_coalesce *c) 3590 { 3591 struct mvneta_port *pp = netdev_priv(dev); 3592 3593 c->rx_coalesce_usecs = pp->rxqs[0].time_coal; 3594 c->rx_max_coalesced_frames = pp->rxqs[0].pkts_coal; 3595 3596 c->tx_max_coalesced_frames = pp->txqs[0].done_pkts_coal; 3597 return 0; 3598 } 3599 3600 3601 static void mvneta_ethtool_get_drvinfo(struct net_device *dev, 3602 struct ethtool_drvinfo *drvinfo) 3603 { 3604 strlcpy(drvinfo->driver, MVNETA_DRIVER_NAME, 3605 sizeof(drvinfo->driver)); 3606 strlcpy(drvinfo->version, MVNETA_DRIVER_VERSION, 3607 sizeof(drvinfo->version)); 3608 strlcpy(drvinfo->bus_info, dev_name(&dev->dev), 3609 sizeof(drvinfo->bus_info)); 3610 } 3611 3612 3613 static void mvneta_ethtool_get_ringparam(struct net_device *netdev, 3614 struct ethtool_ringparam *ring) 3615 { 3616 struct mvneta_port *pp = netdev_priv(netdev); 3617 3618 ring->rx_max_pending = MVNETA_MAX_RXD; 3619 ring->tx_max_pending = MVNETA_MAX_TXD; 3620 ring->rx_pending = pp->rx_ring_size; 3621 ring->tx_pending = pp->tx_ring_size; 3622 } 3623 3624 static int mvneta_ethtool_set_ringparam(struct net_device *dev, 3625 struct ethtool_ringparam *ring) 3626 { 3627 struct mvneta_port *pp = netdev_priv(dev); 3628 3629 if ((ring->rx_pending == 0) || (ring->tx_pending == 0)) 3630 return -EINVAL; 3631 pp->rx_ring_size = ring->rx_pending < MVNETA_MAX_RXD ? 3632 ring->rx_pending : MVNETA_MAX_RXD; 3633 3634 pp->tx_ring_size = clamp_t(u16, ring->tx_pending, 3635 MVNETA_MAX_SKB_DESCS * 2, MVNETA_MAX_TXD); 3636 if (pp->tx_ring_size != ring->tx_pending) 3637 netdev_warn(dev, "TX queue size set to %u (requested %u)\n", 3638 pp->tx_ring_size, ring->tx_pending); 3639 3640 if (netif_running(dev)) { 3641 mvneta_stop(dev); 3642 if (mvneta_open(dev)) { 3643 netdev_err(dev, 3644 "error on opening device after ring param change\n"); 3645 return -ENOMEM; 3646 } 3647 } 3648 3649 return 0; 3650 } 3651 3652 static void mvneta_ethtool_get_strings(struct net_device *netdev, u32 sset, 3653 u8 *data) 3654 { 3655 if (sset == ETH_SS_STATS) { 3656 int i; 3657 3658 for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) 3659 memcpy(data + i * ETH_GSTRING_LEN, 3660 mvneta_statistics[i].name, ETH_GSTRING_LEN); 3661 } 3662 } 3663 3664 static void mvneta_ethtool_update_stats(struct mvneta_port *pp) 3665 { 3666 const struct mvneta_statistic *s; 3667 void __iomem *base = pp->base; 3668 u32 high, low, val; 3669 u64 val64; 3670 int i; 3671 3672 for (i = 0, s = mvneta_statistics; 3673 s < mvneta_statistics + ARRAY_SIZE(mvneta_statistics); 3674 s++, i++) { 3675 switch (s->type) { 3676 case T_REG_32: 3677 val = readl_relaxed(base + s->offset); 3678 pp->ethtool_stats[i] += val; 3679 break; 3680 case T_REG_64: 3681 /* Docs say to read low 32-bit then high */ 3682 low = readl_relaxed(base + s->offset); 3683 high = readl_relaxed(base + s->offset + 4); 3684 val64 = (u64)high << 32 | low; 3685 pp->ethtool_stats[i] += val64; 3686 break; 3687 } 3688 } 3689 } 3690 3691 static void mvneta_ethtool_get_stats(struct net_device *dev, 3692 struct ethtool_stats *stats, u64 *data) 3693 { 3694 struct mvneta_port *pp = netdev_priv(dev); 3695 int i; 3696 3697 mvneta_ethtool_update_stats(pp); 3698 3699 for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) 3700 *data++ = pp->ethtool_stats[i]; 3701 } 3702 3703 static int mvneta_ethtool_get_sset_count(struct net_device *dev, int sset) 3704 { 3705 if (sset == ETH_SS_STATS) 3706 return ARRAY_SIZE(mvneta_statistics); 3707 return -EOPNOTSUPP; 3708 } 3709 3710 static u32 mvneta_ethtool_get_rxfh_indir_size(struct net_device *dev) 3711 { 3712 return MVNETA_RSS_LU_TABLE_SIZE; 3713 } 3714 3715 static int mvneta_ethtool_get_rxnfc(struct net_device *dev, 3716 struct ethtool_rxnfc *info, 3717 u32 *rules __always_unused) 3718 { 3719 switch (info->cmd) { 3720 case ETHTOOL_GRXRINGS: 3721 info->data = rxq_number; 3722 return 0; 3723 case ETHTOOL_GRXFH: 3724 return -EOPNOTSUPP; 3725 default: 3726 return -EOPNOTSUPP; 3727 } 3728 } 3729 3730 static int mvneta_config_rss(struct mvneta_port *pp) 3731 { 3732 int cpu; 3733 u32 val; 3734 3735 netif_tx_stop_all_queues(pp->dev); 3736 3737 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3738 3739 /* We have to synchronise on the napi of each CPU */ 3740 for_each_online_cpu(cpu) { 3741 struct mvneta_pcpu_port *pcpu_port = 3742 per_cpu_ptr(pp->ports, cpu); 3743 3744 napi_synchronize(&pcpu_port->napi); 3745 napi_disable(&pcpu_port->napi); 3746 } 3747 3748 pp->rxq_def = pp->indir[0]; 3749 3750 /* Update unicast mapping */ 3751 mvneta_set_rx_mode(pp->dev); 3752 3753 /* Update val of portCfg register accordingly with all RxQueue types */ 3754 val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def); 3755 mvreg_write(pp, MVNETA_PORT_CONFIG, val); 3756 3757 /* Update the elected CPU matching the new rxq_def */ 3758 spin_lock(&pp->lock); 3759 mvneta_percpu_elect(pp); 3760 spin_unlock(&pp->lock); 3761 3762 /* We have to synchronise on the napi of each CPU */ 3763 for_each_online_cpu(cpu) { 3764 struct mvneta_pcpu_port *pcpu_port = 3765 per_cpu_ptr(pp->ports, cpu); 3766 3767 napi_enable(&pcpu_port->napi); 3768 } 3769 3770 netif_tx_start_all_queues(pp->dev); 3771 3772 return 0; 3773 } 3774 3775 static int mvneta_ethtool_set_rxfh(struct net_device *dev, const u32 *indir, 3776 const u8 *key, const u8 hfunc) 3777 { 3778 struct mvneta_port *pp = netdev_priv(dev); 3779 /* We require at least one supported parameter to be changed 3780 * and no change in any of the unsupported parameters 3781 */ 3782 if (key || 3783 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) 3784 return -EOPNOTSUPP; 3785 3786 if (!indir) 3787 return 0; 3788 3789 memcpy(pp->indir, indir, MVNETA_RSS_LU_TABLE_SIZE); 3790 3791 return mvneta_config_rss(pp); 3792 } 3793 3794 static int mvneta_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, 3795 u8 *hfunc) 3796 { 3797 struct mvneta_port *pp = netdev_priv(dev); 3798 3799 if (hfunc) 3800 *hfunc = ETH_RSS_HASH_TOP; 3801 3802 if (!indir) 3803 return 0; 3804 3805 memcpy(indir, pp->indir, MVNETA_RSS_LU_TABLE_SIZE); 3806 3807 return 0; 3808 } 3809 3810 static const struct net_device_ops mvneta_netdev_ops = { 3811 .ndo_open = mvneta_open, 3812 .ndo_stop = mvneta_stop, 3813 .ndo_start_xmit = mvneta_tx, 3814 .ndo_set_rx_mode = mvneta_set_rx_mode, 3815 .ndo_set_mac_address = mvneta_set_mac_addr, 3816 .ndo_change_mtu = mvneta_change_mtu, 3817 .ndo_fix_features = mvneta_fix_features, 3818 .ndo_get_stats64 = mvneta_get_stats64, 3819 .ndo_do_ioctl = mvneta_ioctl, 3820 }; 3821 3822 const struct ethtool_ops mvneta_eth_tool_ops = { 3823 .get_link = ethtool_op_get_link, 3824 .get_settings = mvneta_ethtool_get_settings, 3825 .set_settings = mvneta_ethtool_set_settings, 3826 .set_coalesce = mvneta_ethtool_set_coalesce, 3827 .get_coalesce = mvneta_ethtool_get_coalesce, 3828 .get_drvinfo = mvneta_ethtool_get_drvinfo, 3829 .get_ringparam = mvneta_ethtool_get_ringparam, 3830 .set_ringparam = mvneta_ethtool_set_ringparam, 3831 .get_strings = mvneta_ethtool_get_strings, 3832 .get_ethtool_stats = mvneta_ethtool_get_stats, 3833 .get_sset_count = mvneta_ethtool_get_sset_count, 3834 .get_rxfh_indir_size = mvneta_ethtool_get_rxfh_indir_size, 3835 .get_rxnfc = mvneta_ethtool_get_rxnfc, 3836 .get_rxfh = mvneta_ethtool_get_rxfh, 3837 .set_rxfh = mvneta_ethtool_set_rxfh, 3838 }; 3839 3840 /* Initialize hw */ 3841 static int mvneta_init(struct device *dev, struct mvneta_port *pp) 3842 { 3843 int queue; 3844 3845 /* Disable port */ 3846 mvneta_port_disable(pp); 3847 3848 /* Set port default values */ 3849 mvneta_defaults_set(pp); 3850 3851 pp->txqs = devm_kcalloc(dev, txq_number, sizeof(struct mvneta_tx_queue), 3852 GFP_KERNEL); 3853 if (!pp->txqs) 3854 return -ENOMEM; 3855 3856 /* Initialize TX descriptor rings */ 3857 for (queue = 0; queue < txq_number; queue++) { 3858 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 3859 txq->id = queue; 3860 txq->size = pp->tx_ring_size; 3861 txq->done_pkts_coal = MVNETA_TXDONE_COAL_PKTS; 3862 } 3863 3864 pp->rxqs = devm_kcalloc(dev, rxq_number, sizeof(struct mvneta_rx_queue), 3865 GFP_KERNEL); 3866 if (!pp->rxqs) 3867 return -ENOMEM; 3868 3869 /* Create Rx descriptor rings */ 3870 for (queue = 0; queue < rxq_number; queue++) { 3871 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 3872 rxq->id = queue; 3873 rxq->size = pp->rx_ring_size; 3874 rxq->pkts_coal = MVNETA_RX_COAL_PKTS; 3875 rxq->time_coal = MVNETA_RX_COAL_USEC; 3876 } 3877 3878 return 0; 3879 } 3880 3881 /* platform glue : initialize decoding windows */ 3882 static void mvneta_conf_mbus_windows(struct mvneta_port *pp, 3883 const struct mbus_dram_target_info *dram) 3884 { 3885 u32 win_enable; 3886 u32 win_protect; 3887 int i; 3888 3889 for (i = 0; i < 6; i++) { 3890 mvreg_write(pp, MVNETA_WIN_BASE(i), 0); 3891 mvreg_write(pp, MVNETA_WIN_SIZE(i), 0); 3892 3893 if (i < 4) 3894 mvreg_write(pp, MVNETA_WIN_REMAP(i), 0); 3895 } 3896 3897 win_enable = 0x3f; 3898 win_protect = 0; 3899 3900 for (i = 0; i < dram->num_cs; i++) { 3901 const struct mbus_dram_window *cs = dram->cs + i; 3902 mvreg_write(pp, MVNETA_WIN_BASE(i), (cs->base & 0xffff0000) | 3903 (cs->mbus_attr << 8) | dram->mbus_dram_target_id); 3904 3905 mvreg_write(pp, MVNETA_WIN_SIZE(i), 3906 (cs->size - 1) & 0xffff0000); 3907 3908 win_enable &= ~(1 << i); 3909 win_protect |= 3 << (2 * i); 3910 } 3911 3912 mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable); 3913 mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect); 3914 } 3915 3916 /* Power up the port */ 3917 static int mvneta_port_power_up(struct mvneta_port *pp, int phy_mode) 3918 { 3919 u32 ctrl; 3920 3921 /* MAC Cause register should be cleared */ 3922 mvreg_write(pp, MVNETA_UNIT_INTR_CAUSE, 0); 3923 3924 ctrl = mvreg_read(pp, MVNETA_GMAC_CTRL_2); 3925 3926 /* Even though it might look weird, when we're configured in 3927 * SGMII or QSGMII mode, the RGMII bit needs to be set. 3928 */ 3929 switch(phy_mode) { 3930 case PHY_INTERFACE_MODE_QSGMII: 3931 mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_QSGMII_SERDES_PROTO); 3932 ctrl |= MVNETA_GMAC2_PCS_ENABLE | MVNETA_GMAC2_PORT_RGMII; 3933 break; 3934 case PHY_INTERFACE_MODE_SGMII: 3935 mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_SGMII_SERDES_PROTO); 3936 ctrl |= MVNETA_GMAC2_PCS_ENABLE | MVNETA_GMAC2_PORT_RGMII; 3937 break; 3938 case PHY_INTERFACE_MODE_RGMII: 3939 case PHY_INTERFACE_MODE_RGMII_ID: 3940 ctrl |= MVNETA_GMAC2_PORT_RGMII; 3941 break; 3942 default: 3943 return -EINVAL; 3944 } 3945 3946 /* Cancel Port Reset */ 3947 ctrl &= ~MVNETA_GMAC2_PORT_RESET; 3948 mvreg_write(pp, MVNETA_GMAC_CTRL_2, ctrl); 3949 3950 while ((mvreg_read(pp, MVNETA_GMAC_CTRL_2) & 3951 MVNETA_GMAC2_PORT_RESET) != 0) 3952 continue; 3953 3954 return 0; 3955 } 3956 3957 /* Device initialization routine */ 3958 static int mvneta_probe(struct platform_device *pdev) 3959 { 3960 const struct mbus_dram_target_info *dram_target_info; 3961 struct resource *res; 3962 struct device_node *dn = pdev->dev.of_node; 3963 struct device_node *phy_node; 3964 struct device_node *bm_node; 3965 struct mvneta_port *pp; 3966 struct net_device *dev; 3967 const char *dt_mac_addr; 3968 char hw_mac_addr[ETH_ALEN]; 3969 const char *mac_from; 3970 const char *managed; 3971 int tx_csum_limit; 3972 int phy_mode; 3973 int err; 3974 int cpu; 3975 3976 dev = alloc_etherdev_mqs(sizeof(struct mvneta_port), txq_number, rxq_number); 3977 if (!dev) 3978 return -ENOMEM; 3979 3980 dev->irq = irq_of_parse_and_map(dn, 0); 3981 if (dev->irq == 0) { 3982 err = -EINVAL; 3983 goto err_free_netdev; 3984 } 3985 3986 phy_node = of_parse_phandle(dn, "phy", 0); 3987 if (!phy_node) { 3988 if (!of_phy_is_fixed_link(dn)) { 3989 dev_err(&pdev->dev, "no PHY specified\n"); 3990 err = -ENODEV; 3991 goto err_free_irq; 3992 } 3993 3994 err = of_phy_register_fixed_link(dn); 3995 if (err < 0) { 3996 dev_err(&pdev->dev, "cannot register fixed PHY\n"); 3997 goto err_free_irq; 3998 } 3999 4000 /* In the case of a fixed PHY, the DT node associated 4001 * to the PHY is the Ethernet MAC DT node. 4002 */ 4003 phy_node = of_node_get(dn); 4004 } 4005 4006 phy_mode = of_get_phy_mode(dn); 4007 if (phy_mode < 0) { 4008 dev_err(&pdev->dev, "incorrect phy-mode\n"); 4009 err = -EINVAL; 4010 goto err_put_phy_node; 4011 } 4012 4013 dev->tx_queue_len = MVNETA_MAX_TXD; 4014 dev->watchdog_timeo = 5 * HZ; 4015 dev->netdev_ops = &mvneta_netdev_ops; 4016 4017 dev->ethtool_ops = &mvneta_eth_tool_ops; 4018 4019 pp = netdev_priv(dev); 4020 spin_lock_init(&pp->lock); 4021 pp->phy_node = phy_node; 4022 pp->phy_interface = phy_mode; 4023 4024 err = of_property_read_string(dn, "managed", &managed); 4025 pp->use_inband_status = (err == 0 && 4026 strcmp(managed, "in-band-status") == 0); 4027 pp->cpu_notifier.notifier_call = mvneta_percpu_notifier; 4028 4029 pp->rxq_def = rxq_def; 4030 4031 pp->indir[0] = rxq_def; 4032 4033 pp->clk = devm_clk_get(&pdev->dev, "core"); 4034 if (IS_ERR(pp->clk)) 4035 pp->clk = devm_clk_get(&pdev->dev, NULL); 4036 if (IS_ERR(pp->clk)) { 4037 err = PTR_ERR(pp->clk); 4038 goto err_put_phy_node; 4039 } 4040 4041 clk_prepare_enable(pp->clk); 4042 4043 pp->clk_bus = devm_clk_get(&pdev->dev, "bus"); 4044 if (!IS_ERR(pp->clk_bus)) 4045 clk_prepare_enable(pp->clk_bus); 4046 4047 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4048 pp->base = devm_ioremap_resource(&pdev->dev, res); 4049 if (IS_ERR(pp->base)) { 4050 err = PTR_ERR(pp->base); 4051 goto err_clk; 4052 } 4053 4054 /* Alloc per-cpu port structure */ 4055 pp->ports = alloc_percpu(struct mvneta_pcpu_port); 4056 if (!pp->ports) { 4057 err = -ENOMEM; 4058 goto err_clk; 4059 } 4060 4061 /* Alloc per-cpu stats */ 4062 pp->stats = netdev_alloc_pcpu_stats(struct mvneta_pcpu_stats); 4063 if (!pp->stats) { 4064 err = -ENOMEM; 4065 goto err_free_ports; 4066 } 4067 4068 dt_mac_addr = of_get_mac_address(dn); 4069 if (dt_mac_addr) { 4070 mac_from = "device tree"; 4071 memcpy(dev->dev_addr, dt_mac_addr, ETH_ALEN); 4072 } else { 4073 mvneta_get_mac_addr(pp, hw_mac_addr); 4074 if (is_valid_ether_addr(hw_mac_addr)) { 4075 mac_from = "hardware"; 4076 memcpy(dev->dev_addr, hw_mac_addr, ETH_ALEN); 4077 } else { 4078 mac_from = "random"; 4079 eth_hw_addr_random(dev); 4080 } 4081 } 4082 4083 if (!of_property_read_u32(dn, "tx-csum-limit", &tx_csum_limit)) { 4084 if (tx_csum_limit < 0 || 4085 tx_csum_limit > MVNETA_TX_CSUM_MAX_SIZE) { 4086 tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE; 4087 dev_info(&pdev->dev, 4088 "Wrong TX csum limit in DT, set to %dB\n", 4089 MVNETA_TX_CSUM_DEF_SIZE); 4090 } 4091 } else if (of_device_is_compatible(dn, "marvell,armada-370-neta")) { 4092 tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE; 4093 } else { 4094 tx_csum_limit = MVNETA_TX_CSUM_MAX_SIZE; 4095 } 4096 4097 pp->tx_csum_limit = tx_csum_limit; 4098 4099 dram_target_info = mv_mbus_dram_info(); 4100 if (dram_target_info) 4101 mvneta_conf_mbus_windows(pp, dram_target_info); 4102 4103 pp->tx_ring_size = MVNETA_MAX_TXD; 4104 pp->rx_ring_size = MVNETA_MAX_RXD; 4105 4106 pp->dev = dev; 4107 SET_NETDEV_DEV(dev, &pdev->dev); 4108 4109 pp->id = global_port_id++; 4110 4111 /* Obtain access to BM resources if enabled and already initialized */ 4112 bm_node = of_parse_phandle(dn, "buffer-manager", 0); 4113 if (bm_node && bm_node->data) { 4114 pp->bm_priv = bm_node->data; 4115 err = mvneta_bm_port_init(pdev, pp); 4116 if (err < 0) { 4117 dev_info(&pdev->dev, "use SW buffer management\n"); 4118 pp->bm_priv = NULL; 4119 } 4120 } 4121 4122 err = mvneta_init(&pdev->dev, pp); 4123 if (err < 0) 4124 goto err_netdev; 4125 4126 err = mvneta_port_power_up(pp, phy_mode); 4127 if (err < 0) { 4128 dev_err(&pdev->dev, "can't power up port\n"); 4129 goto err_netdev; 4130 } 4131 4132 for_each_present_cpu(cpu) { 4133 struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); 4134 4135 netif_napi_add(dev, &port->napi, mvneta_poll, NAPI_POLL_WEIGHT); 4136 port->pp = pp; 4137 } 4138 4139 dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_TSO; 4140 dev->hw_features |= dev->features; 4141 dev->vlan_features |= dev->features; 4142 dev->priv_flags |= IFF_UNICAST_FLT | IFF_LIVE_ADDR_CHANGE; 4143 dev->gso_max_segs = MVNETA_MAX_TSO_SEGS; 4144 4145 err = register_netdev(dev); 4146 if (err < 0) { 4147 dev_err(&pdev->dev, "failed to register\n"); 4148 goto err_free_stats; 4149 } 4150 4151 netdev_info(dev, "Using %s mac address %pM\n", mac_from, 4152 dev->dev_addr); 4153 4154 platform_set_drvdata(pdev, pp->dev); 4155 4156 if (pp->use_inband_status) { 4157 struct phy_device *phy = of_phy_find_device(dn); 4158 4159 mvneta_fixed_link_update(pp, phy); 4160 4161 put_device(&phy->mdio.dev); 4162 } 4163 4164 return 0; 4165 4166 err_netdev: 4167 unregister_netdev(dev); 4168 if (pp->bm_priv) { 4169 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 4170 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 4171 1 << pp->id); 4172 } 4173 err_free_stats: 4174 free_percpu(pp->stats); 4175 err_free_ports: 4176 free_percpu(pp->ports); 4177 err_clk: 4178 clk_disable_unprepare(pp->clk_bus); 4179 clk_disable_unprepare(pp->clk); 4180 err_put_phy_node: 4181 of_node_put(phy_node); 4182 err_free_irq: 4183 irq_dispose_mapping(dev->irq); 4184 err_free_netdev: 4185 free_netdev(dev); 4186 return err; 4187 } 4188 4189 /* Device removal routine */ 4190 static int mvneta_remove(struct platform_device *pdev) 4191 { 4192 struct net_device *dev = platform_get_drvdata(pdev); 4193 struct mvneta_port *pp = netdev_priv(dev); 4194 4195 unregister_netdev(dev); 4196 clk_disable_unprepare(pp->clk_bus); 4197 clk_disable_unprepare(pp->clk); 4198 free_percpu(pp->ports); 4199 free_percpu(pp->stats); 4200 irq_dispose_mapping(dev->irq); 4201 of_node_put(pp->phy_node); 4202 free_netdev(dev); 4203 4204 if (pp->bm_priv) { 4205 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 4206 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 4207 1 << pp->id); 4208 } 4209 4210 return 0; 4211 } 4212 4213 static const struct of_device_id mvneta_match[] = { 4214 { .compatible = "marvell,armada-370-neta" }, 4215 { .compatible = "marvell,armada-xp-neta" }, 4216 { } 4217 }; 4218 MODULE_DEVICE_TABLE(of, mvneta_match); 4219 4220 static struct platform_driver mvneta_driver = { 4221 .probe = mvneta_probe, 4222 .remove = mvneta_remove, 4223 .driver = { 4224 .name = MVNETA_DRIVER_NAME, 4225 .of_match_table = mvneta_match, 4226 }, 4227 }; 4228 4229 module_platform_driver(mvneta_driver); 4230 4231 MODULE_DESCRIPTION("Marvell NETA Ethernet Driver - www.marvell.com"); 4232 MODULE_AUTHOR("Rami Rosen <rosenr@marvell.com>, Thomas Petazzoni <thomas.petazzoni@free-electrons.com>"); 4233 MODULE_LICENSE("GPL"); 4234 4235 module_param(rxq_number, int, S_IRUGO); 4236 module_param(txq_number, int, S_IRUGO); 4237 4238 module_param(rxq_def, int, S_IRUGO); 4239 module_param(rx_copybreak, int, S_IRUGO | S_IWUSR); 4240