1 /* 2 * Driver for Marvell NETA network card for Armada XP and Armada 370 SoCs. 3 * 4 * Copyright (C) 2012 Marvell 5 * 6 * Rami Rosen <rosenr@marvell.com> 7 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> 8 * 9 * This file is licensed under the terms of the GNU General Public 10 * License version 2. This program is licensed "as is" without any 11 * warranty of any kind, whether express or implied. 12 */ 13 14 #include <linux/clk.h> 15 #include <linux/cpu.h> 16 #include <linux/etherdevice.h> 17 #include <linux/if_vlan.h> 18 #include <linux/inetdevice.h> 19 #include <linux/interrupt.h> 20 #include <linux/io.h> 21 #include <linux/kernel.h> 22 #include <linux/mbus.h> 23 #include <linux/module.h> 24 #include <linux/netdevice.h> 25 #include <linux/of.h> 26 #include <linux/of_address.h> 27 #include <linux/of_irq.h> 28 #include <linux/of_mdio.h> 29 #include <linux/of_net.h> 30 #include <linux/phy/phy.h> 31 #include <linux/phy.h> 32 #include <linux/phylink.h> 33 #include <linux/platform_device.h> 34 #include <linux/skbuff.h> 35 #include <net/hwbm.h> 36 #include "mvneta_bm.h" 37 #include <net/ip.h> 38 #include <net/ipv6.h> 39 #include <net/tso.h> 40 41 /* Registers */ 42 #define MVNETA_RXQ_CONFIG_REG(q) (0x1400 + ((q) << 2)) 43 #define MVNETA_RXQ_HW_BUF_ALLOC BIT(0) 44 #define MVNETA_RXQ_SHORT_POOL_ID_SHIFT 4 45 #define MVNETA_RXQ_SHORT_POOL_ID_MASK 0x30 46 #define MVNETA_RXQ_LONG_POOL_ID_SHIFT 6 47 #define MVNETA_RXQ_LONG_POOL_ID_MASK 0xc0 48 #define MVNETA_RXQ_PKT_OFFSET_ALL_MASK (0xf << 8) 49 #define MVNETA_RXQ_PKT_OFFSET_MASK(offs) ((offs) << 8) 50 #define MVNETA_RXQ_THRESHOLD_REG(q) (0x14c0 + ((q) << 2)) 51 #define MVNETA_RXQ_NON_OCCUPIED(v) ((v) << 16) 52 #define MVNETA_RXQ_BASE_ADDR_REG(q) (0x1480 + ((q) << 2)) 53 #define MVNETA_RXQ_SIZE_REG(q) (0x14a0 + ((q) << 2)) 54 #define MVNETA_RXQ_BUF_SIZE_SHIFT 19 55 #define MVNETA_RXQ_BUF_SIZE_MASK (0x1fff << 19) 56 #define MVNETA_RXQ_STATUS_REG(q) (0x14e0 + ((q) << 2)) 57 #define MVNETA_RXQ_OCCUPIED_ALL_MASK 0x3fff 58 #define MVNETA_RXQ_STATUS_UPDATE_REG(q) (0x1500 + ((q) << 2)) 59 #define MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT 16 60 #define MVNETA_RXQ_ADD_NON_OCCUPIED_MAX 255 61 #define MVNETA_PORT_POOL_BUFFER_SZ_REG(pool) (0x1700 + ((pool) << 2)) 62 #define MVNETA_PORT_POOL_BUFFER_SZ_SHIFT 3 63 #define MVNETA_PORT_POOL_BUFFER_SZ_MASK 0xfff8 64 #define MVNETA_PORT_RX_RESET 0x1cc0 65 #define MVNETA_PORT_RX_DMA_RESET BIT(0) 66 #define MVNETA_PHY_ADDR 0x2000 67 #define MVNETA_PHY_ADDR_MASK 0x1f 68 #define MVNETA_MBUS_RETRY 0x2010 69 #define MVNETA_UNIT_INTR_CAUSE 0x2080 70 #define MVNETA_UNIT_CONTROL 0x20B0 71 #define MVNETA_PHY_POLLING_ENABLE BIT(1) 72 #define MVNETA_WIN_BASE(w) (0x2200 + ((w) << 3)) 73 #define MVNETA_WIN_SIZE(w) (0x2204 + ((w) << 3)) 74 #define MVNETA_WIN_REMAP(w) (0x2280 + ((w) << 2)) 75 #define MVNETA_BASE_ADDR_ENABLE 0x2290 76 #define MVNETA_ACCESS_PROTECT_ENABLE 0x2294 77 #define MVNETA_PORT_CONFIG 0x2400 78 #define MVNETA_UNI_PROMISC_MODE BIT(0) 79 #define MVNETA_DEF_RXQ(q) ((q) << 1) 80 #define MVNETA_DEF_RXQ_ARP(q) ((q) << 4) 81 #define MVNETA_TX_UNSET_ERR_SUM BIT(12) 82 #define MVNETA_DEF_RXQ_TCP(q) ((q) << 16) 83 #define MVNETA_DEF_RXQ_UDP(q) ((q) << 19) 84 #define MVNETA_DEF_RXQ_BPDU(q) ((q) << 22) 85 #define MVNETA_RX_CSUM_WITH_PSEUDO_HDR BIT(25) 86 #define MVNETA_PORT_CONFIG_DEFL_VALUE(q) (MVNETA_DEF_RXQ(q) | \ 87 MVNETA_DEF_RXQ_ARP(q) | \ 88 MVNETA_DEF_RXQ_TCP(q) | \ 89 MVNETA_DEF_RXQ_UDP(q) | \ 90 MVNETA_DEF_RXQ_BPDU(q) | \ 91 MVNETA_TX_UNSET_ERR_SUM | \ 92 MVNETA_RX_CSUM_WITH_PSEUDO_HDR) 93 #define MVNETA_PORT_CONFIG_EXTEND 0x2404 94 #define MVNETA_MAC_ADDR_LOW 0x2414 95 #define MVNETA_MAC_ADDR_HIGH 0x2418 96 #define MVNETA_SDMA_CONFIG 0x241c 97 #define MVNETA_SDMA_BRST_SIZE_16 4 98 #define MVNETA_RX_BRST_SZ_MASK(burst) ((burst) << 1) 99 #define MVNETA_RX_NO_DATA_SWAP BIT(4) 100 #define MVNETA_TX_NO_DATA_SWAP BIT(5) 101 #define MVNETA_DESC_SWAP BIT(6) 102 #define MVNETA_TX_BRST_SZ_MASK(burst) ((burst) << 22) 103 #define MVNETA_PORT_STATUS 0x2444 104 #define MVNETA_TX_IN_PRGRS BIT(1) 105 #define MVNETA_TX_FIFO_EMPTY BIT(8) 106 #define MVNETA_RX_MIN_FRAME_SIZE 0x247c 107 #define MVNETA_SERDES_CFG 0x24A0 108 #define MVNETA_SGMII_SERDES_PROTO 0x0cc7 109 #define MVNETA_QSGMII_SERDES_PROTO 0x0667 110 #define MVNETA_TYPE_PRIO 0x24bc 111 #define MVNETA_FORCE_UNI BIT(21) 112 #define MVNETA_TXQ_CMD_1 0x24e4 113 #define MVNETA_TXQ_CMD 0x2448 114 #define MVNETA_TXQ_DISABLE_SHIFT 8 115 #define MVNETA_TXQ_ENABLE_MASK 0x000000ff 116 #define MVNETA_RX_DISCARD_FRAME_COUNT 0x2484 117 #define MVNETA_OVERRUN_FRAME_COUNT 0x2488 118 #define MVNETA_GMAC_CLOCK_DIVIDER 0x24f4 119 #define MVNETA_GMAC_1MS_CLOCK_ENABLE BIT(31) 120 #define MVNETA_ACC_MODE 0x2500 121 #define MVNETA_BM_ADDRESS 0x2504 122 #define MVNETA_CPU_MAP(cpu) (0x2540 + ((cpu) << 2)) 123 #define MVNETA_CPU_RXQ_ACCESS_ALL_MASK 0x000000ff 124 #define MVNETA_CPU_TXQ_ACCESS_ALL_MASK 0x0000ff00 125 #define MVNETA_CPU_RXQ_ACCESS(rxq) BIT(rxq) 126 #define MVNETA_CPU_TXQ_ACCESS(txq) BIT(txq + 8) 127 #define MVNETA_RXQ_TIME_COAL_REG(q) (0x2580 + ((q) << 2)) 128 129 /* Exception Interrupt Port/Queue Cause register 130 * 131 * Their behavior depend of the mapping done using the PCPX2Q 132 * registers. For a given CPU if the bit associated to a queue is not 133 * set, then for the register a read from this CPU will always return 134 * 0 and a write won't do anything 135 */ 136 137 #define MVNETA_INTR_NEW_CAUSE 0x25a0 138 #define MVNETA_INTR_NEW_MASK 0x25a4 139 140 /* bits 0..7 = TXQ SENT, one bit per queue. 141 * bits 8..15 = RXQ OCCUP, one bit per queue. 142 * bits 16..23 = RXQ FREE, one bit per queue. 143 * bit 29 = OLD_REG_SUM, see old reg ? 144 * bit 30 = TX_ERR_SUM, one bit for 4 ports 145 * bit 31 = MISC_SUM, one bit for 4 ports 146 */ 147 #define MVNETA_TX_INTR_MASK(nr_txqs) (((1 << nr_txqs) - 1) << 0) 148 #define MVNETA_TX_INTR_MASK_ALL (0xff << 0) 149 #define MVNETA_RX_INTR_MASK(nr_rxqs) (((1 << nr_rxqs) - 1) << 8) 150 #define MVNETA_RX_INTR_MASK_ALL (0xff << 8) 151 #define MVNETA_MISCINTR_INTR_MASK BIT(31) 152 153 #define MVNETA_INTR_OLD_CAUSE 0x25a8 154 #define MVNETA_INTR_OLD_MASK 0x25ac 155 156 /* Data Path Port/Queue Cause Register */ 157 #define MVNETA_INTR_MISC_CAUSE 0x25b0 158 #define MVNETA_INTR_MISC_MASK 0x25b4 159 160 #define MVNETA_CAUSE_PHY_STATUS_CHANGE BIT(0) 161 #define MVNETA_CAUSE_LINK_CHANGE BIT(1) 162 #define MVNETA_CAUSE_PTP BIT(4) 163 164 #define MVNETA_CAUSE_INTERNAL_ADDR_ERR BIT(7) 165 #define MVNETA_CAUSE_RX_OVERRUN BIT(8) 166 #define MVNETA_CAUSE_RX_CRC_ERROR BIT(9) 167 #define MVNETA_CAUSE_RX_LARGE_PKT BIT(10) 168 #define MVNETA_CAUSE_TX_UNDERUN BIT(11) 169 #define MVNETA_CAUSE_PRBS_ERR BIT(12) 170 #define MVNETA_CAUSE_PSC_SYNC_CHANGE BIT(13) 171 #define MVNETA_CAUSE_SERDES_SYNC_ERR BIT(14) 172 173 #define MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT 16 174 #define MVNETA_CAUSE_BMU_ALLOC_ERR_ALL_MASK (0xF << MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT) 175 #define MVNETA_CAUSE_BMU_ALLOC_ERR_MASK(pool) (1 << (MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT + (pool))) 176 177 #define MVNETA_CAUSE_TXQ_ERROR_SHIFT 24 178 #define MVNETA_CAUSE_TXQ_ERROR_ALL_MASK (0xFF << MVNETA_CAUSE_TXQ_ERROR_SHIFT) 179 #define MVNETA_CAUSE_TXQ_ERROR_MASK(q) (1 << (MVNETA_CAUSE_TXQ_ERROR_SHIFT + (q))) 180 181 #define MVNETA_INTR_ENABLE 0x25b8 182 #define MVNETA_TXQ_INTR_ENABLE_ALL_MASK 0x0000ff00 183 #define MVNETA_RXQ_INTR_ENABLE_ALL_MASK 0x000000ff 184 185 #define MVNETA_RXQ_CMD 0x2680 186 #define MVNETA_RXQ_DISABLE_SHIFT 8 187 #define MVNETA_RXQ_ENABLE_MASK 0x000000ff 188 #define MVETH_TXQ_TOKEN_COUNT_REG(q) (0x2700 + ((q) << 4)) 189 #define MVETH_TXQ_TOKEN_CFG_REG(q) (0x2704 + ((q) << 4)) 190 #define MVNETA_GMAC_CTRL_0 0x2c00 191 #define MVNETA_GMAC_MAX_RX_SIZE_SHIFT 2 192 #define MVNETA_GMAC_MAX_RX_SIZE_MASK 0x7ffc 193 #define MVNETA_GMAC0_PORT_1000BASE_X BIT(1) 194 #define MVNETA_GMAC0_PORT_ENABLE BIT(0) 195 #define MVNETA_GMAC_CTRL_2 0x2c08 196 #define MVNETA_GMAC2_INBAND_AN_ENABLE BIT(0) 197 #define MVNETA_GMAC2_PCS_ENABLE BIT(3) 198 #define MVNETA_GMAC2_PORT_RGMII BIT(4) 199 #define MVNETA_GMAC2_PORT_RESET BIT(6) 200 #define MVNETA_GMAC_STATUS 0x2c10 201 #define MVNETA_GMAC_LINK_UP BIT(0) 202 #define MVNETA_GMAC_SPEED_1000 BIT(1) 203 #define MVNETA_GMAC_SPEED_100 BIT(2) 204 #define MVNETA_GMAC_FULL_DUPLEX BIT(3) 205 #define MVNETA_GMAC_RX_FLOW_CTRL_ENABLE BIT(4) 206 #define MVNETA_GMAC_TX_FLOW_CTRL_ENABLE BIT(5) 207 #define MVNETA_GMAC_RX_FLOW_CTRL_ACTIVE BIT(6) 208 #define MVNETA_GMAC_TX_FLOW_CTRL_ACTIVE BIT(7) 209 #define MVNETA_GMAC_AN_COMPLETE BIT(11) 210 #define MVNETA_GMAC_SYNC_OK BIT(14) 211 #define MVNETA_GMAC_AUTONEG_CONFIG 0x2c0c 212 #define MVNETA_GMAC_FORCE_LINK_DOWN BIT(0) 213 #define MVNETA_GMAC_FORCE_LINK_PASS BIT(1) 214 #define MVNETA_GMAC_INBAND_AN_ENABLE BIT(2) 215 #define MVNETA_GMAC_AN_BYPASS_ENABLE BIT(3) 216 #define MVNETA_GMAC_INBAND_RESTART_AN BIT(4) 217 #define MVNETA_GMAC_CONFIG_MII_SPEED BIT(5) 218 #define MVNETA_GMAC_CONFIG_GMII_SPEED BIT(6) 219 #define MVNETA_GMAC_AN_SPEED_EN BIT(7) 220 #define MVNETA_GMAC_CONFIG_FLOW_CTRL BIT(8) 221 #define MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL BIT(9) 222 #define MVNETA_GMAC_AN_FLOW_CTRL_EN BIT(11) 223 #define MVNETA_GMAC_CONFIG_FULL_DUPLEX BIT(12) 224 #define MVNETA_GMAC_AN_DUPLEX_EN BIT(13) 225 #define MVNETA_GMAC_CTRL_4 0x2c90 226 #define MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE BIT(1) 227 #define MVNETA_MIB_COUNTERS_BASE 0x3000 228 #define MVNETA_MIB_LATE_COLLISION 0x7c 229 #define MVNETA_DA_FILT_SPEC_MCAST 0x3400 230 #define MVNETA_DA_FILT_OTH_MCAST 0x3500 231 #define MVNETA_DA_FILT_UCAST_BASE 0x3600 232 #define MVNETA_TXQ_BASE_ADDR_REG(q) (0x3c00 + ((q) << 2)) 233 #define MVNETA_TXQ_SIZE_REG(q) (0x3c20 + ((q) << 2)) 234 #define MVNETA_TXQ_SENT_THRESH_ALL_MASK 0x3fff0000 235 #define MVNETA_TXQ_SENT_THRESH_MASK(coal) ((coal) << 16) 236 #define MVNETA_TXQ_UPDATE_REG(q) (0x3c60 + ((q) << 2)) 237 #define MVNETA_TXQ_DEC_SENT_SHIFT 16 238 #define MVNETA_TXQ_DEC_SENT_MASK 0xff 239 #define MVNETA_TXQ_STATUS_REG(q) (0x3c40 + ((q) << 2)) 240 #define MVNETA_TXQ_SENT_DESC_SHIFT 16 241 #define MVNETA_TXQ_SENT_DESC_MASK 0x3fff0000 242 #define MVNETA_PORT_TX_RESET 0x3cf0 243 #define MVNETA_PORT_TX_DMA_RESET BIT(0) 244 #define MVNETA_TX_MTU 0x3e0c 245 #define MVNETA_TX_TOKEN_SIZE 0x3e14 246 #define MVNETA_TX_TOKEN_SIZE_MAX 0xffffffff 247 #define MVNETA_TXQ_TOKEN_SIZE_REG(q) (0x3e40 + ((q) << 2)) 248 #define MVNETA_TXQ_TOKEN_SIZE_MAX 0x7fffffff 249 250 #define MVNETA_LPI_CTRL_0 0x2cc0 251 #define MVNETA_LPI_CTRL_1 0x2cc4 252 #define MVNETA_LPI_REQUEST_ENABLE BIT(0) 253 #define MVNETA_LPI_CTRL_2 0x2cc8 254 #define MVNETA_LPI_STATUS 0x2ccc 255 256 #define MVNETA_CAUSE_TXQ_SENT_DESC_ALL_MASK 0xff 257 258 /* Descriptor ring Macros */ 259 #define MVNETA_QUEUE_NEXT_DESC(q, index) \ 260 (((index) < (q)->last_desc) ? ((index) + 1) : 0) 261 262 /* Various constants */ 263 264 /* Coalescing */ 265 #define MVNETA_TXDONE_COAL_PKTS 0 /* interrupt per packet */ 266 #define MVNETA_RX_COAL_PKTS 32 267 #define MVNETA_RX_COAL_USEC 100 268 269 /* The two bytes Marvell header. Either contains a special value used 270 * by Marvell switches when a specific hardware mode is enabled (not 271 * supported by this driver) or is filled automatically by zeroes on 272 * the RX side. Those two bytes being at the front of the Ethernet 273 * header, they allow to have the IP header aligned on a 4 bytes 274 * boundary automatically: the hardware skips those two bytes on its 275 * own. 276 */ 277 #define MVNETA_MH_SIZE 2 278 279 #define MVNETA_VLAN_TAG_LEN 4 280 281 #define MVNETA_TX_CSUM_DEF_SIZE 1600 282 #define MVNETA_TX_CSUM_MAX_SIZE 9800 283 #define MVNETA_ACC_MODE_EXT1 1 284 #define MVNETA_ACC_MODE_EXT2 2 285 286 #define MVNETA_MAX_DECODE_WIN 6 287 288 /* Timeout constants */ 289 #define MVNETA_TX_DISABLE_TIMEOUT_MSEC 1000 290 #define MVNETA_RX_DISABLE_TIMEOUT_MSEC 1000 291 #define MVNETA_TX_FIFO_EMPTY_TIMEOUT 10000 292 293 #define MVNETA_TX_MTU_MAX 0x3ffff 294 295 /* The RSS lookup table actually has 256 entries but we do not use 296 * them yet 297 */ 298 #define MVNETA_RSS_LU_TABLE_SIZE 1 299 300 /* Max number of Rx descriptors */ 301 #define MVNETA_MAX_RXD 512 302 303 /* Max number of Tx descriptors */ 304 #define MVNETA_MAX_TXD 1024 305 306 /* Max number of allowed TCP segments for software TSO */ 307 #define MVNETA_MAX_TSO_SEGS 100 308 309 #define MVNETA_MAX_SKB_DESCS (MVNETA_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS) 310 311 /* descriptor aligned size */ 312 #define MVNETA_DESC_ALIGNED_SIZE 32 313 314 /* Number of bytes to be taken into account by HW when putting incoming data 315 * to the buffers. It is needed in case NET_SKB_PAD exceeds maximum packet 316 * offset supported in MVNETA_RXQ_CONFIG_REG(q) registers. 317 */ 318 #define MVNETA_RX_PKT_OFFSET_CORRECTION 64 319 320 #define MVNETA_RX_PKT_SIZE(mtu) \ 321 ALIGN((mtu) + MVNETA_MH_SIZE + MVNETA_VLAN_TAG_LEN + \ 322 ETH_HLEN + ETH_FCS_LEN, \ 323 cache_line_size()) 324 325 #define IS_TSO_HEADER(txq, addr) \ 326 ((addr >= txq->tso_hdrs_phys) && \ 327 (addr < txq->tso_hdrs_phys + txq->size * TSO_HEADER_SIZE)) 328 329 #define MVNETA_RX_GET_BM_POOL_ID(rxd) \ 330 (((rxd)->status & MVNETA_RXD_BM_POOL_MASK) >> MVNETA_RXD_BM_POOL_SHIFT) 331 332 enum { 333 ETHTOOL_STAT_EEE_WAKEUP, 334 ETHTOOL_STAT_SKB_ALLOC_ERR, 335 ETHTOOL_STAT_REFILL_ERR, 336 ETHTOOL_MAX_STATS, 337 }; 338 339 struct mvneta_statistic { 340 unsigned short offset; 341 unsigned short type; 342 const char name[ETH_GSTRING_LEN]; 343 }; 344 345 #define T_REG_32 32 346 #define T_REG_64 64 347 #define T_SW 1 348 349 static const struct mvneta_statistic mvneta_statistics[] = { 350 { 0x3000, T_REG_64, "good_octets_received", }, 351 { 0x3010, T_REG_32, "good_frames_received", }, 352 { 0x3008, T_REG_32, "bad_octets_received", }, 353 { 0x3014, T_REG_32, "bad_frames_received", }, 354 { 0x3018, T_REG_32, "broadcast_frames_received", }, 355 { 0x301c, T_REG_32, "multicast_frames_received", }, 356 { 0x3050, T_REG_32, "unrec_mac_control_received", }, 357 { 0x3058, T_REG_32, "good_fc_received", }, 358 { 0x305c, T_REG_32, "bad_fc_received", }, 359 { 0x3060, T_REG_32, "undersize_received", }, 360 { 0x3064, T_REG_32, "fragments_received", }, 361 { 0x3068, T_REG_32, "oversize_received", }, 362 { 0x306c, T_REG_32, "jabber_received", }, 363 { 0x3070, T_REG_32, "mac_receive_error", }, 364 { 0x3074, T_REG_32, "bad_crc_event", }, 365 { 0x3078, T_REG_32, "collision", }, 366 { 0x307c, T_REG_32, "late_collision", }, 367 { 0x2484, T_REG_32, "rx_discard", }, 368 { 0x2488, T_REG_32, "rx_overrun", }, 369 { 0x3020, T_REG_32, "frames_64_octets", }, 370 { 0x3024, T_REG_32, "frames_65_to_127_octets", }, 371 { 0x3028, T_REG_32, "frames_128_to_255_octets", }, 372 { 0x302c, T_REG_32, "frames_256_to_511_octets", }, 373 { 0x3030, T_REG_32, "frames_512_to_1023_octets", }, 374 { 0x3034, T_REG_32, "frames_1024_to_max_octets", }, 375 { 0x3038, T_REG_64, "good_octets_sent", }, 376 { 0x3040, T_REG_32, "good_frames_sent", }, 377 { 0x3044, T_REG_32, "excessive_collision", }, 378 { 0x3048, T_REG_32, "multicast_frames_sent", }, 379 { 0x304c, T_REG_32, "broadcast_frames_sent", }, 380 { 0x3054, T_REG_32, "fc_sent", }, 381 { 0x300c, T_REG_32, "internal_mac_transmit_err", }, 382 { ETHTOOL_STAT_EEE_WAKEUP, T_SW, "eee_wakeup_errors", }, 383 { ETHTOOL_STAT_SKB_ALLOC_ERR, T_SW, "skb_alloc_errors", }, 384 { ETHTOOL_STAT_REFILL_ERR, T_SW, "refill_errors", }, 385 }; 386 387 struct mvneta_pcpu_stats { 388 struct u64_stats_sync syncp; 389 u64 rx_packets; 390 u64 rx_bytes; 391 u64 tx_packets; 392 u64 tx_bytes; 393 }; 394 395 struct mvneta_pcpu_port { 396 /* Pointer to the shared port */ 397 struct mvneta_port *pp; 398 399 /* Pointer to the CPU-local NAPI struct */ 400 struct napi_struct napi; 401 402 /* Cause of the previous interrupt */ 403 u32 cause_rx_tx; 404 }; 405 406 struct mvneta_port { 407 u8 id; 408 struct mvneta_pcpu_port __percpu *ports; 409 struct mvneta_pcpu_stats __percpu *stats; 410 411 int pkt_size; 412 void __iomem *base; 413 struct mvneta_rx_queue *rxqs; 414 struct mvneta_tx_queue *txqs; 415 struct net_device *dev; 416 struct hlist_node node_online; 417 struct hlist_node node_dead; 418 int rxq_def; 419 /* Protect the access to the percpu interrupt registers, 420 * ensuring that the configuration remains coherent. 421 */ 422 spinlock_t lock; 423 bool is_stopped; 424 425 u32 cause_rx_tx; 426 struct napi_struct napi; 427 428 /* Core clock */ 429 struct clk *clk; 430 /* AXI clock */ 431 struct clk *clk_bus; 432 u8 mcast_count[256]; 433 u16 tx_ring_size; 434 u16 rx_ring_size; 435 436 phy_interface_t phy_interface; 437 struct device_node *dn; 438 unsigned int tx_csum_limit; 439 struct phylink *phylink; 440 struct phy *comphy; 441 442 struct mvneta_bm *bm_priv; 443 struct mvneta_bm_pool *pool_long; 444 struct mvneta_bm_pool *pool_short; 445 int bm_win_id; 446 447 bool eee_enabled; 448 bool eee_active; 449 bool tx_lpi_enabled; 450 451 u64 ethtool_stats[ARRAY_SIZE(mvneta_statistics)]; 452 453 u32 indir[MVNETA_RSS_LU_TABLE_SIZE]; 454 455 /* Flags for special SoC configurations */ 456 bool neta_armada3700; 457 u16 rx_offset_correction; 458 const struct mbus_dram_target_info *dram_target_info; 459 }; 460 461 /* The mvneta_tx_desc and mvneta_rx_desc structures describe the 462 * layout of the transmit and reception DMA descriptors, and their 463 * layout is therefore defined by the hardware design 464 */ 465 466 #define MVNETA_TX_L3_OFF_SHIFT 0 467 #define MVNETA_TX_IP_HLEN_SHIFT 8 468 #define MVNETA_TX_L4_UDP BIT(16) 469 #define MVNETA_TX_L3_IP6 BIT(17) 470 #define MVNETA_TXD_IP_CSUM BIT(18) 471 #define MVNETA_TXD_Z_PAD BIT(19) 472 #define MVNETA_TXD_L_DESC BIT(20) 473 #define MVNETA_TXD_F_DESC BIT(21) 474 #define MVNETA_TXD_FLZ_DESC (MVNETA_TXD_Z_PAD | \ 475 MVNETA_TXD_L_DESC | \ 476 MVNETA_TXD_F_DESC) 477 #define MVNETA_TX_L4_CSUM_FULL BIT(30) 478 #define MVNETA_TX_L4_CSUM_NOT BIT(31) 479 480 #define MVNETA_RXD_ERR_CRC 0x0 481 #define MVNETA_RXD_BM_POOL_SHIFT 13 482 #define MVNETA_RXD_BM_POOL_MASK (BIT(13) | BIT(14)) 483 #define MVNETA_RXD_ERR_SUMMARY BIT(16) 484 #define MVNETA_RXD_ERR_OVERRUN BIT(17) 485 #define MVNETA_RXD_ERR_LEN BIT(18) 486 #define MVNETA_RXD_ERR_RESOURCE (BIT(17) | BIT(18)) 487 #define MVNETA_RXD_ERR_CODE_MASK (BIT(17) | BIT(18)) 488 #define MVNETA_RXD_L3_IP4 BIT(25) 489 #define MVNETA_RXD_LAST_DESC BIT(26) 490 #define MVNETA_RXD_FIRST_DESC BIT(27) 491 #define MVNETA_RXD_FIRST_LAST_DESC (MVNETA_RXD_FIRST_DESC | \ 492 MVNETA_RXD_LAST_DESC) 493 #define MVNETA_RXD_L4_CSUM_OK BIT(30) 494 495 #if defined(__LITTLE_ENDIAN) 496 struct mvneta_tx_desc { 497 u32 command; /* Options used by HW for packet transmitting.*/ 498 u16 reserved1; /* csum_l4 (for future use) */ 499 u16 data_size; /* Data size of transmitted packet in bytes */ 500 u32 buf_phys_addr; /* Physical addr of transmitted buffer */ 501 u32 reserved2; /* hw_cmd - (for future use, PMT) */ 502 u32 reserved3[4]; /* Reserved - (for future use) */ 503 }; 504 505 struct mvneta_rx_desc { 506 u32 status; /* Info about received packet */ 507 u16 reserved1; /* pnc_info - (for future use, PnC) */ 508 u16 data_size; /* Size of received packet in bytes */ 509 510 u32 buf_phys_addr; /* Physical address of the buffer */ 511 u32 reserved2; /* pnc_flow_id (for future use, PnC) */ 512 513 u32 buf_cookie; /* cookie for access to RX buffer in rx path */ 514 u16 reserved3; /* prefetch_cmd, for future use */ 515 u16 reserved4; /* csum_l4 - (for future use, PnC) */ 516 517 u32 reserved5; /* pnc_extra PnC (for future use, PnC) */ 518 u32 reserved6; /* hw_cmd (for future use, PnC and HWF) */ 519 }; 520 #else 521 struct mvneta_tx_desc { 522 u16 data_size; /* Data size of transmitted packet in bytes */ 523 u16 reserved1; /* csum_l4 (for future use) */ 524 u32 command; /* Options used by HW for packet transmitting.*/ 525 u32 reserved2; /* hw_cmd - (for future use, PMT) */ 526 u32 buf_phys_addr; /* Physical addr of transmitted buffer */ 527 u32 reserved3[4]; /* Reserved - (for future use) */ 528 }; 529 530 struct mvneta_rx_desc { 531 u16 data_size; /* Size of received packet in bytes */ 532 u16 reserved1; /* pnc_info - (for future use, PnC) */ 533 u32 status; /* Info about received packet */ 534 535 u32 reserved2; /* pnc_flow_id (for future use, PnC) */ 536 u32 buf_phys_addr; /* Physical address of the buffer */ 537 538 u16 reserved4; /* csum_l4 - (for future use, PnC) */ 539 u16 reserved3; /* prefetch_cmd, for future use */ 540 u32 buf_cookie; /* cookie for access to RX buffer in rx path */ 541 542 u32 reserved5; /* pnc_extra PnC (for future use, PnC) */ 543 u32 reserved6; /* hw_cmd (for future use, PnC and HWF) */ 544 }; 545 #endif 546 547 struct mvneta_tx_queue { 548 /* Number of this TX queue, in the range 0-7 */ 549 u8 id; 550 551 /* Number of TX DMA descriptors in the descriptor ring */ 552 int size; 553 554 /* Number of currently used TX DMA descriptor in the 555 * descriptor ring 556 */ 557 int count; 558 int pending; 559 int tx_stop_threshold; 560 int tx_wake_threshold; 561 562 /* Array of transmitted skb */ 563 struct sk_buff **tx_skb; 564 565 /* Index of last TX DMA descriptor that was inserted */ 566 int txq_put_index; 567 568 /* Index of the TX DMA descriptor to be cleaned up */ 569 int txq_get_index; 570 571 u32 done_pkts_coal; 572 573 /* Virtual address of the TX DMA descriptors array */ 574 struct mvneta_tx_desc *descs; 575 576 /* DMA address of the TX DMA descriptors array */ 577 dma_addr_t descs_phys; 578 579 /* Index of the last TX DMA descriptor */ 580 int last_desc; 581 582 /* Index of the next TX DMA descriptor to process */ 583 int next_desc_to_proc; 584 585 /* DMA buffers for TSO headers */ 586 char *tso_hdrs; 587 588 /* DMA address of TSO headers */ 589 dma_addr_t tso_hdrs_phys; 590 591 /* Affinity mask for CPUs*/ 592 cpumask_t affinity_mask; 593 }; 594 595 struct mvneta_rx_queue { 596 /* rx queue number, in the range 0-7 */ 597 u8 id; 598 599 /* num of rx descriptors in the rx descriptor ring */ 600 int size; 601 602 u32 pkts_coal; 603 u32 time_coal; 604 605 /* Virtual address of the RX buffer */ 606 void **buf_virt_addr; 607 608 /* Virtual address of the RX DMA descriptors array */ 609 struct mvneta_rx_desc *descs; 610 611 /* DMA address of the RX DMA descriptors array */ 612 dma_addr_t descs_phys; 613 614 /* Index of the last RX DMA descriptor */ 615 int last_desc; 616 617 /* Index of the next RX DMA descriptor to process */ 618 int next_desc_to_proc; 619 620 /* Index of first RX DMA descriptor to refill */ 621 int first_to_refill; 622 u32 refill_num; 623 624 /* pointer to uncomplete skb buffer */ 625 struct sk_buff *skb; 626 int left_size; 627 628 /* error counters */ 629 u32 skb_alloc_err; 630 u32 refill_err; 631 }; 632 633 static enum cpuhp_state online_hpstate; 634 /* The hardware supports eight (8) rx queues, but we are only allowing 635 * the first one to be used. Therefore, let's just allocate one queue. 636 */ 637 static int rxq_number = 8; 638 static int txq_number = 8; 639 640 static int rxq_def; 641 642 static int rx_copybreak __read_mostly = 256; 643 static int rx_header_size __read_mostly = 128; 644 645 /* HW BM need that each port be identify by a unique ID */ 646 static int global_port_id; 647 648 #define MVNETA_DRIVER_NAME "mvneta" 649 #define MVNETA_DRIVER_VERSION "1.0" 650 651 /* Utility/helper methods */ 652 653 /* Write helper method */ 654 static void mvreg_write(struct mvneta_port *pp, u32 offset, u32 data) 655 { 656 writel(data, pp->base + offset); 657 } 658 659 /* Read helper method */ 660 static u32 mvreg_read(struct mvneta_port *pp, u32 offset) 661 { 662 return readl(pp->base + offset); 663 } 664 665 /* Increment txq get counter */ 666 static void mvneta_txq_inc_get(struct mvneta_tx_queue *txq) 667 { 668 txq->txq_get_index++; 669 if (txq->txq_get_index == txq->size) 670 txq->txq_get_index = 0; 671 } 672 673 /* Increment txq put counter */ 674 static void mvneta_txq_inc_put(struct mvneta_tx_queue *txq) 675 { 676 txq->txq_put_index++; 677 if (txq->txq_put_index == txq->size) 678 txq->txq_put_index = 0; 679 } 680 681 682 /* Clear all MIB counters */ 683 static void mvneta_mib_counters_clear(struct mvneta_port *pp) 684 { 685 int i; 686 u32 dummy; 687 688 /* Perform dummy reads from MIB counters */ 689 for (i = 0; i < MVNETA_MIB_LATE_COLLISION; i += 4) 690 dummy = mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i)); 691 dummy = mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT); 692 dummy = mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT); 693 } 694 695 /* Get System Network Statistics */ 696 static void 697 mvneta_get_stats64(struct net_device *dev, 698 struct rtnl_link_stats64 *stats) 699 { 700 struct mvneta_port *pp = netdev_priv(dev); 701 unsigned int start; 702 int cpu; 703 704 for_each_possible_cpu(cpu) { 705 struct mvneta_pcpu_stats *cpu_stats; 706 u64 rx_packets; 707 u64 rx_bytes; 708 u64 tx_packets; 709 u64 tx_bytes; 710 711 cpu_stats = per_cpu_ptr(pp->stats, cpu); 712 do { 713 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); 714 rx_packets = cpu_stats->rx_packets; 715 rx_bytes = cpu_stats->rx_bytes; 716 tx_packets = cpu_stats->tx_packets; 717 tx_bytes = cpu_stats->tx_bytes; 718 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); 719 720 stats->rx_packets += rx_packets; 721 stats->rx_bytes += rx_bytes; 722 stats->tx_packets += tx_packets; 723 stats->tx_bytes += tx_bytes; 724 } 725 726 stats->rx_errors = dev->stats.rx_errors; 727 stats->rx_dropped = dev->stats.rx_dropped; 728 729 stats->tx_dropped = dev->stats.tx_dropped; 730 } 731 732 /* Rx descriptors helper methods */ 733 734 /* Checks whether the RX descriptor having this status is both the first 735 * and the last descriptor for the RX packet. Each RX packet is currently 736 * received through a single RX descriptor, so not having each RX 737 * descriptor with its first and last bits set is an error 738 */ 739 static int mvneta_rxq_desc_is_first_last(u32 status) 740 { 741 return (status & MVNETA_RXD_FIRST_LAST_DESC) == 742 MVNETA_RXD_FIRST_LAST_DESC; 743 } 744 745 /* Add number of descriptors ready to receive new packets */ 746 static void mvneta_rxq_non_occup_desc_add(struct mvneta_port *pp, 747 struct mvneta_rx_queue *rxq, 748 int ndescs) 749 { 750 /* Only MVNETA_RXQ_ADD_NON_OCCUPIED_MAX (255) descriptors can 751 * be added at once 752 */ 753 while (ndescs > MVNETA_RXQ_ADD_NON_OCCUPIED_MAX) { 754 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), 755 (MVNETA_RXQ_ADD_NON_OCCUPIED_MAX << 756 MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT)); 757 ndescs -= MVNETA_RXQ_ADD_NON_OCCUPIED_MAX; 758 } 759 760 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), 761 (ndescs << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT)); 762 } 763 764 /* Get number of RX descriptors occupied by received packets */ 765 static int mvneta_rxq_busy_desc_num_get(struct mvneta_port *pp, 766 struct mvneta_rx_queue *rxq) 767 { 768 u32 val; 769 770 val = mvreg_read(pp, MVNETA_RXQ_STATUS_REG(rxq->id)); 771 return val & MVNETA_RXQ_OCCUPIED_ALL_MASK; 772 } 773 774 /* Update num of rx desc called upon return from rx path or 775 * from mvneta_rxq_drop_pkts(). 776 */ 777 static void mvneta_rxq_desc_num_update(struct mvneta_port *pp, 778 struct mvneta_rx_queue *rxq, 779 int rx_done, int rx_filled) 780 { 781 u32 val; 782 783 if ((rx_done <= 0xff) && (rx_filled <= 0xff)) { 784 val = rx_done | 785 (rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT); 786 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val); 787 return; 788 } 789 790 /* Only 255 descriptors can be added at once */ 791 while ((rx_done > 0) || (rx_filled > 0)) { 792 if (rx_done <= 0xff) { 793 val = rx_done; 794 rx_done = 0; 795 } else { 796 val = 0xff; 797 rx_done -= 0xff; 798 } 799 if (rx_filled <= 0xff) { 800 val |= rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT; 801 rx_filled = 0; 802 } else { 803 val |= 0xff << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT; 804 rx_filled -= 0xff; 805 } 806 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val); 807 } 808 } 809 810 /* Get pointer to next RX descriptor to be processed by SW */ 811 static struct mvneta_rx_desc * 812 mvneta_rxq_next_desc_get(struct mvneta_rx_queue *rxq) 813 { 814 int rx_desc = rxq->next_desc_to_proc; 815 816 rxq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(rxq, rx_desc); 817 prefetch(rxq->descs + rxq->next_desc_to_proc); 818 return rxq->descs + rx_desc; 819 } 820 821 /* Change maximum receive size of the port. */ 822 static void mvneta_max_rx_size_set(struct mvneta_port *pp, int max_rx_size) 823 { 824 u32 val; 825 826 val = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 827 val &= ~MVNETA_GMAC_MAX_RX_SIZE_MASK; 828 val |= ((max_rx_size - MVNETA_MH_SIZE) / 2) << 829 MVNETA_GMAC_MAX_RX_SIZE_SHIFT; 830 mvreg_write(pp, MVNETA_GMAC_CTRL_0, val); 831 } 832 833 834 /* Set rx queue offset */ 835 static void mvneta_rxq_offset_set(struct mvneta_port *pp, 836 struct mvneta_rx_queue *rxq, 837 int offset) 838 { 839 u32 val; 840 841 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 842 val &= ~MVNETA_RXQ_PKT_OFFSET_ALL_MASK; 843 844 /* Offset is in */ 845 val |= MVNETA_RXQ_PKT_OFFSET_MASK(offset >> 3); 846 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 847 } 848 849 850 /* Tx descriptors helper methods */ 851 852 /* Update HW with number of TX descriptors to be sent */ 853 static void mvneta_txq_pend_desc_add(struct mvneta_port *pp, 854 struct mvneta_tx_queue *txq, 855 int pend_desc) 856 { 857 u32 val; 858 859 pend_desc += txq->pending; 860 861 /* Only 255 Tx descriptors can be added at once */ 862 do { 863 val = min(pend_desc, 255); 864 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val); 865 pend_desc -= val; 866 } while (pend_desc > 0); 867 txq->pending = 0; 868 } 869 870 /* Get pointer to next TX descriptor to be processed (send) by HW */ 871 static struct mvneta_tx_desc * 872 mvneta_txq_next_desc_get(struct mvneta_tx_queue *txq) 873 { 874 int tx_desc = txq->next_desc_to_proc; 875 876 txq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(txq, tx_desc); 877 return txq->descs + tx_desc; 878 } 879 880 /* Release the last allocated TX descriptor. Useful to handle DMA 881 * mapping failures in the TX path. 882 */ 883 static void mvneta_txq_desc_put(struct mvneta_tx_queue *txq) 884 { 885 if (txq->next_desc_to_proc == 0) 886 txq->next_desc_to_proc = txq->last_desc - 1; 887 else 888 txq->next_desc_to_proc--; 889 } 890 891 /* Set rxq buf size */ 892 static void mvneta_rxq_buf_size_set(struct mvneta_port *pp, 893 struct mvneta_rx_queue *rxq, 894 int buf_size) 895 { 896 u32 val; 897 898 val = mvreg_read(pp, MVNETA_RXQ_SIZE_REG(rxq->id)); 899 900 val &= ~MVNETA_RXQ_BUF_SIZE_MASK; 901 val |= ((buf_size >> 3) << MVNETA_RXQ_BUF_SIZE_SHIFT); 902 903 mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), val); 904 } 905 906 /* Disable buffer management (BM) */ 907 static void mvneta_rxq_bm_disable(struct mvneta_port *pp, 908 struct mvneta_rx_queue *rxq) 909 { 910 u32 val; 911 912 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 913 val &= ~MVNETA_RXQ_HW_BUF_ALLOC; 914 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 915 } 916 917 /* Enable buffer management (BM) */ 918 static void mvneta_rxq_bm_enable(struct mvneta_port *pp, 919 struct mvneta_rx_queue *rxq) 920 { 921 u32 val; 922 923 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 924 val |= MVNETA_RXQ_HW_BUF_ALLOC; 925 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 926 } 927 928 /* Notify HW about port's assignment of pool for bigger packets */ 929 static void mvneta_rxq_long_pool_set(struct mvneta_port *pp, 930 struct mvneta_rx_queue *rxq) 931 { 932 u32 val; 933 934 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 935 val &= ~MVNETA_RXQ_LONG_POOL_ID_MASK; 936 val |= (pp->pool_long->id << MVNETA_RXQ_LONG_POOL_ID_SHIFT); 937 938 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 939 } 940 941 /* Notify HW about port's assignment of pool for smaller packets */ 942 static void mvneta_rxq_short_pool_set(struct mvneta_port *pp, 943 struct mvneta_rx_queue *rxq) 944 { 945 u32 val; 946 947 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 948 val &= ~MVNETA_RXQ_SHORT_POOL_ID_MASK; 949 val |= (pp->pool_short->id << MVNETA_RXQ_SHORT_POOL_ID_SHIFT); 950 951 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 952 } 953 954 /* Set port's receive buffer size for assigned BM pool */ 955 static inline void mvneta_bm_pool_bufsize_set(struct mvneta_port *pp, 956 int buf_size, 957 u8 pool_id) 958 { 959 u32 val; 960 961 if (!IS_ALIGNED(buf_size, 8)) { 962 dev_warn(pp->dev->dev.parent, 963 "illegal buf_size value %d, round to %d\n", 964 buf_size, ALIGN(buf_size, 8)); 965 buf_size = ALIGN(buf_size, 8); 966 } 967 968 val = mvreg_read(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id)); 969 val |= buf_size & MVNETA_PORT_POOL_BUFFER_SZ_MASK; 970 mvreg_write(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id), val); 971 } 972 973 /* Configure MBUS window in order to enable access BM internal SRAM */ 974 static int mvneta_mbus_io_win_set(struct mvneta_port *pp, u32 base, u32 wsize, 975 u8 target, u8 attr) 976 { 977 u32 win_enable, win_protect; 978 int i; 979 980 win_enable = mvreg_read(pp, MVNETA_BASE_ADDR_ENABLE); 981 982 if (pp->bm_win_id < 0) { 983 /* Find first not occupied window */ 984 for (i = 0; i < MVNETA_MAX_DECODE_WIN; i++) { 985 if (win_enable & (1 << i)) { 986 pp->bm_win_id = i; 987 break; 988 } 989 } 990 if (i == MVNETA_MAX_DECODE_WIN) 991 return -ENOMEM; 992 } else { 993 i = pp->bm_win_id; 994 } 995 996 mvreg_write(pp, MVNETA_WIN_BASE(i), 0); 997 mvreg_write(pp, MVNETA_WIN_SIZE(i), 0); 998 999 if (i < 4) 1000 mvreg_write(pp, MVNETA_WIN_REMAP(i), 0); 1001 1002 mvreg_write(pp, MVNETA_WIN_BASE(i), (base & 0xffff0000) | 1003 (attr << 8) | target); 1004 1005 mvreg_write(pp, MVNETA_WIN_SIZE(i), (wsize - 1) & 0xffff0000); 1006 1007 win_protect = mvreg_read(pp, MVNETA_ACCESS_PROTECT_ENABLE); 1008 win_protect |= 3 << (2 * i); 1009 mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect); 1010 1011 win_enable &= ~(1 << i); 1012 mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable); 1013 1014 return 0; 1015 } 1016 1017 static int mvneta_bm_port_mbus_init(struct mvneta_port *pp) 1018 { 1019 u32 wsize; 1020 u8 target, attr; 1021 int err; 1022 1023 /* Get BM window information */ 1024 err = mvebu_mbus_get_io_win_info(pp->bm_priv->bppi_phys_addr, &wsize, 1025 &target, &attr); 1026 if (err < 0) 1027 return err; 1028 1029 pp->bm_win_id = -1; 1030 1031 /* Open NETA -> BM window */ 1032 err = mvneta_mbus_io_win_set(pp, pp->bm_priv->bppi_phys_addr, wsize, 1033 target, attr); 1034 if (err < 0) { 1035 netdev_info(pp->dev, "fail to configure mbus window to BM\n"); 1036 return err; 1037 } 1038 return 0; 1039 } 1040 1041 /* Assign and initialize pools for port. In case of fail 1042 * buffer manager will remain disabled for current port. 1043 */ 1044 static int mvneta_bm_port_init(struct platform_device *pdev, 1045 struct mvneta_port *pp) 1046 { 1047 struct device_node *dn = pdev->dev.of_node; 1048 u32 long_pool_id, short_pool_id; 1049 1050 if (!pp->neta_armada3700) { 1051 int ret; 1052 1053 ret = mvneta_bm_port_mbus_init(pp); 1054 if (ret) 1055 return ret; 1056 } 1057 1058 if (of_property_read_u32(dn, "bm,pool-long", &long_pool_id)) { 1059 netdev_info(pp->dev, "missing long pool id\n"); 1060 return -EINVAL; 1061 } 1062 1063 /* Create port's long pool depending on mtu */ 1064 pp->pool_long = mvneta_bm_pool_use(pp->bm_priv, long_pool_id, 1065 MVNETA_BM_LONG, pp->id, 1066 MVNETA_RX_PKT_SIZE(pp->dev->mtu)); 1067 if (!pp->pool_long) { 1068 netdev_info(pp->dev, "fail to obtain long pool for port\n"); 1069 return -ENOMEM; 1070 } 1071 1072 pp->pool_long->port_map |= 1 << pp->id; 1073 1074 mvneta_bm_pool_bufsize_set(pp, pp->pool_long->buf_size, 1075 pp->pool_long->id); 1076 1077 /* If short pool id is not defined, assume using single pool */ 1078 if (of_property_read_u32(dn, "bm,pool-short", &short_pool_id)) 1079 short_pool_id = long_pool_id; 1080 1081 /* Create port's short pool */ 1082 pp->pool_short = mvneta_bm_pool_use(pp->bm_priv, short_pool_id, 1083 MVNETA_BM_SHORT, pp->id, 1084 MVNETA_BM_SHORT_PKT_SIZE); 1085 if (!pp->pool_short) { 1086 netdev_info(pp->dev, "fail to obtain short pool for port\n"); 1087 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 1088 return -ENOMEM; 1089 } 1090 1091 if (short_pool_id != long_pool_id) { 1092 pp->pool_short->port_map |= 1 << pp->id; 1093 mvneta_bm_pool_bufsize_set(pp, pp->pool_short->buf_size, 1094 pp->pool_short->id); 1095 } 1096 1097 return 0; 1098 } 1099 1100 /* Update settings of a pool for bigger packets */ 1101 static void mvneta_bm_update_mtu(struct mvneta_port *pp, int mtu) 1102 { 1103 struct mvneta_bm_pool *bm_pool = pp->pool_long; 1104 struct hwbm_pool *hwbm_pool = &bm_pool->hwbm_pool; 1105 int num; 1106 1107 /* Release all buffers from long pool */ 1108 mvneta_bm_bufs_free(pp->bm_priv, bm_pool, 1 << pp->id); 1109 if (hwbm_pool->buf_num) { 1110 WARN(1, "cannot free all buffers in pool %d\n", 1111 bm_pool->id); 1112 goto bm_mtu_err; 1113 } 1114 1115 bm_pool->pkt_size = MVNETA_RX_PKT_SIZE(mtu); 1116 bm_pool->buf_size = MVNETA_RX_BUF_SIZE(bm_pool->pkt_size); 1117 hwbm_pool->frag_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1118 SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(bm_pool->pkt_size)); 1119 1120 /* Fill entire long pool */ 1121 num = hwbm_pool_add(hwbm_pool, hwbm_pool->size, GFP_ATOMIC); 1122 if (num != hwbm_pool->size) { 1123 WARN(1, "pool %d: %d of %d allocated\n", 1124 bm_pool->id, num, hwbm_pool->size); 1125 goto bm_mtu_err; 1126 } 1127 mvneta_bm_pool_bufsize_set(pp, bm_pool->buf_size, bm_pool->id); 1128 1129 return; 1130 1131 bm_mtu_err: 1132 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 1133 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 1 << pp->id); 1134 1135 pp->bm_priv = NULL; 1136 mvreg_write(pp, MVNETA_ACC_MODE, MVNETA_ACC_MODE_EXT1); 1137 netdev_info(pp->dev, "fail to update MTU, fall back to software BM\n"); 1138 } 1139 1140 /* Start the Ethernet port RX and TX activity */ 1141 static void mvneta_port_up(struct mvneta_port *pp) 1142 { 1143 int queue; 1144 u32 q_map; 1145 1146 /* Enable all initialized TXs. */ 1147 q_map = 0; 1148 for (queue = 0; queue < txq_number; queue++) { 1149 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 1150 if (txq->descs) 1151 q_map |= (1 << queue); 1152 } 1153 mvreg_write(pp, MVNETA_TXQ_CMD, q_map); 1154 1155 q_map = 0; 1156 /* Enable all initialized RXQs. */ 1157 for (queue = 0; queue < rxq_number; queue++) { 1158 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 1159 1160 if (rxq->descs) 1161 q_map |= (1 << queue); 1162 } 1163 mvreg_write(pp, MVNETA_RXQ_CMD, q_map); 1164 } 1165 1166 /* Stop the Ethernet port activity */ 1167 static void mvneta_port_down(struct mvneta_port *pp) 1168 { 1169 u32 val; 1170 int count; 1171 1172 /* Stop Rx port activity. Check port Rx activity. */ 1173 val = mvreg_read(pp, MVNETA_RXQ_CMD) & MVNETA_RXQ_ENABLE_MASK; 1174 1175 /* Issue stop command for active channels only */ 1176 if (val != 0) 1177 mvreg_write(pp, MVNETA_RXQ_CMD, 1178 val << MVNETA_RXQ_DISABLE_SHIFT); 1179 1180 /* Wait for all Rx activity to terminate. */ 1181 count = 0; 1182 do { 1183 if (count++ >= MVNETA_RX_DISABLE_TIMEOUT_MSEC) { 1184 netdev_warn(pp->dev, 1185 "TIMEOUT for RX stopped ! rx_queue_cmd: 0x%08x\n", 1186 val); 1187 break; 1188 } 1189 mdelay(1); 1190 1191 val = mvreg_read(pp, MVNETA_RXQ_CMD); 1192 } while (val & MVNETA_RXQ_ENABLE_MASK); 1193 1194 /* Stop Tx port activity. Check port Tx activity. Issue stop 1195 * command for active channels only 1196 */ 1197 val = (mvreg_read(pp, MVNETA_TXQ_CMD)) & MVNETA_TXQ_ENABLE_MASK; 1198 1199 if (val != 0) 1200 mvreg_write(pp, MVNETA_TXQ_CMD, 1201 (val << MVNETA_TXQ_DISABLE_SHIFT)); 1202 1203 /* Wait for all Tx activity to terminate. */ 1204 count = 0; 1205 do { 1206 if (count++ >= MVNETA_TX_DISABLE_TIMEOUT_MSEC) { 1207 netdev_warn(pp->dev, 1208 "TIMEOUT for TX stopped status=0x%08x\n", 1209 val); 1210 break; 1211 } 1212 mdelay(1); 1213 1214 /* Check TX Command reg that all Txqs are stopped */ 1215 val = mvreg_read(pp, MVNETA_TXQ_CMD); 1216 1217 } while (val & MVNETA_TXQ_ENABLE_MASK); 1218 1219 /* Double check to verify that TX FIFO is empty */ 1220 count = 0; 1221 do { 1222 if (count++ >= MVNETA_TX_FIFO_EMPTY_TIMEOUT) { 1223 netdev_warn(pp->dev, 1224 "TX FIFO empty timeout status=0x%08x\n", 1225 val); 1226 break; 1227 } 1228 mdelay(1); 1229 1230 val = mvreg_read(pp, MVNETA_PORT_STATUS); 1231 } while (!(val & MVNETA_TX_FIFO_EMPTY) && 1232 (val & MVNETA_TX_IN_PRGRS)); 1233 1234 udelay(200); 1235 } 1236 1237 /* Enable the port by setting the port enable bit of the MAC control register */ 1238 static void mvneta_port_enable(struct mvneta_port *pp) 1239 { 1240 u32 val; 1241 1242 /* Enable port */ 1243 val = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 1244 val |= MVNETA_GMAC0_PORT_ENABLE; 1245 mvreg_write(pp, MVNETA_GMAC_CTRL_0, val); 1246 } 1247 1248 /* Disable the port and wait for about 200 usec before retuning */ 1249 static void mvneta_port_disable(struct mvneta_port *pp) 1250 { 1251 u32 val; 1252 1253 /* Reset the Enable bit in the Serial Control Register */ 1254 val = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 1255 val &= ~MVNETA_GMAC0_PORT_ENABLE; 1256 mvreg_write(pp, MVNETA_GMAC_CTRL_0, val); 1257 1258 udelay(200); 1259 } 1260 1261 /* Multicast tables methods */ 1262 1263 /* Set all entries in Unicast MAC Table; queue==-1 means reject all */ 1264 static void mvneta_set_ucast_table(struct mvneta_port *pp, int queue) 1265 { 1266 int offset; 1267 u32 val; 1268 1269 if (queue == -1) { 1270 val = 0; 1271 } else { 1272 val = 0x1 | (queue << 1); 1273 val |= (val << 24) | (val << 16) | (val << 8); 1274 } 1275 1276 for (offset = 0; offset <= 0xc; offset += 4) 1277 mvreg_write(pp, MVNETA_DA_FILT_UCAST_BASE + offset, val); 1278 } 1279 1280 /* Set all entries in Special Multicast MAC Table; queue==-1 means reject all */ 1281 static void mvneta_set_special_mcast_table(struct mvneta_port *pp, int queue) 1282 { 1283 int offset; 1284 u32 val; 1285 1286 if (queue == -1) { 1287 val = 0; 1288 } else { 1289 val = 0x1 | (queue << 1); 1290 val |= (val << 24) | (val << 16) | (val << 8); 1291 } 1292 1293 for (offset = 0; offset <= 0xfc; offset += 4) 1294 mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + offset, val); 1295 1296 } 1297 1298 /* Set all entries in Other Multicast MAC Table. queue==-1 means reject all */ 1299 static void mvneta_set_other_mcast_table(struct mvneta_port *pp, int queue) 1300 { 1301 int offset; 1302 u32 val; 1303 1304 if (queue == -1) { 1305 memset(pp->mcast_count, 0, sizeof(pp->mcast_count)); 1306 val = 0; 1307 } else { 1308 memset(pp->mcast_count, 1, sizeof(pp->mcast_count)); 1309 val = 0x1 | (queue << 1); 1310 val |= (val << 24) | (val << 16) | (val << 8); 1311 } 1312 1313 for (offset = 0; offset <= 0xfc; offset += 4) 1314 mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + offset, val); 1315 } 1316 1317 static void mvneta_percpu_unmask_interrupt(void *arg) 1318 { 1319 struct mvneta_port *pp = arg; 1320 1321 /* All the queue are unmasked, but actually only the ones 1322 * mapped to this CPU will be unmasked 1323 */ 1324 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 1325 MVNETA_RX_INTR_MASK_ALL | 1326 MVNETA_TX_INTR_MASK_ALL | 1327 MVNETA_MISCINTR_INTR_MASK); 1328 } 1329 1330 static void mvneta_percpu_mask_interrupt(void *arg) 1331 { 1332 struct mvneta_port *pp = arg; 1333 1334 /* All the queue are masked, but actually only the ones 1335 * mapped to this CPU will be masked 1336 */ 1337 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0); 1338 mvreg_write(pp, MVNETA_INTR_OLD_MASK, 0); 1339 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 0); 1340 } 1341 1342 static void mvneta_percpu_clear_intr_cause(void *arg) 1343 { 1344 struct mvneta_port *pp = arg; 1345 1346 /* All the queue are cleared, but actually only the ones 1347 * mapped to this CPU will be cleared 1348 */ 1349 mvreg_write(pp, MVNETA_INTR_NEW_CAUSE, 0); 1350 mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0); 1351 mvreg_write(pp, MVNETA_INTR_OLD_CAUSE, 0); 1352 } 1353 1354 /* This method sets defaults to the NETA port: 1355 * Clears interrupt Cause and Mask registers. 1356 * Clears all MAC tables. 1357 * Sets defaults to all registers. 1358 * Resets RX and TX descriptor rings. 1359 * Resets PHY. 1360 * This method can be called after mvneta_port_down() to return the port 1361 * settings to defaults. 1362 */ 1363 static void mvneta_defaults_set(struct mvneta_port *pp) 1364 { 1365 int cpu; 1366 int queue; 1367 u32 val; 1368 int max_cpu = num_present_cpus(); 1369 1370 /* Clear all Cause registers */ 1371 on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true); 1372 1373 /* Mask all interrupts */ 1374 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 1375 mvreg_write(pp, MVNETA_INTR_ENABLE, 0); 1376 1377 /* Enable MBUS Retry bit16 */ 1378 mvreg_write(pp, MVNETA_MBUS_RETRY, 0x20); 1379 1380 /* Set CPU queue access map. CPUs are assigned to the RX and 1381 * TX queues modulo their number. If there is only one TX 1382 * queue then it is assigned to the CPU associated to the 1383 * default RX queue. 1384 */ 1385 for_each_present_cpu(cpu) { 1386 int rxq_map = 0, txq_map = 0; 1387 int rxq, txq; 1388 if (!pp->neta_armada3700) { 1389 for (rxq = 0; rxq < rxq_number; rxq++) 1390 if ((rxq % max_cpu) == cpu) 1391 rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq); 1392 1393 for (txq = 0; txq < txq_number; txq++) 1394 if ((txq % max_cpu) == cpu) 1395 txq_map |= MVNETA_CPU_TXQ_ACCESS(txq); 1396 1397 /* With only one TX queue we configure a special case 1398 * which will allow to get all the irq on a single 1399 * CPU 1400 */ 1401 if (txq_number == 1) 1402 txq_map = (cpu == pp->rxq_def) ? 1403 MVNETA_CPU_TXQ_ACCESS(1) : 0; 1404 1405 } else { 1406 txq_map = MVNETA_CPU_TXQ_ACCESS_ALL_MASK; 1407 rxq_map = MVNETA_CPU_RXQ_ACCESS_ALL_MASK; 1408 } 1409 1410 mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map); 1411 } 1412 1413 /* Reset RX and TX DMAs */ 1414 mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET); 1415 mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET); 1416 1417 /* Disable Legacy WRR, Disable EJP, Release from reset */ 1418 mvreg_write(pp, MVNETA_TXQ_CMD_1, 0); 1419 for (queue = 0; queue < txq_number; queue++) { 1420 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(queue), 0); 1421 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(queue), 0); 1422 } 1423 1424 mvreg_write(pp, MVNETA_PORT_TX_RESET, 0); 1425 mvreg_write(pp, MVNETA_PORT_RX_RESET, 0); 1426 1427 /* Set Port Acceleration Mode */ 1428 if (pp->bm_priv) 1429 /* HW buffer management + legacy parser */ 1430 val = MVNETA_ACC_MODE_EXT2; 1431 else 1432 /* SW buffer management + legacy parser */ 1433 val = MVNETA_ACC_MODE_EXT1; 1434 mvreg_write(pp, MVNETA_ACC_MODE, val); 1435 1436 if (pp->bm_priv) 1437 mvreg_write(pp, MVNETA_BM_ADDRESS, pp->bm_priv->bppi_phys_addr); 1438 1439 /* Update val of portCfg register accordingly with all RxQueue types */ 1440 val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def); 1441 mvreg_write(pp, MVNETA_PORT_CONFIG, val); 1442 1443 val = 0; 1444 mvreg_write(pp, MVNETA_PORT_CONFIG_EXTEND, val); 1445 mvreg_write(pp, MVNETA_RX_MIN_FRAME_SIZE, 64); 1446 1447 /* Build PORT_SDMA_CONFIG_REG */ 1448 val = 0; 1449 1450 /* Default burst size */ 1451 val |= MVNETA_TX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16); 1452 val |= MVNETA_RX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16); 1453 val |= MVNETA_RX_NO_DATA_SWAP | MVNETA_TX_NO_DATA_SWAP; 1454 1455 #if defined(__BIG_ENDIAN) 1456 val |= MVNETA_DESC_SWAP; 1457 #endif 1458 1459 /* Assign port SDMA configuration */ 1460 mvreg_write(pp, MVNETA_SDMA_CONFIG, val); 1461 1462 /* Disable PHY polling in hardware, since we're using the 1463 * kernel phylib to do this. 1464 */ 1465 val = mvreg_read(pp, MVNETA_UNIT_CONTROL); 1466 val &= ~MVNETA_PHY_POLLING_ENABLE; 1467 mvreg_write(pp, MVNETA_UNIT_CONTROL, val); 1468 1469 mvneta_set_ucast_table(pp, -1); 1470 mvneta_set_special_mcast_table(pp, -1); 1471 mvneta_set_other_mcast_table(pp, -1); 1472 1473 /* Set port interrupt enable register - default enable all */ 1474 mvreg_write(pp, MVNETA_INTR_ENABLE, 1475 (MVNETA_RXQ_INTR_ENABLE_ALL_MASK 1476 | MVNETA_TXQ_INTR_ENABLE_ALL_MASK)); 1477 1478 mvneta_mib_counters_clear(pp); 1479 } 1480 1481 /* Set max sizes for tx queues */ 1482 static void mvneta_txq_max_tx_size_set(struct mvneta_port *pp, int max_tx_size) 1483 1484 { 1485 u32 val, size, mtu; 1486 int queue; 1487 1488 mtu = max_tx_size * 8; 1489 if (mtu > MVNETA_TX_MTU_MAX) 1490 mtu = MVNETA_TX_MTU_MAX; 1491 1492 /* Set MTU */ 1493 val = mvreg_read(pp, MVNETA_TX_MTU); 1494 val &= ~MVNETA_TX_MTU_MAX; 1495 val |= mtu; 1496 mvreg_write(pp, MVNETA_TX_MTU, val); 1497 1498 /* TX token size and all TXQs token size must be larger that MTU */ 1499 val = mvreg_read(pp, MVNETA_TX_TOKEN_SIZE); 1500 1501 size = val & MVNETA_TX_TOKEN_SIZE_MAX; 1502 if (size < mtu) { 1503 size = mtu; 1504 val &= ~MVNETA_TX_TOKEN_SIZE_MAX; 1505 val |= size; 1506 mvreg_write(pp, MVNETA_TX_TOKEN_SIZE, val); 1507 } 1508 for (queue = 0; queue < txq_number; queue++) { 1509 val = mvreg_read(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue)); 1510 1511 size = val & MVNETA_TXQ_TOKEN_SIZE_MAX; 1512 if (size < mtu) { 1513 size = mtu; 1514 val &= ~MVNETA_TXQ_TOKEN_SIZE_MAX; 1515 val |= size; 1516 mvreg_write(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue), val); 1517 } 1518 } 1519 } 1520 1521 /* Set unicast address */ 1522 static void mvneta_set_ucast_addr(struct mvneta_port *pp, u8 last_nibble, 1523 int queue) 1524 { 1525 unsigned int unicast_reg; 1526 unsigned int tbl_offset; 1527 unsigned int reg_offset; 1528 1529 /* Locate the Unicast table entry */ 1530 last_nibble = (0xf & last_nibble); 1531 1532 /* offset from unicast tbl base */ 1533 tbl_offset = (last_nibble / 4) * 4; 1534 1535 /* offset within the above reg */ 1536 reg_offset = last_nibble % 4; 1537 1538 unicast_reg = mvreg_read(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset)); 1539 1540 if (queue == -1) { 1541 /* Clear accepts frame bit at specified unicast DA tbl entry */ 1542 unicast_reg &= ~(0xff << (8 * reg_offset)); 1543 } else { 1544 unicast_reg &= ~(0xff << (8 * reg_offset)); 1545 unicast_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset)); 1546 } 1547 1548 mvreg_write(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset), unicast_reg); 1549 } 1550 1551 /* Set mac address */ 1552 static void mvneta_mac_addr_set(struct mvneta_port *pp, unsigned char *addr, 1553 int queue) 1554 { 1555 unsigned int mac_h; 1556 unsigned int mac_l; 1557 1558 if (queue != -1) { 1559 mac_l = (addr[4] << 8) | (addr[5]); 1560 mac_h = (addr[0] << 24) | (addr[1] << 16) | 1561 (addr[2] << 8) | (addr[3] << 0); 1562 1563 mvreg_write(pp, MVNETA_MAC_ADDR_LOW, mac_l); 1564 mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, mac_h); 1565 } 1566 1567 /* Accept frames of this address */ 1568 mvneta_set_ucast_addr(pp, addr[5], queue); 1569 } 1570 1571 /* Set the number of packets that will be received before RX interrupt 1572 * will be generated by HW. 1573 */ 1574 static void mvneta_rx_pkts_coal_set(struct mvneta_port *pp, 1575 struct mvneta_rx_queue *rxq, u32 value) 1576 { 1577 mvreg_write(pp, MVNETA_RXQ_THRESHOLD_REG(rxq->id), 1578 value | MVNETA_RXQ_NON_OCCUPIED(0)); 1579 } 1580 1581 /* Set the time delay in usec before RX interrupt will be generated by 1582 * HW. 1583 */ 1584 static void mvneta_rx_time_coal_set(struct mvneta_port *pp, 1585 struct mvneta_rx_queue *rxq, u32 value) 1586 { 1587 u32 val; 1588 unsigned long clk_rate; 1589 1590 clk_rate = clk_get_rate(pp->clk); 1591 val = (clk_rate / 1000000) * value; 1592 1593 mvreg_write(pp, MVNETA_RXQ_TIME_COAL_REG(rxq->id), val); 1594 } 1595 1596 /* Set threshold for TX_DONE pkts coalescing */ 1597 static void mvneta_tx_done_pkts_coal_set(struct mvneta_port *pp, 1598 struct mvneta_tx_queue *txq, u32 value) 1599 { 1600 u32 val; 1601 1602 val = mvreg_read(pp, MVNETA_TXQ_SIZE_REG(txq->id)); 1603 1604 val &= ~MVNETA_TXQ_SENT_THRESH_ALL_MASK; 1605 val |= MVNETA_TXQ_SENT_THRESH_MASK(value); 1606 1607 mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), val); 1608 } 1609 1610 /* Handle rx descriptor fill by setting buf_cookie and buf_phys_addr */ 1611 static void mvneta_rx_desc_fill(struct mvneta_rx_desc *rx_desc, 1612 u32 phys_addr, void *virt_addr, 1613 struct mvneta_rx_queue *rxq) 1614 { 1615 int i; 1616 1617 rx_desc->buf_phys_addr = phys_addr; 1618 i = rx_desc - rxq->descs; 1619 rxq->buf_virt_addr[i] = virt_addr; 1620 } 1621 1622 /* Decrement sent descriptors counter */ 1623 static void mvneta_txq_sent_desc_dec(struct mvneta_port *pp, 1624 struct mvneta_tx_queue *txq, 1625 int sent_desc) 1626 { 1627 u32 val; 1628 1629 /* Only 255 TX descriptors can be updated at once */ 1630 while (sent_desc > 0xff) { 1631 val = 0xff << MVNETA_TXQ_DEC_SENT_SHIFT; 1632 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val); 1633 sent_desc = sent_desc - 0xff; 1634 } 1635 1636 val = sent_desc << MVNETA_TXQ_DEC_SENT_SHIFT; 1637 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val); 1638 } 1639 1640 /* Get number of TX descriptors already sent by HW */ 1641 static int mvneta_txq_sent_desc_num_get(struct mvneta_port *pp, 1642 struct mvneta_tx_queue *txq) 1643 { 1644 u32 val; 1645 int sent_desc; 1646 1647 val = mvreg_read(pp, MVNETA_TXQ_STATUS_REG(txq->id)); 1648 sent_desc = (val & MVNETA_TXQ_SENT_DESC_MASK) >> 1649 MVNETA_TXQ_SENT_DESC_SHIFT; 1650 1651 return sent_desc; 1652 } 1653 1654 /* Get number of sent descriptors and decrement counter. 1655 * The number of sent descriptors is returned. 1656 */ 1657 static int mvneta_txq_sent_desc_proc(struct mvneta_port *pp, 1658 struct mvneta_tx_queue *txq) 1659 { 1660 int sent_desc; 1661 1662 /* Get number of sent descriptors */ 1663 sent_desc = mvneta_txq_sent_desc_num_get(pp, txq); 1664 1665 /* Decrement sent descriptors counter */ 1666 if (sent_desc) 1667 mvneta_txq_sent_desc_dec(pp, txq, sent_desc); 1668 1669 return sent_desc; 1670 } 1671 1672 /* Set TXQ descriptors fields relevant for CSUM calculation */ 1673 static u32 mvneta_txq_desc_csum(int l3_offs, int l3_proto, 1674 int ip_hdr_len, int l4_proto) 1675 { 1676 u32 command; 1677 1678 /* Fields: L3_offset, IP_hdrlen, L3_type, G_IPv4_chk, 1679 * G_L4_chk, L4_type; required only for checksum 1680 * calculation 1681 */ 1682 command = l3_offs << MVNETA_TX_L3_OFF_SHIFT; 1683 command |= ip_hdr_len << MVNETA_TX_IP_HLEN_SHIFT; 1684 1685 if (l3_proto == htons(ETH_P_IP)) 1686 command |= MVNETA_TXD_IP_CSUM; 1687 else 1688 command |= MVNETA_TX_L3_IP6; 1689 1690 if (l4_proto == IPPROTO_TCP) 1691 command |= MVNETA_TX_L4_CSUM_FULL; 1692 else if (l4_proto == IPPROTO_UDP) 1693 command |= MVNETA_TX_L4_UDP | MVNETA_TX_L4_CSUM_FULL; 1694 else 1695 command |= MVNETA_TX_L4_CSUM_NOT; 1696 1697 return command; 1698 } 1699 1700 1701 /* Display more error info */ 1702 static void mvneta_rx_error(struct mvneta_port *pp, 1703 struct mvneta_rx_desc *rx_desc) 1704 { 1705 u32 status = rx_desc->status; 1706 1707 switch (status & MVNETA_RXD_ERR_CODE_MASK) { 1708 case MVNETA_RXD_ERR_CRC: 1709 netdev_err(pp->dev, "bad rx status %08x (crc error), size=%d\n", 1710 status, rx_desc->data_size); 1711 break; 1712 case MVNETA_RXD_ERR_OVERRUN: 1713 netdev_err(pp->dev, "bad rx status %08x (overrun error), size=%d\n", 1714 status, rx_desc->data_size); 1715 break; 1716 case MVNETA_RXD_ERR_LEN: 1717 netdev_err(pp->dev, "bad rx status %08x (max frame length error), size=%d\n", 1718 status, rx_desc->data_size); 1719 break; 1720 case MVNETA_RXD_ERR_RESOURCE: 1721 netdev_err(pp->dev, "bad rx status %08x (resource error), size=%d\n", 1722 status, rx_desc->data_size); 1723 break; 1724 } 1725 } 1726 1727 /* Handle RX checksum offload based on the descriptor's status */ 1728 static void mvneta_rx_csum(struct mvneta_port *pp, u32 status, 1729 struct sk_buff *skb) 1730 { 1731 if ((pp->dev->features & NETIF_F_RXCSUM) && 1732 (status & MVNETA_RXD_L3_IP4) && 1733 (status & MVNETA_RXD_L4_CSUM_OK)) { 1734 skb->csum = 0; 1735 skb->ip_summed = CHECKSUM_UNNECESSARY; 1736 return; 1737 } 1738 1739 skb->ip_summed = CHECKSUM_NONE; 1740 } 1741 1742 /* Return tx queue pointer (find last set bit) according to <cause> returned 1743 * form tx_done reg. <cause> must not be null. The return value is always a 1744 * valid queue for matching the first one found in <cause>. 1745 */ 1746 static struct mvneta_tx_queue *mvneta_tx_done_policy(struct mvneta_port *pp, 1747 u32 cause) 1748 { 1749 int queue = fls(cause) - 1; 1750 1751 return &pp->txqs[queue]; 1752 } 1753 1754 /* Free tx queue skbuffs */ 1755 static void mvneta_txq_bufs_free(struct mvneta_port *pp, 1756 struct mvneta_tx_queue *txq, int num, 1757 struct netdev_queue *nq) 1758 { 1759 unsigned int bytes_compl = 0, pkts_compl = 0; 1760 int i; 1761 1762 for (i = 0; i < num; i++) { 1763 struct mvneta_tx_desc *tx_desc = txq->descs + 1764 txq->txq_get_index; 1765 struct sk_buff *skb = txq->tx_skb[txq->txq_get_index]; 1766 1767 if (skb) { 1768 bytes_compl += skb->len; 1769 pkts_compl++; 1770 } 1771 1772 mvneta_txq_inc_get(txq); 1773 1774 if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr)) 1775 dma_unmap_single(pp->dev->dev.parent, 1776 tx_desc->buf_phys_addr, 1777 tx_desc->data_size, DMA_TO_DEVICE); 1778 if (!skb) 1779 continue; 1780 dev_kfree_skb_any(skb); 1781 } 1782 1783 netdev_tx_completed_queue(nq, pkts_compl, bytes_compl); 1784 } 1785 1786 /* Handle end of transmission */ 1787 static void mvneta_txq_done(struct mvneta_port *pp, 1788 struct mvneta_tx_queue *txq) 1789 { 1790 struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id); 1791 int tx_done; 1792 1793 tx_done = mvneta_txq_sent_desc_proc(pp, txq); 1794 if (!tx_done) 1795 return; 1796 1797 mvneta_txq_bufs_free(pp, txq, tx_done, nq); 1798 1799 txq->count -= tx_done; 1800 1801 if (netif_tx_queue_stopped(nq)) { 1802 if (txq->count <= txq->tx_wake_threshold) 1803 netif_tx_wake_queue(nq); 1804 } 1805 } 1806 1807 /* Refill processing for SW buffer management */ 1808 /* Allocate page per descriptor */ 1809 static int mvneta_rx_refill(struct mvneta_port *pp, 1810 struct mvneta_rx_desc *rx_desc, 1811 struct mvneta_rx_queue *rxq, 1812 gfp_t gfp_mask) 1813 { 1814 dma_addr_t phys_addr; 1815 struct page *page; 1816 1817 page = __dev_alloc_page(gfp_mask); 1818 if (!page) 1819 return -ENOMEM; 1820 1821 /* map page for use */ 1822 phys_addr = dma_map_page(pp->dev->dev.parent, page, 0, PAGE_SIZE, 1823 DMA_FROM_DEVICE); 1824 if (unlikely(dma_mapping_error(pp->dev->dev.parent, phys_addr))) { 1825 __free_page(page); 1826 return -ENOMEM; 1827 } 1828 1829 phys_addr += pp->rx_offset_correction; 1830 mvneta_rx_desc_fill(rx_desc, phys_addr, page, rxq); 1831 return 0; 1832 } 1833 1834 /* Handle tx checksum */ 1835 static u32 mvneta_skb_tx_csum(struct mvneta_port *pp, struct sk_buff *skb) 1836 { 1837 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1838 int ip_hdr_len = 0; 1839 __be16 l3_proto = vlan_get_protocol(skb); 1840 u8 l4_proto; 1841 1842 if (l3_proto == htons(ETH_P_IP)) { 1843 struct iphdr *ip4h = ip_hdr(skb); 1844 1845 /* Calculate IPv4 checksum and L4 checksum */ 1846 ip_hdr_len = ip4h->ihl; 1847 l4_proto = ip4h->protocol; 1848 } else if (l3_proto == htons(ETH_P_IPV6)) { 1849 struct ipv6hdr *ip6h = ipv6_hdr(skb); 1850 1851 /* Read l4_protocol from one of IPv6 extra headers */ 1852 if (skb_network_header_len(skb) > 0) 1853 ip_hdr_len = (skb_network_header_len(skb) >> 2); 1854 l4_proto = ip6h->nexthdr; 1855 } else 1856 return MVNETA_TX_L4_CSUM_NOT; 1857 1858 return mvneta_txq_desc_csum(skb_network_offset(skb), 1859 l3_proto, ip_hdr_len, l4_proto); 1860 } 1861 1862 return MVNETA_TX_L4_CSUM_NOT; 1863 } 1864 1865 /* Drop packets received by the RXQ and free buffers */ 1866 static void mvneta_rxq_drop_pkts(struct mvneta_port *pp, 1867 struct mvneta_rx_queue *rxq) 1868 { 1869 int rx_done, i; 1870 1871 rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); 1872 if (rx_done) 1873 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); 1874 1875 if (pp->bm_priv) { 1876 for (i = 0; i < rx_done; i++) { 1877 struct mvneta_rx_desc *rx_desc = 1878 mvneta_rxq_next_desc_get(rxq); 1879 u8 pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc); 1880 struct mvneta_bm_pool *bm_pool; 1881 1882 bm_pool = &pp->bm_priv->bm_pools[pool_id]; 1883 /* Return dropped buffer to the pool */ 1884 mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool, 1885 rx_desc->buf_phys_addr); 1886 } 1887 return; 1888 } 1889 1890 for (i = 0; i < rxq->size; i++) { 1891 struct mvneta_rx_desc *rx_desc = rxq->descs + i; 1892 void *data = rxq->buf_virt_addr[i]; 1893 if (!data || !(rx_desc->buf_phys_addr)) 1894 continue; 1895 1896 dma_unmap_page(pp->dev->dev.parent, rx_desc->buf_phys_addr, 1897 PAGE_SIZE, DMA_FROM_DEVICE); 1898 __free_page(data); 1899 } 1900 } 1901 1902 static inline 1903 int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) 1904 { 1905 struct mvneta_rx_desc *rx_desc; 1906 int curr_desc = rxq->first_to_refill; 1907 int i; 1908 1909 for (i = 0; (i < rxq->refill_num) && (i < 64); i++) { 1910 rx_desc = rxq->descs + curr_desc; 1911 if (!(rx_desc->buf_phys_addr)) { 1912 if (mvneta_rx_refill(pp, rx_desc, rxq, GFP_ATOMIC)) { 1913 pr_err("Can't refill queue %d. Done %d from %d\n", 1914 rxq->id, i, rxq->refill_num); 1915 rxq->refill_err++; 1916 break; 1917 } 1918 } 1919 curr_desc = MVNETA_QUEUE_NEXT_DESC(rxq, curr_desc); 1920 } 1921 rxq->refill_num -= i; 1922 rxq->first_to_refill = curr_desc; 1923 1924 return i; 1925 } 1926 1927 /* Main rx processing when using software buffer management */ 1928 static int mvneta_rx_swbm(struct napi_struct *napi, 1929 struct mvneta_port *pp, int budget, 1930 struct mvneta_rx_queue *rxq) 1931 { 1932 struct net_device *dev = pp->dev; 1933 int rx_todo, rx_proc; 1934 int refill = 0; 1935 u32 rcvd_pkts = 0; 1936 u32 rcvd_bytes = 0; 1937 1938 /* Get number of received packets */ 1939 rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq); 1940 rx_proc = 0; 1941 1942 /* Fairness NAPI loop */ 1943 while ((rcvd_pkts < budget) && (rx_proc < rx_todo)) { 1944 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); 1945 unsigned char *data; 1946 struct page *page; 1947 dma_addr_t phys_addr; 1948 u32 rx_status, index; 1949 int rx_bytes, skb_size, copy_size; 1950 int frag_num, frag_size, frag_offset; 1951 1952 index = rx_desc - rxq->descs; 1953 page = (struct page *)rxq->buf_virt_addr[index]; 1954 data = page_address(page); 1955 /* Prefetch header */ 1956 prefetch(data); 1957 1958 phys_addr = rx_desc->buf_phys_addr; 1959 rx_status = rx_desc->status; 1960 rx_proc++; 1961 rxq->refill_num++; 1962 1963 if (rx_status & MVNETA_RXD_FIRST_DESC) { 1964 /* Check errors only for FIRST descriptor */ 1965 if (rx_status & MVNETA_RXD_ERR_SUMMARY) { 1966 mvneta_rx_error(pp, rx_desc); 1967 dev->stats.rx_errors++; 1968 /* leave the descriptor untouched */ 1969 continue; 1970 } 1971 rx_bytes = rx_desc->data_size - 1972 (ETH_FCS_LEN + MVNETA_MH_SIZE); 1973 1974 /* Allocate small skb for each new packet */ 1975 skb_size = max(rx_copybreak, rx_header_size); 1976 rxq->skb = netdev_alloc_skb_ip_align(dev, skb_size); 1977 if (unlikely(!rxq->skb)) { 1978 netdev_err(dev, 1979 "Can't allocate skb on queue %d\n", 1980 rxq->id); 1981 dev->stats.rx_dropped++; 1982 rxq->skb_alloc_err++; 1983 continue; 1984 } 1985 copy_size = min(skb_size, rx_bytes); 1986 1987 /* Copy data from buffer to SKB, skip Marvell header */ 1988 memcpy(rxq->skb->data, data + MVNETA_MH_SIZE, 1989 copy_size); 1990 skb_put(rxq->skb, copy_size); 1991 rxq->left_size = rx_bytes - copy_size; 1992 1993 mvneta_rx_csum(pp, rx_status, rxq->skb); 1994 if (rxq->left_size == 0) { 1995 int size = copy_size + MVNETA_MH_SIZE; 1996 1997 dma_sync_single_range_for_cpu(dev->dev.parent, 1998 phys_addr, 0, 1999 size, 2000 DMA_FROM_DEVICE); 2001 2002 /* leave the descriptor and buffer untouched */ 2003 } else { 2004 /* refill descriptor with new buffer later */ 2005 rx_desc->buf_phys_addr = 0; 2006 2007 frag_num = 0; 2008 frag_offset = copy_size + MVNETA_MH_SIZE; 2009 frag_size = min(rxq->left_size, 2010 (int)(PAGE_SIZE - frag_offset)); 2011 skb_add_rx_frag(rxq->skb, frag_num, page, 2012 frag_offset, frag_size, 2013 PAGE_SIZE); 2014 dma_unmap_page(dev->dev.parent, phys_addr, 2015 PAGE_SIZE, DMA_FROM_DEVICE); 2016 rxq->left_size -= frag_size; 2017 } 2018 } else { 2019 /* Middle or Last descriptor */ 2020 if (unlikely(!rxq->skb)) { 2021 pr_debug("no skb for rx_status 0x%x\n", 2022 rx_status); 2023 continue; 2024 } 2025 if (!rxq->left_size) { 2026 /* last descriptor has only FCS */ 2027 /* and can be discarded */ 2028 dma_sync_single_range_for_cpu(dev->dev.parent, 2029 phys_addr, 0, 2030 ETH_FCS_LEN, 2031 DMA_FROM_DEVICE); 2032 /* leave the descriptor and buffer untouched */ 2033 } else { 2034 /* refill descriptor with new buffer later */ 2035 rx_desc->buf_phys_addr = 0; 2036 2037 frag_num = skb_shinfo(rxq->skb)->nr_frags; 2038 frag_offset = 0; 2039 frag_size = min(rxq->left_size, 2040 (int)(PAGE_SIZE - frag_offset)); 2041 skb_add_rx_frag(rxq->skb, frag_num, page, 2042 frag_offset, frag_size, 2043 PAGE_SIZE); 2044 2045 dma_unmap_page(dev->dev.parent, phys_addr, 2046 PAGE_SIZE, DMA_FROM_DEVICE); 2047 2048 rxq->left_size -= frag_size; 2049 } 2050 } /* Middle or Last descriptor */ 2051 2052 if (!(rx_status & MVNETA_RXD_LAST_DESC)) 2053 /* no last descriptor this time */ 2054 continue; 2055 2056 if (rxq->left_size) { 2057 pr_err("get last desc, but left_size (%d) != 0\n", 2058 rxq->left_size); 2059 dev_kfree_skb_any(rxq->skb); 2060 rxq->left_size = 0; 2061 rxq->skb = NULL; 2062 continue; 2063 } 2064 rcvd_pkts++; 2065 rcvd_bytes += rxq->skb->len; 2066 2067 /* Linux processing */ 2068 rxq->skb->protocol = eth_type_trans(rxq->skb, dev); 2069 2070 napi_gro_receive(napi, rxq->skb); 2071 2072 /* clean uncomplete skb pointer in queue */ 2073 rxq->skb = NULL; 2074 rxq->left_size = 0; 2075 } 2076 2077 if (rcvd_pkts) { 2078 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2079 2080 u64_stats_update_begin(&stats->syncp); 2081 stats->rx_packets += rcvd_pkts; 2082 stats->rx_bytes += rcvd_bytes; 2083 u64_stats_update_end(&stats->syncp); 2084 } 2085 2086 /* return some buffers to hardware queue, one at a time is too slow */ 2087 refill = mvneta_rx_refill_queue(pp, rxq); 2088 2089 /* Update rxq management counters */ 2090 mvneta_rxq_desc_num_update(pp, rxq, rx_proc, refill); 2091 2092 return rcvd_pkts; 2093 } 2094 2095 /* Main rx processing when using hardware buffer management */ 2096 static int mvneta_rx_hwbm(struct napi_struct *napi, 2097 struct mvneta_port *pp, int rx_todo, 2098 struct mvneta_rx_queue *rxq) 2099 { 2100 struct net_device *dev = pp->dev; 2101 int rx_done; 2102 u32 rcvd_pkts = 0; 2103 u32 rcvd_bytes = 0; 2104 2105 /* Get number of received packets */ 2106 rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); 2107 2108 if (rx_todo > rx_done) 2109 rx_todo = rx_done; 2110 2111 rx_done = 0; 2112 2113 /* Fairness NAPI loop */ 2114 while (rx_done < rx_todo) { 2115 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); 2116 struct mvneta_bm_pool *bm_pool = NULL; 2117 struct sk_buff *skb; 2118 unsigned char *data; 2119 dma_addr_t phys_addr; 2120 u32 rx_status, frag_size; 2121 int rx_bytes, err; 2122 u8 pool_id; 2123 2124 rx_done++; 2125 rx_status = rx_desc->status; 2126 rx_bytes = rx_desc->data_size - (ETH_FCS_LEN + MVNETA_MH_SIZE); 2127 data = (u8 *)(uintptr_t)rx_desc->buf_cookie; 2128 phys_addr = rx_desc->buf_phys_addr; 2129 pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc); 2130 bm_pool = &pp->bm_priv->bm_pools[pool_id]; 2131 2132 if (!mvneta_rxq_desc_is_first_last(rx_status) || 2133 (rx_status & MVNETA_RXD_ERR_SUMMARY)) { 2134 err_drop_frame_ret_pool: 2135 /* Return the buffer to the pool */ 2136 mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool, 2137 rx_desc->buf_phys_addr); 2138 err_drop_frame: 2139 dev->stats.rx_errors++; 2140 mvneta_rx_error(pp, rx_desc); 2141 /* leave the descriptor untouched */ 2142 continue; 2143 } 2144 2145 if (rx_bytes <= rx_copybreak) { 2146 /* better copy a small frame and not unmap the DMA region */ 2147 skb = netdev_alloc_skb_ip_align(dev, rx_bytes); 2148 if (unlikely(!skb)) 2149 goto err_drop_frame_ret_pool; 2150 2151 dma_sync_single_range_for_cpu(&pp->bm_priv->pdev->dev, 2152 rx_desc->buf_phys_addr, 2153 MVNETA_MH_SIZE + NET_SKB_PAD, 2154 rx_bytes, 2155 DMA_FROM_DEVICE); 2156 skb_put_data(skb, data + MVNETA_MH_SIZE + NET_SKB_PAD, 2157 rx_bytes); 2158 2159 skb->protocol = eth_type_trans(skb, dev); 2160 mvneta_rx_csum(pp, rx_status, skb); 2161 napi_gro_receive(napi, skb); 2162 2163 rcvd_pkts++; 2164 rcvd_bytes += rx_bytes; 2165 2166 /* Return the buffer to the pool */ 2167 mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool, 2168 rx_desc->buf_phys_addr); 2169 2170 /* leave the descriptor and buffer untouched */ 2171 continue; 2172 } 2173 2174 /* Refill processing */ 2175 err = hwbm_pool_refill(&bm_pool->hwbm_pool, GFP_ATOMIC); 2176 if (err) { 2177 netdev_err(dev, "Linux processing - Can't refill\n"); 2178 rxq->refill_err++; 2179 goto err_drop_frame_ret_pool; 2180 } 2181 2182 frag_size = bm_pool->hwbm_pool.frag_size; 2183 2184 skb = build_skb(data, frag_size > PAGE_SIZE ? 0 : frag_size); 2185 2186 /* After refill old buffer has to be unmapped regardless 2187 * the skb is successfully built or not. 2188 */ 2189 dma_unmap_single(&pp->bm_priv->pdev->dev, phys_addr, 2190 bm_pool->buf_size, DMA_FROM_DEVICE); 2191 if (!skb) 2192 goto err_drop_frame; 2193 2194 rcvd_pkts++; 2195 rcvd_bytes += rx_bytes; 2196 2197 /* Linux processing */ 2198 skb_reserve(skb, MVNETA_MH_SIZE + NET_SKB_PAD); 2199 skb_put(skb, rx_bytes); 2200 2201 skb->protocol = eth_type_trans(skb, dev); 2202 2203 mvneta_rx_csum(pp, rx_status, skb); 2204 2205 napi_gro_receive(napi, skb); 2206 } 2207 2208 if (rcvd_pkts) { 2209 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2210 2211 u64_stats_update_begin(&stats->syncp); 2212 stats->rx_packets += rcvd_pkts; 2213 stats->rx_bytes += rcvd_bytes; 2214 u64_stats_update_end(&stats->syncp); 2215 } 2216 2217 /* Update rxq management counters */ 2218 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); 2219 2220 return rx_done; 2221 } 2222 2223 static inline void 2224 mvneta_tso_put_hdr(struct sk_buff *skb, 2225 struct mvneta_port *pp, struct mvneta_tx_queue *txq) 2226 { 2227 struct mvneta_tx_desc *tx_desc; 2228 int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2229 2230 txq->tx_skb[txq->txq_put_index] = NULL; 2231 tx_desc = mvneta_txq_next_desc_get(txq); 2232 tx_desc->data_size = hdr_len; 2233 tx_desc->command = mvneta_skb_tx_csum(pp, skb); 2234 tx_desc->command |= MVNETA_TXD_F_DESC; 2235 tx_desc->buf_phys_addr = txq->tso_hdrs_phys + 2236 txq->txq_put_index * TSO_HEADER_SIZE; 2237 mvneta_txq_inc_put(txq); 2238 } 2239 2240 static inline int 2241 mvneta_tso_put_data(struct net_device *dev, struct mvneta_tx_queue *txq, 2242 struct sk_buff *skb, char *data, int size, 2243 bool last_tcp, bool is_last) 2244 { 2245 struct mvneta_tx_desc *tx_desc; 2246 2247 tx_desc = mvneta_txq_next_desc_get(txq); 2248 tx_desc->data_size = size; 2249 tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, data, 2250 size, DMA_TO_DEVICE); 2251 if (unlikely(dma_mapping_error(dev->dev.parent, 2252 tx_desc->buf_phys_addr))) { 2253 mvneta_txq_desc_put(txq); 2254 return -ENOMEM; 2255 } 2256 2257 tx_desc->command = 0; 2258 txq->tx_skb[txq->txq_put_index] = NULL; 2259 2260 if (last_tcp) { 2261 /* last descriptor in the TCP packet */ 2262 tx_desc->command = MVNETA_TXD_L_DESC; 2263 2264 /* last descriptor in SKB */ 2265 if (is_last) 2266 txq->tx_skb[txq->txq_put_index] = skb; 2267 } 2268 mvneta_txq_inc_put(txq); 2269 return 0; 2270 } 2271 2272 static int mvneta_tx_tso(struct sk_buff *skb, struct net_device *dev, 2273 struct mvneta_tx_queue *txq) 2274 { 2275 int total_len, data_left; 2276 int desc_count = 0; 2277 struct mvneta_port *pp = netdev_priv(dev); 2278 struct tso_t tso; 2279 int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2280 int i; 2281 2282 /* Count needed descriptors */ 2283 if ((txq->count + tso_count_descs(skb)) >= txq->size) 2284 return 0; 2285 2286 if (skb_headlen(skb) < (skb_transport_offset(skb) + tcp_hdrlen(skb))) { 2287 pr_info("*** Is this even possible???!?!?\n"); 2288 return 0; 2289 } 2290 2291 /* Initialize the TSO handler, and prepare the first payload */ 2292 tso_start(skb, &tso); 2293 2294 total_len = skb->len - hdr_len; 2295 while (total_len > 0) { 2296 char *hdr; 2297 2298 data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len); 2299 total_len -= data_left; 2300 desc_count++; 2301 2302 /* prepare packet headers: MAC + IP + TCP */ 2303 hdr = txq->tso_hdrs + txq->txq_put_index * TSO_HEADER_SIZE; 2304 tso_build_hdr(skb, hdr, &tso, data_left, total_len == 0); 2305 2306 mvneta_tso_put_hdr(skb, pp, txq); 2307 2308 while (data_left > 0) { 2309 int size; 2310 desc_count++; 2311 2312 size = min_t(int, tso.size, data_left); 2313 2314 if (mvneta_tso_put_data(dev, txq, skb, 2315 tso.data, size, 2316 size == data_left, 2317 total_len == 0)) 2318 goto err_release; 2319 data_left -= size; 2320 2321 tso_build_data(skb, &tso, size); 2322 } 2323 } 2324 2325 return desc_count; 2326 2327 err_release: 2328 /* Release all used data descriptors; header descriptors must not 2329 * be DMA-unmapped. 2330 */ 2331 for (i = desc_count - 1; i >= 0; i--) { 2332 struct mvneta_tx_desc *tx_desc = txq->descs + i; 2333 if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr)) 2334 dma_unmap_single(pp->dev->dev.parent, 2335 tx_desc->buf_phys_addr, 2336 tx_desc->data_size, 2337 DMA_TO_DEVICE); 2338 mvneta_txq_desc_put(txq); 2339 } 2340 return 0; 2341 } 2342 2343 /* Handle tx fragmentation processing */ 2344 static int mvneta_tx_frag_process(struct mvneta_port *pp, struct sk_buff *skb, 2345 struct mvneta_tx_queue *txq) 2346 { 2347 struct mvneta_tx_desc *tx_desc; 2348 int i, nr_frags = skb_shinfo(skb)->nr_frags; 2349 2350 for (i = 0; i < nr_frags; i++) { 2351 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2352 void *addr = page_address(frag->page.p) + frag->page_offset; 2353 2354 tx_desc = mvneta_txq_next_desc_get(txq); 2355 tx_desc->data_size = frag->size; 2356 2357 tx_desc->buf_phys_addr = 2358 dma_map_single(pp->dev->dev.parent, addr, 2359 tx_desc->data_size, DMA_TO_DEVICE); 2360 2361 if (dma_mapping_error(pp->dev->dev.parent, 2362 tx_desc->buf_phys_addr)) { 2363 mvneta_txq_desc_put(txq); 2364 goto error; 2365 } 2366 2367 if (i == nr_frags - 1) { 2368 /* Last descriptor */ 2369 tx_desc->command = MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD; 2370 txq->tx_skb[txq->txq_put_index] = skb; 2371 } else { 2372 /* Descriptor in the middle: Not First, Not Last */ 2373 tx_desc->command = 0; 2374 txq->tx_skb[txq->txq_put_index] = NULL; 2375 } 2376 mvneta_txq_inc_put(txq); 2377 } 2378 2379 return 0; 2380 2381 error: 2382 /* Release all descriptors that were used to map fragments of 2383 * this packet, as well as the corresponding DMA mappings 2384 */ 2385 for (i = i - 1; i >= 0; i--) { 2386 tx_desc = txq->descs + i; 2387 dma_unmap_single(pp->dev->dev.parent, 2388 tx_desc->buf_phys_addr, 2389 tx_desc->data_size, 2390 DMA_TO_DEVICE); 2391 mvneta_txq_desc_put(txq); 2392 } 2393 2394 return -ENOMEM; 2395 } 2396 2397 /* Main tx processing */ 2398 static netdev_tx_t mvneta_tx(struct sk_buff *skb, struct net_device *dev) 2399 { 2400 struct mvneta_port *pp = netdev_priv(dev); 2401 u16 txq_id = skb_get_queue_mapping(skb); 2402 struct mvneta_tx_queue *txq = &pp->txqs[txq_id]; 2403 struct mvneta_tx_desc *tx_desc; 2404 int len = skb->len; 2405 int frags = 0; 2406 u32 tx_cmd; 2407 2408 if (!netif_running(dev)) 2409 goto out; 2410 2411 if (skb_is_gso(skb)) { 2412 frags = mvneta_tx_tso(skb, dev, txq); 2413 goto out; 2414 } 2415 2416 frags = skb_shinfo(skb)->nr_frags + 1; 2417 2418 /* Get a descriptor for the first part of the packet */ 2419 tx_desc = mvneta_txq_next_desc_get(txq); 2420 2421 tx_cmd = mvneta_skb_tx_csum(pp, skb); 2422 2423 tx_desc->data_size = skb_headlen(skb); 2424 2425 tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, skb->data, 2426 tx_desc->data_size, 2427 DMA_TO_DEVICE); 2428 if (unlikely(dma_mapping_error(dev->dev.parent, 2429 tx_desc->buf_phys_addr))) { 2430 mvneta_txq_desc_put(txq); 2431 frags = 0; 2432 goto out; 2433 } 2434 2435 if (frags == 1) { 2436 /* First and Last descriptor */ 2437 tx_cmd |= MVNETA_TXD_FLZ_DESC; 2438 tx_desc->command = tx_cmd; 2439 txq->tx_skb[txq->txq_put_index] = skb; 2440 mvneta_txq_inc_put(txq); 2441 } else { 2442 /* First but not Last */ 2443 tx_cmd |= MVNETA_TXD_F_DESC; 2444 txq->tx_skb[txq->txq_put_index] = NULL; 2445 mvneta_txq_inc_put(txq); 2446 tx_desc->command = tx_cmd; 2447 /* Continue with other skb fragments */ 2448 if (mvneta_tx_frag_process(pp, skb, txq)) { 2449 dma_unmap_single(dev->dev.parent, 2450 tx_desc->buf_phys_addr, 2451 tx_desc->data_size, 2452 DMA_TO_DEVICE); 2453 mvneta_txq_desc_put(txq); 2454 frags = 0; 2455 goto out; 2456 } 2457 } 2458 2459 out: 2460 if (frags > 0) { 2461 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2462 struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id); 2463 2464 netdev_tx_sent_queue(nq, len); 2465 2466 txq->count += frags; 2467 if (txq->count >= txq->tx_stop_threshold) 2468 netif_tx_stop_queue(nq); 2469 2470 if (!netdev_xmit_more() || netif_xmit_stopped(nq) || 2471 txq->pending + frags > MVNETA_TXQ_DEC_SENT_MASK) 2472 mvneta_txq_pend_desc_add(pp, txq, frags); 2473 else 2474 txq->pending += frags; 2475 2476 u64_stats_update_begin(&stats->syncp); 2477 stats->tx_packets++; 2478 stats->tx_bytes += len; 2479 u64_stats_update_end(&stats->syncp); 2480 } else { 2481 dev->stats.tx_dropped++; 2482 dev_kfree_skb_any(skb); 2483 } 2484 2485 return NETDEV_TX_OK; 2486 } 2487 2488 2489 /* Free tx resources, when resetting a port */ 2490 static void mvneta_txq_done_force(struct mvneta_port *pp, 2491 struct mvneta_tx_queue *txq) 2492 2493 { 2494 struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id); 2495 int tx_done = txq->count; 2496 2497 mvneta_txq_bufs_free(pp, txq, tx_done, nq); 2498 2499 /* reset txq */ 2500 txq->count = 0; 2501 txq->txq_put_index = 0; 2502 txq->txq_get_index = 0; 2503 } 2504 2505 /* Handle tx done - called in softirq context. The <cause_tx_done> argument 2506 * must be a valid cause according to MVNETA_TXQ_INTR_MASK_ALL. 2507 */ 2508 static void mvneta_tx_done_gbe(struct mvneta_port *pp, u32 cause_tx_done) 2509 { 2510 struct mvneta_tx_queue *txq; 2511 struct netdev_queue *nq; 2512 int cpu = smp_processor_id(); 2513 2514 while (cause_tx_done) { 2515 txq = mvneta_tx_done_policy(pp, cause_tx_done); 2516 2517 nq = netdev_get_tx_queue(pp->dev, txq->id); 2518 __netif_tx_lock(nq, cpu); 2519 2520 if (txq->count) 2521 mvneta_txq_done(pp, txq); 2522 2523 __netif_tx_unlock(nq); 2524 cause_tx_done &= ~((1 << txq->id)); 2525 } 2526 } 2527 2528 /* Compute crc8 of the specified address, using a unique algorithm , 2529 * according to hw spec, different than generic crc8 algorithm 2530 */ 2531 static int mvneta_addr_crc(unsigned char *addr) 2532 { 2533 int crc = 0; 2534 int i; 2535 2536 for (i = 0; i < ETH_ALEN; i++) { 2537 int j; 2538 2539 crc = (crc ^ addr[i]) << 8; 2540 for (j = 7; j >= 0; j--) { 2541 if (crc & (0x100 << j)) 2542 crc ^= 0x107 << j; 2543 } 2544 } 2545 2546 return crc; 2547 } 2548 2549 /* This method controls the net device special MAC multicast support. 2550 * The Special Multicast Table for MAC addresses supports MAC of the form 2551 * 0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF). 2552 * The MAC DA[7:0] bits are used as a pointer to the Special Multicast 2553 * Table entries in the DA-Filter table. This method set the Special 2554 * Multicast Table appropriate entry. 2555 */ 2556 static void mvneta_set_special_mcast_addr(struct mvneta_port *pp, 2557 unsigned char last_byte, 2558 int queue) 2559 { 2560 unsigned int smc_table_reg; 2561 unsigned int tbl_offset; 2562 unsigned int reg_offset; 2563 2564 /* Register offset from SMC table base */ 2565 tbl_offset = (last_byte / 4); 2566 /* Entry offset within the above reg */ 2567 reg_offset = last_byte % 4; 2568 2569 smc_table_reg = mvreg_read(pp, (MVNETA_DA_FILT_SPEC_MCAST 2570 + tbl_offset * 4)); 2571 2572 if (queue == -1) 2573 smc_table_reg &= ~(0xff << (8 * reg_offset)); 2574 else { 2575 smc_table_reg &= ~(0xff << (8 * reg_offset)); 2576 smc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset)); 2577 } 2578 2579 mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + tbl_offset * 4, 2580 smc_table_reg); 2581 } 2582 2583 /* This method controls the network device Other MAC multicast support. 2584 * The Other Multicast Table is used for multicast of another type. 2585 * A CRC-8 is used as an index to the Other Multicast Table entries 2586 * in the DA-Filter table. 2587 * The method gets the CRC-8 value from the calling routine and 2588 * sets the Other Multicast Table appropriate entry according to the 2589 * specified CRC-8 . 2590 */ 2591 static void mvneta_set_other_mcast_addr(struct mvneta_port *pp, 2592 unsigned char crc8, 2593 int queue) 2594 { 2595 unsigned int omc_table_reg; 2596 unsigned int tbl_offset; 2597 unsigned int reg_offset; 2598 2599 tbl_offset = (crc8 / 4) * 4; /* Register offset from OMC table base */ 2600 reg_offset = crc8 % 4; /* Entry offset within the above reg */ 2601 2602 omc_table_reg = mvreg_read(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset); 2603 2604 if (queue == -1) { 2605 /* Clear accepts frame bit at specified Other DA table entry */ 2606 omc_table_reg &= ~(0xff << (8 * reg_offset)); 2607 } else { 2608 omc_table_reg &= ~(0xff << (8 * reg_offset)); 2609 omc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset)); 2610 } 2611 2612 mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset, omc_table_reg); 2613 } 2614 2615 /* The network device supports multicast using two tables: 2616 * 1) Special Multicast Table for MAC addresses of the form 2617 * 0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF). 2618 * The MAC DA[7:0] bits are used as a pointer to the Special Multicast 2619 * Table entries in the DA-Filter table. 2620 * 2) Other Multicast Table for multicast of another type. A CRC-8 value 2621 * is used as an index to the Other Multicast Table entries in the 2622 * DA-Filter table. 2623 */ 2624 static int mvneta_mcast_addr_set(struct mvneta_port *pp, unsigned char *p_addr, 2625 int queue) 2626 { 2627 unsigned char crc_result = 0; 2628 2629 if (memcmp(p_addr, "\x01\x00\x5e\x00\x00", 5) == 0) { 2630 mvneta_set_special_mcast_addr(pp, p_addr[5], queue); 2631 return 0; 2632 } 2633 2634 crc_result = mvneta_addr_crc(p_addr); 2635 if (queue == -1) { 2636 if (pp->mcast_count[crc_result] == 0) { 2637 netdev_info(pp->dev, "No valid Mcast for crc8=0x%02x\n", 2638 crc_result); 2639 return -EINVAL; 2640 } 2641 2642 pp->mcast_count[crc_result]--; 2643 if (pp->mcast_count[crc_result] != 0) { 2644 netdev_info(pp->dev, 2645 "After delete there are %d valid Mcast for crc8=0x%02x\n", 2646 pp->mcast_count[crc_result], crc_result); 2647 return -EINVAL; 2648 } 2649 } else 2650 pp->mcast_count[crc_result]++; 2651 2652 mvneta_set_other_mcast_addr(pp, crc_result, queue); 2653 2654 return 0; 2655 } 2656 2657 /* Configure Fitering mode of Ethernet port */ 2658 static void mvneta_rx_unicast_promisc_set(struct mvneta_port *pp, 2659 int is_promisc) 2660 { 2661 u32 port_cfg_reg, val; 2662 2663 port_cfg_reg = mvreg_read(pp, MVNETA_PORT_CONFIG); 2664 2665 val = mvreg_read(pp, MVNETA_TYPE_PRIO); 2666 2667 /* Set / Clear UPM bit in port configuration register */ 2668 if (is_promisc) { 2669 /* Accept all Unicast addresses */ 2670 port_cfg_reg |= MVNETA_UNI_PROMISC_MODE; 2671 val |= MVNETA_FORCE_UNI; 2672 mvreg_write(pp, MVNETA_MAC_ADDR_LOW, 0xffff); 2673 mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, 0xffffffff); 2674 } else { 2675 /* Reject all Unicast addresses */ 2676 port_cfg_reg &= ~MVNETA_UNI_PROMISC_MODE; 2677 val &= ~MVNETA_FORCE_UNI; 2678 } 2679 2680 mvreg_write(pp, MVNETA_PORT_CONFIG, port_cfg_reg); 2681 mvreg_write(pp, MVNETA_TYPE_PRIO, val); 2682 } 2683 2684 /* register unicast and multicast addresses */ 2685 static void mvneta_set_rx_mode(struct net_device *dev) 2686 { 2687 struct mvneta_port *pp = netdev_priv(dev); 2688 struct netdev_hw_addr *ha; 2689 2690 if (dev->flags & IFF_PROMISC) { 2691 /* Accept all: Multicast + Unicast */ 2692 mvneta_rx_unicast_promisc_set(pp, 1); 2693 mvneta_set_ucast_table(pp, pp->rxq_def); 2694 mvneta_set_special_mcast_table(pp, pp->rxq_def); 2695 mvneta_set_other_mcast_table(pp, pp->rxq_def); 2696 } else { 2697 /* Accept single Unicast */ 2698 mvneta_rx_unicast_promisc_set(pp, 0); 2699 mvneta_set_ucast_table(pp, -1); 2700 mvneta_mac_addr_set(pp, dev->dev_addr, pp->rxq_def); 2701 2702 if (dev->flags & IFF_ALLMULTI) { 2703 /* Accept all multicast */ 2704 mvneta_set_special_mcast_table(pp, pp->rxq_def); 2705 mvneta_set_other_mcast_table(pp, pp->rxq_def); 2706 } else { 2707 /* Accept only initialized multicast */ 2708 mvneta_set_special_mcast_table(pp, -1); 2709 mvneta_set_other_mcast_table(pp, -1); 2710 2711 if (!netdev_mc_empty(dev)) { 2712 netdev_for_each_mc_addr(ha, dev) { 2713 mvneta_mcast_addr_set(pp, ha->addr, 2714 pp->rxq_def); 2715 } 2716 } 2717 } 2718 } 2719 } 2720 2721 /* Interrupt handling - the callback for request_irq() */ 2722 static irqreturn_t mvneta_isr(int irq, void *dev_id) 2723 { 2724 struct mvneta_port *pp = (struct mvneta_port *)dev_id; 2725 2726 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0); 2727 napi_schedule(&pp->napi); 2728 2729 return IRQ_HANDLED; 2730 } 2731 2732 /* Interrupt handling - the callback for request_percpu_irq() */ 2733 static irqreturn_t mvneta_percpu_isr(int irq, void *dev_id) 2734 { 2735 struct mvneta_pcpu_port *port = (struct mvneta_pcpu_port *)dev_id; 2736 2737 disable_percpu_irq(port->pp->dev->irq); 2738 napi_schedule(&port->napi); 2739 2740 return IRQ_HANDLED; 2741 } 2742 2743 static void mvneta_link_change(struct mvneta_port *pp) 2744 { 2745 u32 gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS); 2746 2747 phylink_mac_change(pp->phylink, !!(gmac_stat & MVNETA_GMAC_LINK_UP)); 2748 } 2749 2750 /* NAPI handler 2751 * Bits 0 - 7 of the causeRxTx register indicate that are transmitted 2752 * packets on the corresponding TXQ (Bit 0 is for TX queue 1). 2753 * Bits 8 -15 of the cause Rx Tx register indicate that are received 2754 * packets on the corresponding RXQ (Bit 8 is for RX queue 0). 2755 * Each CPU has its own causeRxTx register 2756 */ 2757 static int mvneta_poll(struct napi_struct *napi, int budget) 2758 { 2759 int rx_done = 0; 2760 u32 cause_rx_tx; 2761 int rx_queue; 2762 struct mvneta_port *pp = netdev_priv(napi->dev); 2763 struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports); 2764 2765 if (!netif_running(pp->dev)) { 2766 napi_complete(napi); 2767 return rx_done; 2768 } 2769 2770 /* Read cause register */ 2771 cause_rx_tx = mvreg_read(pp, MVNETA_INTR_NEW_CAUSE); 2772 if (cause_rx_tx & MVNETA_MISCINTR_INTR_MASK) { 2773 u32 cause_misc = mvreg_read(pp, MVNETA_INTR_MISC_CAUSE); 2774 2775 mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0); 2776 2777 if (cause_misc & (MVNETA_CAUSE_PHY_STATUS_CHANGE | 2778 MVNETA_CAUSE_LINK_CHANGE)) 2779 mvneta_link_change(pp); 2780 } 2781 2782 /* Release Tx descriptors */ 2783 if (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL) { 2784 mvneta_tx_done_gbe(pp, (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL)); 2785 cause_rx_tx &= ~MVNETA_TX_INTR_MASK_ALL; 2786 } 2787 2788 /* For the case where the last mvneta_poll did not process all 2789 * RX packets 2790 */ 2791 rx_queue = fls(((cause_rx_tx >> 8) & 0xff)); 2792 2793 cause_rx_tx |= pp->neta_armada3700 ? pp->cause_rx_tx : 2794 port->cause_rx_tx; 2795 2796 if (rx_queue) { 2797 rx_queue = rx_queue - 1; 2798 if (pp->bm_priv) 2799 rx_done = mvneta_rx_hwbm(napi, pp, budget, 2800 &pp->rxqs[rx_queue]); 2801 else 2802 rx_done = mvneta_rx_swbm(napi, pp, budget, 2803 &pp->rxqs[rx_queue]); 2804 } 2805 2806 if (rx_done < budget) { 2807 cause_rx_tx = 0; 2808 napi_complete_done(napi, rx_done); 2809 2810 if (pp->neta_armada3700) { 2811 unsigned long flags; 2812 2813 local_irq_save(flags); 2814 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 2815 MVNETA_RX_INTR_MASK(rxq_number) | 2816 MVNETA_TX_INTR_MASK(txq_number) | 2817 MVNETA_MISCINTR_INTR_MASK); 2818 local_irq_restore(flags); 2819 } else { 2820 enable_percpu_irq(pp->dev->irq, 0); 2821 } 2822 } 2823 2824 if (pp->neta_armada3700) 2825 pp->cause_rx_tx = cause_rx_tx; 2826 else 2827 port->cause_rx_tx = cause_rx_tx; 2828 2829 return rx_done; 2830 } 2831 2832 /* Handle rxq fill: allocates rxq skbs; called when initializing a port */ 2833 static int mvneta_rxq_fill(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, 2834 int num) 2835 { 2836 int i; 2837 2838 for (i = 0; i < num; i++) { 2839 memset(rxq->descs + i, 0, sizeof(struct mvneta_rx_desc)); 2840 if (mvneta_rx_refill(pp, rxq->descs + i, rxq, 2841 GFP_KERNEL) != 0) { 2842 netdev_err(pp->dev, 2843 "%s:rxq %d, %d of %d buffs filled\n", 2844 __func__, rxq->id, i, num); 2845 break; 2846 } 2847 } 2848 2849 /* Add this number of RX descriptors as non occupied (ready to 2850 * get packets) 2851 */ 2852 mvneta_rxq_non_occup_desc_add(pp, rxq, i); 2853 2854 return i; 2855 } 2856 2857 /* Free all packets pending transmit from all TXQs and reset TX port */ 2858 static void mvneta_tx_reset(struct mvneta_port *pp) 2859 { 2860 int queue; 2861 2862 /* free the skb's in the tx ring */ 2863 for (queue = 0; queue < txq_number; queue++) 2864 mvneta_txq_done_force(pp, &pp->txqs[queue]); 2865 2866 mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET); 2867 mvreg_write(pp, MVNETA_PORT_TX_RESET, 0); 2868 } 2869 2870 static void mvneta_rx_reset(struct mvneta_port *pp) 2871 { 2872 mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET); 2873 mvreg_write(pp, MVNETA_PORT_RX_RESET, 0); 2874 } 2875 2876 /* Rx/Tx queue initialization/cleanup methods */ 2877 2878 static int mvneta_rxq_sw_init(struct mvneta_port *pp, 2879 struct mvneta_rx_queue *rxq) 2880 { 2881 rxq->size = pp->rx_ring_size; 2882 2883 /* Allocate memory for RX descriptors */ 2884 rxq->descs = dma_alloc_coherent(pp->dev->dev.parent, 2885 rxq->size * MVNETA_DESC_ALIGNED_SIZE, 2886 &rxq->descs_phys, GFP_KERNEL); 2887 if (!rxq->descs) 2888 return -ENOMEM; 2889 2890 rxq->last_desc = rxq->size - 1; 2891 2892 return 0; 2893 } 2894 2895 static void mvneta_rxq_hw_init(struct mvneta_port *pp, 2896 struct mvneta_rx_queue *rxq) 2897 { 2898 /* Set Rx descriptors queue starting address */ 2899 mvreg_write(pp, MVNETA_RXQ_BASE_ADDR_REG(rxq->id), rxq->descs_phys); 2900 mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), rxq->size); 2901 2902 /* Set coalescing pkts and time */ 2903 mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal); 2904 mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal); 2905 2906 if (!pp->bm_priv) { 2907 /* Set Offset */ 2908 mvneta_rxq_offset_set(pp, rxq, 0); 2909 mvneta_rxq_buf_size_set(pp, rxq, PAGE_SIZE < SZ_64K ? 2910 PAGE_SIZE : 2911 MVNETA_RX_BUF_SIZE(pp->pkt_size)); 2912 mvneta_rxq_bm_disable(pp, rxq); 2913 mvneta_rxq_fill(pp, rxq, rxq->size); 2914 } else { 2915 /* Set Offset */ 2916 mvneta_rxq_offset_set(pp, rxq, 2917 NET_SKB_PAD - pp->rx_offset_correction); 2918 2919 mvneta_rxq_bm_enable(pp, rxq); 2920 /* Fill RXQ with buffers from RX pool */ 2921 mvneta_rxq_long_pool_set(pp, rxq); 2922 mvneta_rxq_short_pool_set(pp, rxq); 2923 mvneta_rxq_non_occup_desc_add(pp, rxq, rxq->size); 2924 } 2925 } 2926 2927 /* Create a specified RX queue */ 2928 static int mvneta_rxq_init(struct mvneta_port *pp, 2929 struct mvneta_rx_queue *rxq) 2930 2931 { 2932 int ret; 2933 2934 ret = mvneta_rxq_sw_init(pp, rxq); 2935 if (ret < 0) 2936 return ret; 2937 2938 mvneta_rxq_hw_init(pp, rxq); 2939 2940 return 0; 2941 } 2942 2943 /* Cleanup Rx queue */ 2944 static void mvneta_rxq_deinit(struct mvneta_port *pp, 2945 struct mvneta_rx_queue *rxq) 2946 { 2947 mvneta_rxq_drop_pkts(pp, rxq); 2948 2949 if (rxq->skb) 2950 dev_kfree_skb_any(rxq->skb); 2951 2952 if (rxq->descs) 2953 dma_free_coherent(pp->dev->dev.parent, 2954 rxq->size * MVNETA_DESC_ALIGNED_SIZE, 2955 rxq->descs, 2956 rxq->descs_phys); 2957 2958 rxq->descs = NULL; 2959 rxq->last_desc = 0; 2960 rxq->next_desc_to_proc = 0; 2961 rxq->descs_phys = 0; 2962 rxq->first_to_refill = 0; 2963 rxq->refill_num = 0; 2964 rxq->skb = NULL; 2965 rxq->left_size = 0; 2966 } 2967 2968 static int mvneta_txq_sw_init(struct mvneta_port *pp, 2969 struct mvneta_tx_queue *txq) 2970 { 2971 int cpu; 2972 2973 txq->size = pp->tx_ring_size; 2974 2975 /* A queue must always have room for at least one skb. 2976 * Therefore, stop the queue when the free entries reaches 2977 * the maximum number of descriptors per skb. 2978 */ 2979 txq->tx_stop_threshold = txq->size - MVNETA_MAX_SKB_DESCS; 2980 txq->tx_wake_threshold = txq->tx_stop_threshold / 2; 2981 2982 /* Allocate memory for TX descriptors */ 2983 txq->descs = dma_alloc_coherent(pp->dev->dev.parent, 2984 txq->size * MVNETA_DESC_ALIGNED_SIZE, 2985 &txq->descs_phys, GFP_KERNEL); 2986 if (!txq->descs) 2987 return -ENOMEM; 2988 2989 txq->last_desc = txq->size - 1; 2990 2991 txq->tx_skb = kmalloc_array(txq->size, sizeof(*txq->tx_skb), 2992 GFP_KERNEL); 2993 if (!txq->tx_skb) { 2994 dma_free_coherent(pp->dev->dev.parent, 2995 txq->size * MVNETA_DESC_ALIGNED_SIZE, 2996 txq->descs, txq->descs_phys); 2997 return -ENOMEM; 2998 } 2999 3000 /* Allocate DMA buffers for TSO MAC/IP/TCP headers */ 3001 txq->tso_hdrs = dma_alloc_coherent(pp->dev->dev.parent, 3002 txq->size * TSO_HEADER_SIZE, 3003 &txq->tso_hdrs_phys, GFP_KERNEL); 3004 if (!txq->tso_hdrs) { 3005 kfree(txq->tx_skb); 3006 dma_free_coherent(pp->dev->dev.parent, 3007 txq->size * MVNETA_DESC_ALIGNED_SIZE, 3008 txq->descs, txq->descs_phys); 3009 return -ENOMEM; 3010 } 3011 3012 /* Setup XPS mapping */ 3013 if (txq_number > 1) 3014 cpu = txq->id % num_present_cpus(); 3015 else 3016 cpu = pp->rxq_def % num_present_cpus(); 3017 cpumask_set_cpu(cpu, &txq->affinity_mask); 3018 netif_set_xps_queue(pp->dev, &txq->affinity_mask, txq->id); 3019 3020 return 0; 3021 } 3022 3023 static void mvneta_txq_hw_init(struct mvneta_port *pp, 3024 struct mvneta_tx_queue *txq) 3025 { 3026 /* Set maximum bandwidth for enabled TXQs */ 3027 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0x03ffffff); 3028 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0x3fffffff); 3029 3030 /* Set Tx descriptors queue starting address */ 3031 mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), txq->descs_phys); 3032 mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), txq->size); 3033 3034 mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal); 3035 } 3036 3037 /* Create and initialize a tx queue */ 3038 static int mvneta_txq_init(struct mvneta_port *pp, 3039 struct mvneta_tx_queue *txq) 3040 { 3041 int ret; 3042 3043 ret = mvneta_txq_sw_init(pp, txq); 3044 if (ret < 0) 3045 return ret; 3046 3047 mvneta_txq_hw_init(pp, txq); 3048 3049 return 0; 3050 } 3051 3052 /* Free allocated resources when mvneta_txq_init() fails to allocate memory*/ 3053 static void mvneta_txq_sw_deinit(struct mvneta_port *pp, 3054 struct mvneta_tx_queue *txq) 3055 { 3056 struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id); 3057 3058 kfree(txq->tx_skb); 3059 3060 if (txq->tso_hdrs) 3061 dma_free_coherent(pp->dev->dev.parent, 3062 txq->size * TSO_HEADER_SIZE, 3063 txq->tso_hdrs, txq->tso_hdrs_phys); 3064 if (txq->descs) 3065 dma_free_coherent(pp->dev->dev.parent, 3066 txq->size * MVNETA_DESC_ALIGNED_SIZE, 3067 txq->descs, txq->descs_phys); 3068 3069 netdev_tx_reset_queue(nq); 3070 3071 txq->descs = NULL; 3072 txq->last_desc = 0; 3073 txq->next_desc_to_proc = 0; 3074 txq->descs_phys = 0; 3075 } 3076 3077 static void mvneta_txq_hw_deinit(struct mvneta_port *pp, 3078 struct mvneta_tx_queue *txq) 3079 { 3080 /* Set minimum bandwidth for disabled TXQs */ 3081 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0); 3082 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0); 3083 3084 /* Set Tx descriptors queue starting address and size */ 3085 mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), 0); 3086 mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), 0); 3087 } 3088 3089 static void mvneta_txq_deinit(struct mvneta_port *pp, 3090 struct mvneta_tx_queue *txq) 3091 { 3092 mvneta_txq_sw_deinit(pp, txq); 3093 mvneta_txq_hw_deinit(pp, txq); 3094 } 3095 3096 /* Cleanup all Tx queues */ 3097 static void mvneta_cleanup_txqs(struct mvneta_port *pp) 3098 { 3099 int queue; 3100 3101 for (queue = 0; queue < txq_number; queue++) 3102 mvneta_txq_deinit(pp, &pp->txqs[queue]); 3103 } 3104 3105 /* Cleanup all Rx queues */ 3106 static void mvneta_cleanup_rxqs(struct mvneta_port *pp) 3107 { 3108 int queue; 3109 3110 for (queue = 0; queue < rxq_number; queue++) 3111 mvneta_rxq_deinit(pp, &pp->rxqs[queue]); 3112 } 3113 3114 3115 /* Init all Rx queues */ 3116 static int mvneta_setup_rxqs(struct mvneta_port *pp) 3117 { 3118 int queue; 3119 3120 for (queue = 0; queue < rxq_number; queue++) { 3121 int err = mvneta_rxq_init(pp, &pp->rxqs[queue]); 3122 3123 if (err) { 3124 netdev_err(pp->dev, "%s: can't create rxq=%d\n", 3125 __func__, queue); 3126 mvneta_cleanup_rxqs(pp); 3127 return err; 3128 } 3129 } 3130 3131 return 0; 3132 } 3133 3134 /* Init all tx queues */ 3135 static int mvneta_setup_txqs(struct mvneta_port *pp) 3136 { 3137 int queue; 3138 3139 for (queue = 0; queue < txq_number; queue++) { 3140 int err = mvneta_txq_init(pp, &pp->txqs[queue]); 3141 if (err) { 3142 netdev_err(pp->dev, "%s: can't create txq=%d\n", 3143 __func__, queue); 3144 mvneta_cleanup_txqs(pp); 3145 return err; 3146 } 3147 } 3148 3149 return 0; 3150 } 3151 3152 static int mvneta_comphy_init(struct mvneta_port *pp) 3153 { 3154 int ret; 3155 3156 if (!pp->comphy) 3157 return 0; 3158 3159 ret = phy_set_mode_ext(pp->comphy, PHY_MODE_ETHERNET, 3160 pp->phy_interface); 3161 if (ret) 3162 return ret; 3163 3164 return phy_power_on(pp->comphy); 3165 } 3166 3167 static void mvneta_start_dev(struct mvneta_port *pp) 3168 { 3169 int cpu; 3170 3171 WARN_ON(mvneta_comphy_init(pp)); 3172 3173 mvneta_max_rx_size_set(pp, pp->pkt_size); 3174 mvneta_txq_max_tx_size_set(pp, pp->pkt_size); 3175 3176 /* start the Rx/Tx activity */ 3177 mvneta_port_enable(pp); 3178 3179 if (!pp->neta_armada3700) { 3180 /* Enable polling on the port */ 3181 for_each_online_cpu(cpu) { 3182 struct mvneta_pcpu_port *port = 3183 per_cpu_ptr(pp->ports, cpu); 3184 3185 napi_enable(&port->napi); 3186 } 3187 } else { 3188 napi_enable(&pp->napi); 3189 } 3190 3191 /* Unmask interrupts. It has to be done from each CPU */ 3192 on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true); 3193 3194 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 3195 MVNETA_CAUSE_PHY_STATUS_CHANGE | 3196 MVNETA_CAUSE_LINK_CHANGE); 3197 3198 phylink_start(pp->phylink); 3199 netif_tx_start_all_queues(pp->dev); 3200 } 3201 3202 static void mvneta_stop_dev(struct mvneta_port *pp) 3203 { 3204 unsigned int cpu; 3205 3206 phylink_stop(pp->phylink); 3207 3208 if (!pp->neta_armada3700) { 3209 for_each_online_cpu(cpu) { 3210 struct mvneta_pcpu_port *port = 3211 per_cpu_ptr(pp->ports, cpu); 3212 3213 napi_disable(&port->napi); 3214 } 3215 } else { 3216 napi_disable(&pp->napi); 3217 } 3218 3219 netif_carrier_off(pp->dev); 3220 3221 mvneta_port_down(pp); 3222 netif_tx_stop_all_queues(pp->dev); 3223 3224 /* Stop the port activity */ 3225 mvneta_port_disable(pp); 3226 3227 /* Clear all ethernet port interrupts */ 3228 on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true); 3229 3230 /* Mask all ethernet port interrupts */ 3231 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3232 3233 mvneta_tx_reset(pp); 3234 mvneta_rx_reset(pp); 3235 3236 WARN_ON(phy_power_off(pp->comphy)); 3237 } 3238 3239 static void mvneta_percpu_enable(void *arg) 3240 { 3241 struct mvneta_port *pp = arg; 3242 3243 enable_percpu_irq(pp->dev->irq, IRQ_TYPE_NONE); 3244 } 3245 3246 static void mvneta_percpu_disable(void *arg) 3247 { 3248 struct mvneta_port *pp = arg; 3249 3250 disable_percpu_irq(pp->dev->irq); 3251 } 3252 3253 /* Change the device mtu */ 3254 static int mvneta_change_mtu(struct net_device *dev, int mtu) 3255 { 3256 struct mvneta_port *pp = netdev_priv(dev); 3257 int ret; 3258 3259 if (!IS_ALIGNED(MVNETA_RX_PKT_SIZE(mtu), 8)) { 3260 netdev_info(dev, "Illegal MTU value %d, rounding to %d\n", 3261 mtu, ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8)); 3262 mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8); 3263 } 3264 3265 dev->mtu = mtu; 3266 3267 if (!netif_running(dev)) { 3268 if (pp->bm_priv) 3269 mvneta_bm_update_mtu(pp, mtu); 3270 3271 netdev_update_features(dev); 3272 return 0; 3273 } 3274 3275 /* The interface is running, so we have to force a 3276 * reallocation of the queues 3277 */ 3278 mvneta_stop_dev(pp); 3279 on_each_cpu(mvneta_percpu_disable, pp, true); 3280 3281 mvneta_cleanup_txqs(pp); 3282 mvneta_cleanup_rxqs(pp); 3283 3284 if (pp->bm_priv) 3285 mvneta_bm_update_mtu(pp, mtu); 3286 3287 pp->pkt_size = MVNETA_RX_PKT_SIZE(dev->mtu); 3288 3289 ret = mvneta_setup_rxqs(pp); 3290 if (ret) { 3291 netdev_err(dev, "unable to setup rxqs after MTU change\n"); 3292 return ret; 3293 } 3294 3295 ret = mvneta_setup_txqs(pp); 3296 if (ret) { 3297 netdev_err(dev, "unable to setup txqs after MTU change\n"); 3298 return ret; 3299 } 3300 3301 on_each_cpu(mvneta_percpu_enable, pp, true); 3302 mvneta_start_dev(pp); 3303 3304 netdev_update_features(dev); 3305 3306 return 0; 3307 } 3308 3309 static netdev_features_t mvneta_fix_features(struct net_device *dev, 3310 netdev_features_t features) 3311 { 3312 struct mvneta_port *pp = netdev_priv(dev); 3313 3314 if (pp->tx_csum_limit && dev->mtu > pp->tx_csum_limit) { 3315 features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO); 3316 netdev_info(dev, 3317 "Disable IP checksum for MTU greater than %dB\n", 3318 pp->tx_csum_limit); 3319 } 3320 3321 return features; 3322 } 3323 3324 /* Get mac address */ 3325 static void mvneta_get_mac_addr(struct mvneta_port *pp, unsigned char *addr) 3326 { 3327 u32 mac_addr_l, mac_addr_h; 3328 3329 mac_addr_l = mvreg_read(pp, MVNETA_MAC_ADDR_LOW); 3330 mac_addr_h = mvreg_read(pp, MVNETA_MAC_ADDR_HIGH); 3331 addr[0] = (mac_addr_h >> 24) & 0xFF; 3332 addr[1] = (mac_addr_h >> 16) & 0xFF; 3333 addr[2] = (mac_addr_h >> 8) & 0xFF; 3334 addr[3] = mac_addr_h & 0xFF; 3335 addr[4] = (mac_addr_l >> 8) & 0xFF; 3336 addr[5] = mac_addr_l & 0xFF; 3337 } 3338 3339 /* Handle setting mac address */ 3340 static int mvneta_set_mac_addr(struct net_device *dev, void *addr) 3341 { 3342 struct mvneta_port *pp = netdev_priv(dev); 3343 struct sockaddr *sockaddr = addr; 3344 int ret; 3345 3346 ret = eth_prepare_mac_addr_change(dev, addr); 3347 if (ret < 0) 3348 return ret; 3349 /* Remove previous address table entry */ 3350 mvneta_mac_addr_set(pp, dev->dev_addr, -1); 3351 3352 /* Set new addr in hw */ 3353 mvneta_mac_addr_set(pp, sockaddr->sa_data, pp->rxq_def); 3354 3355 eth_commit_mac_addr_change(dev, addr); 3356 return 0; 3357 } 3358 3359 static void mvneta_validate(struct net_device *ndev, unsigned long *supported, 3360 struct phylink_link_state *state) 3361 { 3362 struct mvneta_port *pp = netdev_priv(ndev); 3363 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; 3364 3365 /* We only support QSGMII, SGMII, 802.3z and RGMII modes */ 3366 if (state->interface != PHY_INTERFACE_MODE_NA && 3367 state->interface != PHY_INTERFACE_MODE_QSGMII && 3368 state->interface != PHY_INTERFACE_MODE_SGMII && 3369 !phy_interface_mode_is_8023z(state->interface) && 3370 !phy_interface_mode_is_rgmii(state->interface)) { 3371 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS); 3372 return; 3373 } 3374 3375 /* Allow all the expected bits */ 3376 phylink_set(mask, Autoneg); 3377 phylink_set_port_modes(mask); 3378 3379 /* Asymmetric pause is unsupported */ 3380 phylink_set(mask, Pause); 3381 3382 /* Half-duplex at speeds higher than 100Mbit is unsupported */ 3383 if (pp->comphy || state->interface != PHY_INTERFACE_MODE_2500BASEX) { 3384 phylink_set(mask, 1000baseT_Full); 3385 phylink_set(mask, 1000baseX_Full); 3386 } 3387 if (pp->comphy || state->interface == PHY_INTERFACE_MODE_2500BASEX) { 3388 phylink_set(mask, 2500baseT_Full); 3389 phylink_set(mask, 2500baseX_Full); 3390 } 3391 3392 if (!phy_interface_mode_is_8023z(state->interface)) { 3393 /* 10M and 100M are only supported in non-802.3z mode */ 3394 phylink_set(mask, 10baseT_Half); 3395 phylink_set(mask, 10baseT_Full); 3396 phylink_set(mask, 100baseT_Half); 3397 phylink_set(mask, 100baseT_Full); 3398 } 3399 3400 bitmap_and(supported, supported, mask, 3401 __ETHTOOL_LINK_MODE_MASK_NBITS); 3402 bitmap_and(state->advertising, state->advertising, mask, 3403 __ETHTOOL_LINK_MODE_MASK_NBITS); 3404 3405 /* We can only operate at 2500BaseX or 1000BaseX. If requested 3406 * to advertise both, only report advertising at 2500BaseX. 3407 */ 3408 phylink_helper_basex_speed(state); 3409 } 3410 3411 static int mvneta_mac_link_state(struct net_device *ndev, 3412 struct phylink_link_state *state) 3413 { 3414 struct mvneta_port *pp = netdev_priv(ndev); 3415 u32 gmac_stat; 3416 3417 gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS); 3418 3419 if (gmac_stat & MVNETA_GMAC_SPEED_1000) 3420 state->speed = 3421 state->interface == PHY_INTERFACE_MODE_2500BASEX ? 3422 SPEED_2500 : SPEED_1000; 3423 else if (gmac_stat & MVNETA_GMAC_SPEED_100) 3424 state->speed = SPEED_100; 3425 else 3426 state->speed = SPEED_10; 3427 3428 state->an_complete = !!(gmac_stat & MVNETA_GMAC_AN_COMPLETE); 3429 state->link = !!(gmac_stat & MVNETA_GMAC_LINK_UP); 3430 state->duplex = !!(gmac_stat & MVNETA_GMAC_FULL_DUPLEX); 3431 3432 state->pause = 0; 3433 if (gmac_stat & MVNETA_GMAC_RX_FLOW_CTRL_ENABLE) 3434 state->pause |= MLO_PAUSE_RX; 3435 if (gmac_stat & MVNETA_GMAC_TX_FLOW_CTRL_ENABLE) 3436 state->pause |= MLO_PAUSE_TX; 3437 3438 return 1; 3439 } 3440 3441 static void mvneta_mac_an_restart(struct net_device *ndev) 3442 { 3443 struct mvneta_port *pp = netdev_priv(ndev); 3444 u32 gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3445 3446 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, 3447 gmac_an | MVNETA_GMAC_INBAND_RESTART_AN); 3448 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, 3449 gmac_an & ~MVNETA_GMAC_INBAND_RESTART_AN); 3450 } 3451 3452 static void mvneta_mac_config(struct net_device *ndev, unsigned int mode, 3453 const struct phylink_link_state *state) 3454 { 3455 struct mvneta_port *pp = netdev_priv(ndev); 3456 u32 new_ctrl0, gmac_ctrl0 = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 3457 u32 new_ctrl2, gmac_ctrl2 = mvreg_read(pp, MVNETA_GMAC_CTRL_2); 3458 u32 new_ctrl4, gmac_ctrl4 = mvreg_read(pp, MVNETA_GMAC_CTRL_4); 3459 u32 new_clk, gmac_clk = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER); 3460 u32 new_an, gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3461 3462 new_ctrl0 = gmac_ctrl0 & ~MVNETA_GMAC0_PORT_1000BASE_X; 3463 new_ctrl2 = gmac_ctrl2 & ~(MVNETA_GMAC2_INBAND_AN_ENABLE | 3464 MVNETA_GMAC2_PORT_RESET); 3465 new_ctrl4 = gmac_ctrl4 & ~(MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE); 3466 new_clk = gmac_clk & ~MVNETA_GMAC_1MS_CLOCK_ENABLE; 3467 new_an = gmac_an & ~(MVNETA_GMAC_INBAND_AN_ENABLE | 3468 MVNETA_GMAC_INBAND_RESTART_AN | 3469 MVNETA_GMAC_CONFIG_MII_SPEED | 3470 MVNETA_GMAC_CONFIG_GMII_SPEED | 3471 MVNETA_GMAC_AN_SPEED_EN | 3472 MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL | 3473 MVNETA_GMAC_CONFIG_FLOW_CTRL | 3474 MVNETA_GMAC_AN_FLOW_CTRL_EN | 3475 MVNETA_GMAC_CONFIG_FULL_DUPLEX | 3476 MVNETA_GMAC_AN_DUPLEX_EN); 3477 3478 /* Even though it might look weird, when we're configured in 3479 * SGMII or QSGMII mode, the RGMII bit needs to be set. 3480 */ 3481 new_ctrl2 |= MVNETA_GMAC2_PORT_RGMII; 3482 3483 if (state->interface == PHY_INTERFACE_MODE_QSGMII || 3484 state->interface == PHY_INTERFACE_MODE_SGMII || 3485 phy_interface_mode_is_8023z(state->interface)) 3486 new_ctrl2 |= MVNETA_GMAC2_PCS_ENABLE; 3487 3488 if (phylink_test(state->advertising, Pause)) 3489 new_an |= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL; 3490 if (state->pause & MLO_PAUSE_TXRX_MASK) 3491 new_an |= MVNETA_GMAC_CONFIG_FLOW_CTRL; 3492 3493 if (!phylink_autoneg_inband(mode)) { 3494 /* Phy or fixed speed */ 3495 if (state->duplex) 3496 new_an |= MVNETA_GMAC_CONFIG_FULL_DUPLEX; 3497 3498 if (state->speed == SPEED_1000 || state->speed == SPEED_2500) 3499 new_an |= MVNETA_GMAC_CONFIG_GMII_SPEED; 3500 else if (state->speed == SPEED_100) 3501 new_an |= MVNETA_GMAC_CONFIG_MII_SPEED; 3502 } else if (state->interface == PHY_INTERFACE_MODE_SGMII) { 3503 /* SGMII mode receives the state from the PHY */ 3504 new_ctrl2 |= MVNETA_GMAC2_INBAND_AN_ENABLE; 3505 new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE; 3506 new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN | 3507 MVNETA_GMAC_FORCE_LINK_PASS)) | 3508 MVNETA_GMAC_INBAND_AN_ENABLE | 3509 MVNETA_GMAC_AN_SPEED_EN | 3510 MVNETA_GMAC_AN_DUPLEX_EN; 3511 } else { 3512 /* 802.3z negotiation - only 1000base-X */ 3513 new_ctrl0 |= MVNETA_GMAC0_PORT_1000BASE_X; 3514 new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE; 3515 new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN | 3516 MVNETA_GMAC_FORCE_LINK_PASS)) | 3517 MVNETA_GMAC_INBAND_AN_ENABLE | 3518 MVNETA_GMAC_CONFIG_GMII_SPEED | 3519 /* The MAC only supports FD mode */ 3520 MVNETA_GMAC_CONFIG_FULL_DUPLEX; 3521 3522 if (state->pause & MLO_PAUSE_AN && state->an_enabled) 3523 new_an |= MVNETA_GMAC_AN_FLOW_CTRL_EN; 3524 } 3525 3526 /* Armada 370 documentation says we can only change the port mode 3527 * and in-band enable when the link is down, so force it down 3528 * while making these changes. We also do this for GMAC_CTRL2 */ 3529 if ((new_ctrl0 ^ gmac_ctrl0) & MVNETA_GMAC0_PORT_1000BASE_X || 3530 (new_ctrl2 ^ gmac_ctrl2) & MVNETA_GMAC2_INBAND_AN_ENABLE || 3531 (new_an ^ gmac_an) & MVNETA_GMAC_INBAND_AN_ENABLE) { 3532 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, 3533 (gmac_an & ~MVNETA_GMAC_FORCE_LINK_PASS) | 3534 MVNETA_GMAC_FORCE_LINK_DOWN); 3535 } 3536 3537 3538 /* When at 2.5G, the link partner can send frames with shortened 3539 * preambles. 3540 */ 3541 if (state->speed == SPEED_2500) 3542 new_ctrl4 |= MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE; 3543 3544 if (pp->comphy && pp->phy_interface != state->interface && 3545 (state->interface == PHY_INTERFACE_MODE_SGMII || 3546 state->interface == PHY_INTERFACE_MODE_1000BASEX || 3547 state->interface == PHY_INTERFACE_MODE_2500BASEX)) { 3548 pp->phy_interface = state->interface; 3549 3550 WARN_ON(phy_power_off(pp->comphy)); 3551 WARN_ON(mvneta_comphy_init(pp)); 3552 } 3553 3554 if (new_ctrl0 != gmac_ctrl0) 3555 mvreg_write(pp, MVNETA_GMAC_CTRL_0, new_ctrl0); 3556 if (new_ctrl2 != gmac_ctrl2) 3557 mvreg_write(pp, MVNETA_GMAC_CTRL_2, new_ctrl2); 3558 if (new_ctrl4 != gmac_ctrl4) 3559 mvreg_write(pp, MVNETA_GMAC_CTRL_4, new_ctrl4); 3560 if (new_clk != gmac_clk) 3561 mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, new_clk); 3562 if (new_an != gmac_an) 3563 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, new_an); 3564 3565 if (gmac_ctrl2 & MVNETA_GMAC2_PORT_RESET) { 3566 while ((mvreg_read(pp, MVNETA_GMAC_CTRL_2) & 3567 MVNETA_GMAC2_PORT_RESET) != 0) 3568 continue; 3569 } 3570 } 3571 3572 static void mvneta_set_eee(struct mvneta_port *pp, bool enable) 3573 { 3574 u32 lpi_ctl1; 3575 3576 lpi_ctl1 = mvreg_read(pp, MVNETA_LPI_CTRL_1); 3577 if (enable) 3578 lpi_ctl1 |= MVNETA_LPI_REQUEST_ENABLE; 3579 else 3580 lpi_ctl1 &= ~MVNETA_LPI_REQUEST_ENABLE; 3581 mvreg_write(pp, MVNETA_LPI_CTRL_1, lpi_ctl1); 3582 } 3583 3584 static void mvneta_mac_link_down(struct net_device *ndev, unsigned int mode, 3585 phy_interface_t interface) 3586 { 3587 struct mvneta_port *pp = netdev_priv(ndev); 3588 u32 val; 3589 3590 mvneta_port_down(pp); 3591 3592 if (!phylink_autoneg_inband(mode)) { 3593 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3594 val &= ~MVNETA_GMAC_FORCE_LINK_PASS; 3595 val |= MVNETA_GMAC_FORCE_LINK_DOWN; 3596 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); 3597 } 3598 3599 pp->eee_active = false; 3600 mvneta_set_eee(pp, false); 3601 } 3602 3603 static void mvneta_mac_link_up(struct net_device *ndev, unsigned int mode, 3604 phy_interface_t interface, 3605 struct phy_device *phy) 3606 { 3607 struct mvneta_port *pp = netdev_priv(ndev); 3608 u32 val; 3609 3610 if (!phylink_autoneg_inband(mode)) { 3611 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3612 val &= ~MVNETA_GMAC_FORCE_LINK_DOWN; 3613 val |= MVNETA_GMAC_FORCE_LINK_PASS; 3614 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); 3615 } 3616 3617 mvneta_port_up(pp); 3618 3619 if (phy && pp->eee_enabled) { 3620 pp->eee_active = phy_init_eee(phy, 0) >= 0; 3621 mvneta_set_eee(pp, pp->eee_active && pp->tx_lpi_enabled); 3622 } 3623 } 3624 3625 static const struct phylink_mac_ops mvneta_phylink_ops = { 3626 .validate = mvneta_validate, 3627 .mac_link_state = mvneta_mac_link_state, 3628 .mac_an_restart = mvneta_mac_an_restart, 3629 .mac_config = mvneta_mac_config, 3630 .mac_link_down = mvneta_mac_link_down, 3631 .mac_link_up = mvneta_mac_link_up, 3632 }; 3633 3634 static int mvneta_mdio_probe(struct mvneta_port *pp) 3635 { 3636 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; 3637 int err = phylink_of_phy_connect(pp->phylink, pp->dn, 0); 3638 3639 if (err) 3640 netdev_err(pp->dev, "could not attach PHY: %d\n", err); 3641 3642 phylink_ethtool_get_wol(pp->phylink, &wol); 3643 device_set_wakeup_capable(&pp->dev->dev, !!wol.supported); 3644 3645 return err; 3646 } 3647 3648 static void mvneta_mdio_remove(struct mvneta_port *pp) 3649 { 3650 phylink_disconnect_phy(pp->phylink); 3651 } 3652 3653 /* Electing a CPU must be done in an atomic way: it should be done 3654 * after or before the removal/insertion of a CPU and this function is 3655 * not reentrant. 3656 */ 3657 static void mvneta_percpu_elect(struct mvneta_port *pp) 3658 { 3659 int elected_cpu = 0, max_cpu, cpu, i = 0; 3660 3661 /* Use the cpu associated to the rxq when it is online, in all 3662 * the other cases, use the cpu 0 which can't be offline. 3663 */ 3664 if (cpu_online(pp->rxq_def)) 3665 elected_cpu = pp->rxq_def; 3666 3667 max_cpu = num_present_cpus(); 3668 3669 for_each_online_cpu(cpu) { 3670 int rxq_map = 0, txq_map = 0; 3671 int rxq; 3672 3673 for (rxq = 0; rxq < rxq_number; rxq++) 3674 if ((rxq % max_cpu) == cpu) 3675 rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq); 3676 3677 if (cpu == elected_cpu) 3678 /* Map the default receive queue queue to the 3679 * elected CPU 3680 */ 3681 rxq_map |= MVNETA_CPU_RXQ_ACCESS(pp->rxq_def); 3682 3683 /* We update the TX queue map only if we have one 3684 * queue. In this case we associate the TX queue to 3685 * the CPU bound to the default RX queue 3686 */ 3687 if (txq_number == 1) 3688 txq_map = (cpu == elected_cpu) ? 3689 MVNETA_CPU_TXQ_ACCESS(1) : 0; 3690 else 3691 txq_map = mvreg_read(pp, MVNETA_CPU_MAP(cpu)) & 3692 MVNETA_CPU_TXQ_ACCESS_ALL_MASK; 3693 3694 mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map); 3695 3696 /* Update the interrupt mask on each CPU according the 3697 * new mapping 3698 */ 3699 smp_call_function_single(cpu, mvneta_percpu_unmask_interrupt, 3700 pp, true); 3701 i++; 3702 3703 } 3704 }; 3705 3706 static int mvneta_cpu_online(unsigned int cpu, struct hlist_node *node) 3707 { 3708 int other_cpu; 3709 struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port, 3710 node_online); 3711 struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); 3712 3713 3714 spin_lock(&pp->lock); 3715 /* 3716 * Configuring the driver for a new CPU while the driver is 3717 * stopping is racy, so just avoid it. 3718 */ 3719 if (pp->is_stopped) { 3720 spin_unlock(&pp->lock); 3721 return 0; 3722 } 3723 netif_tx_stop_all_queues(pp->dev); 3724 3725 /* 3726 * We have to synchronise on tha napi of each CPU except the one 3727 * just being woken up 3728 */ 3729 for_each_online_cpu(other_cpu) { 3730 if (other_cpu != cpu) { 3731 struct mvneta_pcpu_port *other_port = 3732 per_cpu_ptr(pp->ports, other_cpu); 3733 3734 napi_synchronize(&other_port->napi); 3735 } 3736 } 3737 3738 /* Mask all ethernet port interrupts */ 3739 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3740 napi_enable(&port->napi); 3741 3742 /* 3743 * Enable per-CPU interrupts on the CPU that is 3744 * brought up. 3745 */ 3746 mvneta_percpu_enable(pp); 3747 3748 /* 3749 * Enable per-CPU interrupt on the one CPU we care 3750 * about. 3751 */ 3752 mvneta_percpu_elect(pp); 3753 3754 /* Unmask all ethernet port interrupts */ 3755 on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true); 3756 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 3757 MVNETA_CAUSE_PHY_STATUS_CHANGE | 3758 MVNETA_CAUSE_LINK_CHANGE); 3759 netif_tx_start_all_queues(pp->dev); 3760 spin_unlock(&pp->lock); 3761 return 0; 3762 } 3763 3764 static int mvneta_cpu_down_prepare(unsigned int cpu, struct hlist_node *node) 3765 { 3766 struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port, 3767 node_online); 3768 struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); 3769 3770 /* 3771 * Thanks to this lock we are sure that any pending cpu election is 3772 * done. 3773 */ 3774 spin_lock(&pp->lock); 3775 /* Mask all ethernet port interrupts */ 3776 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3777 spin_unlock(&pp->lock); 3778 3779 napi_synchronize(&port->napi); 3780 napi_disable(&port->napi); 3781 /* Disable per-CPU interrupts on the CPU that is brought down. */ 3782 mvneta_percpu_disable(pp); 3783 return 0; 3784 } 3785 3786 static int mvneta_cpu_dead(unsigned int cpu, struct hlist_node *node) 3787 { 3788 struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port, 3789 node_dead); 3790 3791 /* Check if a new CPU must be elected now this on is down */ 3792 spin_lock(&pp->lock); 3793 mvneta_percpu_elect(pp); 3794 spin_unlock(&pp->lock); 3795 /* Unmask all ethernet port interrupts */ 3796 on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true); 3797 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 3798 MVNETA_CAUSE_PHY_STATUS_CHANGE | 3799 MVNETA_CAUSE_LINK_CHANGE); 3800 netif_tx_start_all_queues(pp->dev); 3801 return 0; 3802 } 3803 3804 static int mvneta_open(struct net_device *dev) 3805 { 3806 struct mvneta_port *pp = netdev_priv(dev); 3807 int ret; 3808 3809 pp->pkt_size = MVNETA_RX_PKT_SIZE(pp->dev->mtu); 3810 3811 ret = mvneta_setup_rxqs(pp); 3812 if (ret) 3813 return ret; 3814 3815 ret = mvneta_setup_txqs(pp); 3816 if (ret) 3817 goto err_cleanup_rxqs; 3818 3819 /* Connect to port interrupt line */ 3820 if (pp->neta_armada3700) 3821 ret = request_irq(pp->dev->irq, mvneta_isr, 0, 3822 dev->name, pp); 3823 else 3824 ret = request_percpu_irq(pp->dev->irq, mvneta_percpu_isr, 3825 dev->name, pp->ports); 3826 if (ret) { 3827 netdev_err(pp->dev, "cannot request irq %d\n", pp->dev->irq); 3828 goto err_cleanup_txqs; 3829 } 3830 3831 if (!pp->neta_armada3700) { 3832 /* Enable per-CPU interrupt on all the CPU to handle our RX 3833 * queue interrupts 3834 */ 3835 on_each_cpu(mvneta_percpu_enable, pp, true); 3836 3837 pp->is_stopped = false; 3838 /* Register a CPU notifier to handle the case where our CPU 3839 * might be taken offline. 3840 */ 3841 ret = cpuhp_state_add_instance_nocalls(online_hpstate, 3842 &pp->node_online); 3843 if (ret) 3844 goto err_free_irq; 3845 3846 ret = cpuhp_state_add_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 3847 &pp->node_dead); 3848 if (ret) 3849 goto err_free_online_hp; 3850 } 3851 3852 ret = mvneta_mdio_probe(pp); 3853 if (ret < 0) { 3854 netdev_err(dev, "cannot probe MDIO bus\n"); 3855 goto err_free_dead_hp; 3856 } 3857 3858 mvneta_start_dev(pp); 3859 3860 return 0; 3861 3862 err_free_dead_hp: 3863 if (!pp->neta_armada3700) 3864 cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 3865 &pp->node_dead); 3866 err_free_online_hp: 3867 if (!pp->neta_armada3700) 3868 cpuhp_state_remove_instance_nocalls(online_hpstate, 3869 &pp->node_online); 3870 err_free_irq: 3871 if (pp->neta_armada3700) { 3872 free_irq(pp->dev->irq, pp); 3873 } else { 3874 on_each_cpu(mvneta_percpu_disable, pp, true); 3875 free_percpu_irq(pp->dev->irq, pp->ports); 3876 } 3877 err_cleanup_txqs: 3878 mvneta_cleanup_txqs(pp); 3879 err_cleanup_rxqs: 3880 mvneta_cleanup_rxqs(pp); 3881 return ret; 3882 } 3883 3884 /* Stop the port, free port interrupt line */ 3885 static int mvneta_stop(struct net_device *dev) 3886 { 3887 struct mvneta_port *pp = netdev_priv(dev); 3888 3889 if (!pp->neta_armada3700) { 3890 /* Inform that we are stopping so we don't want to setup the 3891 * driver for new CPUs in the notifiers. The code of the 3892 * notifier for CPU online is protected by the same spinlock, 3893 * so when we get the lock, the notifer work is done. 3894 */ 3895 spin_lock(&pp->lock); 3896 pp->is_stopped = true; 3897 spin_unlock(&pp->lock); 3898 3899 mvneta_stop_dev(pp); 3900 mvneta_mdio_remove(pp); 3901 3902 cpuhp_state_remove_instance_nocalls(online_hpstate, 3903 &pp->node_online); 3904 cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 3905 &pp->node_dead); 3906 on_each_cpu(mvneta_percpu_disable, pp, true); 3907 free_percpu_irq(dev->irq, pp->ports); 3908 } else { 3909 mvneta_stop_dev(pp); 3910 mvneta_mdio_remove(pp); 3911 free_irq(dev->irq, pp); 3912 } 3913 3914 mvneta_cleanup_rxqs(pp); 3915 mvneta_cleanup_txqs(pp); 3916 3917 return 0; 3918 } 3919 3920 static int mvneta_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 3921 { 3922 struct mvneta_port *pp = netdev_priv(dev); 3923 3924 return phylink_mii_ioctl(pp->phylink, ifr, cmd); 3925 } 3926 3927 /* Ethtool methods */ 3928 3929 /* Set link ksettings (phy address, speed) for ethtools */ 3930 static int 3931 mvneta_ethtool_set_link_ksettings(struct net_device *ndev, 3932 const struct ethtool_link_ksettings *cmd) 3933 { 3934 struct mvneta_port *pp = netdev_priv(ndev); 3935 3936 return phylink_ethtool_ksettings_set(pp->phylink, cmd); 3937 } 3938 3939 /* Get link ksettings for ethtools */ 3940 static int 3941 mvneta_ethtool_get_link_ksettings(struct net_device *ndev, 3942 struct ethtool_link_ksettings *cmd) 3943 { 3944 struct mvneta_port *pp = netdev_priv(ndev); 3945 3946 return phylink_ethtool_ksettings_get(pp->phylink, cmd); 3947 } 3948 3949 static int mvneta_ethtool_nway_reset(struct net_device *dev) 3950 { 3951 struct mvneta_port *pp = netdev_priv(dev); 3952 3953 return phylink_ethtool_nway_reset(pp->phylink); 3954 } 3955 3956 /* Set interrupt coalescing for ethtools */ 3957 static int mvneta_ethtool_set_coalesce(struct net_device *dev, 3958 struct ethtool_coalesce *c) 3959 { 3960 struct mvneta_port *pp = netdev_priv(dev); 3961 int queue; 3962 3963 for (queue = 0; queue < rxq_number; queue++) { 3964 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 3965 rxq->time_coal = c->rx_coalesce_usecs; 3966 rxq->pkts_coal = c->rx_max_coalesced_frames; 3967 mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal); 3968 mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal); 3969 } 3970 3971 for (queue = 0; queue < txq_number; queue++) { 3972 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 3973 txq->done_pkts_coal = c->tx_max_coalesced_frames; 3974 mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal); 3975 } 3976 3977 return 0; 3978 } 3979 3980 /* get coalescing for ethtools */ 3981 static int mvneta_ethtool_get_coalesce(struct net_device *dev, 3982 struct ethtool_coalesce *c) 3983 { 3984 struct mvneta_port *pp = netdev_priv(dev); 3985 3986 c->rx_coalesce_usecs = pp->rxqs[0].time_coal; 3987 c->rx_max_coalesced_frames = pp->rxqs[0].pkts_coal; 3988 3989 c->tx_max_coalesced_frames = pp->txqs[0].done_pkts_coal; 3990 return 0; 3991 } 3992 3993 3994 static void mvneta_ethtool_get_drvinfo(struct net_device *dev, 3995 struct ethtool_drvinfo *drvinfo) 3996 { 3997 strlcpy(drvinfo->driver, MVNETA_DRIVER_NAME, 3998 sizeof(drvinfo->driver)); 3999 strlcpy(drvinfo->version, MVNETA_DRIVER_VERSION, 4000 sizeof(drvinfo->version)); 4001 strlcpy(drvinfo->bus_info, dev_name(&dev->dev), 4002 sizeof(drvinfo->bus_info)); 4003 } 4004 4005 4006 static void mvneta_ethtool_get_ringparam(struct net_device *netdev, 4007 struct ethtool_ringparam *ring) 4008 { 4009 struct mvneta_port *pp = netdev_priv(netdev); 4010 4011 ring->rx_max_pending = MVNETA_MAX_RXD; 4012 ring->tx_max_pending = MVNETA_MAX_TXD; 4013 ring->rx_pending = pp->rx_ring_size; 4014 ring->tx_pending = pp->tx_ring_size; 4015 } 4016 4017 static int mvneta_ethtool_set_ringparam(struct net_device *dev, 4018 struct ethtool_ringparam *ring) 4019 { 4020 struct mvneta_port *pp = netdev_priv(dev); 4021 4022 if ((ring->rx_pending == 0) || (ring->tx_pending == 0)) 4023 return -EINVAL; 4024 pp->rx_ring_size = ring->rx_pending < MVNETA_MAX_RXD ? 4025 ring->rx_pending : MVNETA_MAX_RXD; 4026 4027 pp->tx_ring_size = clamp_t(u16, ring->tx_pending, 4028 MVNETA_MAX_SKB_DESCS * 2, MVNETA_MAX_TXD); 4029 if (pp->tx_ring_size != ring->tx_pending) 4030 netdev_warn(dev, "TX queue size set to %u (requested %u)\n", 4031 pp->tx_ring_size, ring->tx_pending); 4032 4033 if (netif_running(dev)) { 4034 mvneta_stop(dev); 4035 if (mvneta_open(dev)) { 4036 netdev_err(dev, 4037 "error on opening device after ring param change\n"); 4038 return -ENOMEM; 4039 } 4040 } 4041 4042 return 0; 4043 } 4044 4045 static void mvneta_ethtool_get_pauseparam(struct net_device *dev, 4046 struct ethtool_pauseparam *pause) 4047 { 4048 struct mvneta_port *pp = netdev_priv(dev); 4049 4050 phylink_ethtool_get_pauseparam(pp->phylink, pause); 4051 } 4052 4053 static int mvneta_ethtool_set_pauseparam(struct net_device *dev, 4054 struct ethtool_pauseparam *pause) 4055 { 4056 struct mvneta_port *pp = netdev_priv(dev); 4057 4058 return phylink_ethtool_set_pauseparam(pp->phylink, pause); 4059 } 4060 4061 static void mvneta_ethtool_get_strings(struct net_device *netdev, u32 sset, 4062 u8 *data) 4063 { 4064 if (sset == ETH_SS_STATS) { 4065 int i; 4066 4067 for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) 4068 memcpy(data + i * ETH_GSTRING_LEN, 4069 mvneta_statistics[i].name, ETH_GSTRING_LEN); 4070 } 4071 } 4072 4073 static void mvneta_ethtool_update_stats(struct mvneta_port *pp) 4074 { 4075 const struct mvneta_statistic *s; 4076 void __iomem *base = pp->base; 4077 u32 high, low; 4078 u64 val; 4079 int i; 4080 4081 for (i = 0, s = mvneta_statistics; 4082 s < mvneta_statistics + ARRAY_SIZE(mvneta_statistics); 4083 s++, i++) { 4084 val = 0; 4085 4086 switch (s->type) { 4087 case T_REG_32: 4088 val = readl_relaxed(base + s->offset); 4089 break; 4090 case T_REG_64: 4091 /* Docs say to read low 32-bit then high */ 4092 low = readl_relaxed(base + s->offset); 4093 high = readl_relaxed(base + s->offset + 4); 4094 val = (u64)high << 32 | low; 4095 break; 4096 case T_SW: 4097 switch (s->offset) { 4098 case ETHTOOL_STAT_EEE_WAKEUP: 4099 val = phylink_get_eee_err(pp->phylink); 4100 break; 4101 case ETHTOOL_STAT_SKB_ALLOC_ERR: 4102 val = pp->rxqs[0].skb_alloc_err; 4103 break; 4104 case ETHTOOL_STAT_REFILL_ERR: 4105 val = pp->rxqs[0].refill_err; 4106 break; 4107 } 4108 break; 4109 } 4110 4111 pp->ethtool_stats[i] += val; 4112 } 4113 } 4114 4115 static void mvneta_ethtool_get_stats(struct net_device *dev, 4116 struct ethtool_stats *stats, u64 *data) 4117 { 4118 struct mvneta_port *pp = netdev_priv(dev); 4119 int i; 4120 4121 mvneta_ethtool_update_stats(pp); 4122 4123 for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) 4124 *data++ = pp->ethtool_stats[i]; 4125 } 4126 4127 static int mvneta_ethtool_get_sset_count(struct net_device *dev, int sset) 4128 { 4129 if (sset == ETH_SS_STATS) 4130 return ARRAY_SIZE(mvneta_statistics); 4131 return -EOPNOTSUPP; 4132 } 4133 4134 static u32 mvneta_ethtool_get_rxfh_indir_size(struct net_device *dev) 4135 { 4136 return MVNETA_RSS_LU_TABLE_SIZE; 4137 } 4138 4139 static int mvneta_ethtool_get_rxnfc(struct net_device *dev, 4140 struct ethtool_rxnfc *info, 4141 u32 *rules __always_unused) 4142 { 4143 switch (info->cmd) { 4144 case ETHTOOL_GRXRINGS: 4145 info->data = rxq_number; 4146 return 0; 4147 case ETHTOOL_GRXFH: 4148 return -EOPNOTSUPP; 4149 default: 4150 return -EOPNOTSUPP; 4151 } 4152 } 4153 4154 static int mvneta_config_rss(struct mvneta_port *pp) 4155 { 4156 int cpu; 4157 u32 val; 4158 4159 netif_tx_stop_all_queues(pp->dev); 4160 4161 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 4162 4163 if (!pp->neta_armada3700) { 4164 /* We have to synchronise on the napi of each CPU */ 4165 for_each_online_cpu(cpu) { 4166 struct mvneta_pcpu_port *pcpu_port = 4167 per_cpu_ptr(pp->ports, cpu); 4168 4169 napi_synchronize(&pcpu_port->napi); 4170 napi_disable(&pcpu_port->napi); 4171 } 4172 } else { 4173 napi_synchronize(&pp->napi); 4174 napi_disable(&pp->napi); 4175 } 4176 4177 pp->rxq_def = pp->indir[0]; 4178 4179 /* Update unicast mapping */ 4180 mvneta_set_rx_mode(pp->dev); 4181 4182 /* Update val of portCfg register accordingly with all RxQueue types */ 4183 val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def); 4184 mvreg_write(pp, MVNETA_PORT_CONFIG, val); 4185 4186 /* Update the elected CPU matching the new rxq_def */ 4187 spin_lock(&pp->lock); 4188 mvneta_percpu_elect(pp); 4189 spin_unlock(&pp->lock); 4190 4191 if (!pp->neta_armada3700) { 4192 /* We have to synchronise on the napi of each CPU */ 4193 for_each_online_cpu(cpu) { 4194 struct mvneta_pcpu_port *pcpu_port = 4195 per_cpu_ptr(pp->ports, cpu); 4196 4197 napi_enable(&pcpu_port->napi); 4198 } 4199 } else { 4200 napi_enable(&pp->napi); 4201 } 4202 4203 netif_tx_start_all_queues(pp->dev); 4204 4205 return 0; 4206 } 4207 4208 static int mvneta_ethtool_set_rxfh(struct net_device *dev, const u32 *indir, 4209 const u8 *key, const u8 hfunc) 4210 { 4211 struct mvneta_port *pp = netdev_priv(dev); 4212 4213 /* Current code for Armada 3700 doesn't support RSS features yet */ 4214 if (pp->neta_armada3700) 4215 return -EOPNOTSUPP; 4216 4217 /* We require at least one supported parameter to be changed 4218 * and no change in any of the unsupported parameters 4219 */ 4220 if (key || 4221 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) 4222 return -EOPNOTSUPP; 4223 4224 if (!indir) 4225 return 0; 4226 4227 memcpy(pp->indir, indir, MVNETA_RSS_LU_TABLE_SIZE); 4228 4229 return mvneta_config_rss(pp); 4230 } 4231 4232 static int mvneta_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, 4233 u8 *hfunc) 4234 { 4235 struct mvneta_port *pp = netdev_priv(dev); 4236 4237 /* Current code for Armada 3700 doesn't support RSS features yet */ 4238 if (pp->neta_armada3700) 4239 return -EOPNOTSUPP; 4240 4241 if (hfunc) 4242 *hfunc = ETH_RSS_HASH_TOP; 4243 4244 if (!indir) 4245 return 0; 4246 4247 memcpy(indir, pp->indir, MVNETA_RSS_LU_TABLE_SIZE); 4248 4249 return 0; 4250 } 4251 4252 static void mvneta_ethtool_get_wol(struct net_device *dev, 4253 struct ethtool_wolinfo *wol) 4254 { 4255 struct mvneta_port *pp = netdev_priv(dev); 4256 4257 phylink_ethtool_get_wol(pp->phylink, wol); 4258 } 4259 4260 static int mvneta_ethtool_set_wol(struct net_device *dev, 4261 struct ethtool_wolinfo *wol) 4262 { 4263 struct mvneta_port *pp = netdev_priv(dev); 4264 int ret; 4265 4266 ret = phylink_ethtool_set_wol(pp->phylink, wol); 4267 if (!ret) 4268 device_set_wakeup_enable(&dev->dev, !!wol->wolopts); 4269 4270 return ret; 4271 } 4272 4273 static int mvneta_ethtool_get_eee(struct net_device *dev, 4274 struct ethtool_eee *eee) 4275 { 4276 struct mvneta_port *pp = netdev_priv(dev); 4277 u32 lpi_ctl0; 4278 4279 lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0); 4280 4281 eee->eee_enabled = pp->eee_enabled; 4282 eee->eee_active = pp->eee_active; 4283 eee->tx_lpi_enabled = pp->tx_lpi_enabled; 4284 eee->tx_lpi_timer = (lpi_ctl0) >> 8; // * scale; 4285 4286 return phylink_ethtool_get_eee(pp->phylink, eee); 4287 } 4288 4289 static int mvneta_ethtool_set_eee(struct net_device *dev, 4290 struct ethtool_eee *eee) 4291 { 4292 struct mvneta_port *pp = netdev_priv(dev); 4293 u32 lpi_ctl0; 4294 4295 /* The Armada 37x documents do not give limits for this other than 4296 * it being an 8-bit register. */ 4297 if (eee->tx_lpi_enabled && eee->tx_lpi_timer > 255) 4298 return -EINVAL; 4299 4300 lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0); 4301 lpi_ctl0 &= ~(0xff << 8); 4302 lpi_ctl0 |= eee->tx_lpi_timer << 8; 4303 mvreg_write(pp, MVNETA_LPI_CTRL_0, lpi_ctl0); 4304 4305 pp->eee_enabled = eee->eee_enabled; 4306 pp->tx_lpi_enabled = eee->tx_lpi_enabled; 4307 4308 mvneta_set_eee(pp, eee->tx_lpi_enabled && eee->eee_enabled); 4309 4310 return phylink_ethtool_set_eee(pp->phylink, eee); 4311 } 4312 4313 static const struct net_device_ops mvneta_netdev_ops = { 4314 .ndo_open = mvneta_open, 4315 .ndo_stop = mvneta_stop, 4316 .ndo_start_xmit = mvneta_tx, 4317 .ndo_set_rx_mode = mvneta_set_rx_mode, 4318 .ndo_set_mac_address = mvneta_set_mac_addr, 4319 .ndo_change_mtu = mvneta_change_mtu, 4320 .ndo_fix_features = mvneta_fix_features, 4321 .ndo_get_stats64 = mvneta_get_stats64, 4322 .ndo_do_ioctl = mvneta_ioctl, 4323 }; 4324 4325 static const struct ethtool_ops mvneta_eth_tool_ops = { 4326 .nway_reset = mvneta_ethtool_nway_reset, 4327 .get_link = ethtool_op_get_link, 4328 .set_coalesce = mvneta_ethtool_set_coalesce, 4329 .get_coalesce = mvneta_ethtool_get_coalesce, 4330 .get_drvinfo = mvneta_ethtool_get_drvinfo, 4331 .get_ringparam = mvneta_ethtool_get_ringparam, 4332 .set_ringparam = mvneta_ethtool_set_ringparam, 4333 .get_pauseparam = mvneta_ethtool_get_pauseparam, 4334 .set_pauseparam = mvneta_ethtool_set_pauseparam, 4335 .get_strings = mvneta_ethtool_get_strings, 4336 .get_ethtool_stats = mvneta_ethtool_get_stats, 4337 .get_sset_count = mvneta_ethtool_get_sset_count, 4338 .get_rxfh_indir_size = mvneta_ethtool_get_rxfh_indir_size, 4339 .get_rxnfc = mvneta_ethtool_get_rxnfc, 4340 .get_rxfh = mvneta_ethtool_get_rxfh, 4341 .set_rxfh = mvneta_ethtool_set_rxfh, 4342 .get_link_ksettings = mvneta_ethtool_get_link_ksettings, 4343 .set_link_ksettings = mvneta_ethtool_set_link_ksettings, 4344 .get_wol = mvneta_ethtool_get_wol, 4345 .set_wol = mvneta_ethtool_set_wol, 4346 .get_eee = mvneta_ethtool_get_eee, 4347 .set_eee = mvneta_ethtool_set_eee, 4348 }; 4349 4350 /* Initialize hw */ 4351 static int mvneta_init(struct device *dev, struct mvneta_port *pp) 4352 { 4353 int queue; 4354 4355 /* Disable port */ 4356 mvneta_port_disable(pp); 4357 4358 /* Set port default values */ 4359 mvneta_defaults_set(pp); 4360 4361 pp->txqs = devm_kcalloc(dev, txq_number, sizeof(*pp->txqs), GFP_KERNEL); 4362 if (!pp->txqs) 4363 return -ENOMEM; 4364 4365 /* Initialize TX descriptor rings */ 4366 for (queue = 0; queue < txq_number; queue++) { 4367 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 4368 txq->id = queue; 4369 txq->size = pp->tx_ring_size; 4370 txq->done_pkts_coal = MVNETA_TXDONE_COAL_PKTS; 4371 } 4372 4373 pp->rxqs = devm_kcalloc(dev, rxq_number, sizeof(*pp->rxqs), GFP_KERNEL); 4374 if (!pp->rxqs) 4375 return -ENOMEM; 4376 4377 /* Create Rx descriptor rings */ 4378 for (queue = 0; queue < rxq_number; queue++) { 4379 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 4380 rxq->id = queue; 4381 rxq->size = pp->rx_ring_size; 4382 rxq->pkts_coal = MVNETA_RX_COAL_PKTS; 4383 rxq->time_coal = MVNETA_RX_COAL_USEC; 4384 rxq->buf_virt_addr 4385 = devm_kmalloc_array(pp->dev->dev.parent, 4386 rxq->size, 4387 sizeof(*rxq->buf_virt_addr), 4388 GFP_KERNEL); 4389 if (!rxq->buf_virt_addr) 4390 return -ENOMEM; 4391 } 4392 4393 return 0; 4394 } 4395 4396 /* platform glue : initialize decoding windows */ 4397 static void mvneta_conf_mbus_windows(struct mvneta_port *pp, 4398 const struct mbus_dram_target_info *dram) 4399 { 4400 u32 win_enable; 4401 u32 win_protect; 4402 int i; 4403 4404 for (i = 0; i < 6; i++) { 4405 mvreg_write(pp, MVNETA_WIN_BASE(i), 0); 4406 mvreg_write(pp, MVNETA_WIN_SIZE(i), 0); 4407 4408 if (i < 4) 4409 mvreg_write(pp, MVNETA_WIN_REMAP(i), 0); 4410 } 4411 4412 win_enable = 0x3f; 4413 win_protect = 0; 4414 4415 if (dram) { 4416 for (i = 0; i < dram->num_cs; i++) { 4417 const struct mbus_dram_window *cs = dram->cs + i; 4418 4419 mvreg_write(pp, MVNETA_WIN_BASE(i), 4420 (cs->base & 0xffff0000) | 4421 (cs->mbus_attr << 8) | 4422 dram->mbus_dram_target_id); 4423 4424 mvreg_write(pp, MVNETA_WIN_SIZE(i), 4425 (cs->size - 1) & 0xffff0000); 4426 4427 win_enable &= ~(1 << i); 4428 win_protect |= 3 << (2 * i); 4429 } 4430 } else { 4431 /* For Armada3700 open default 4GB Mbus window, leaving 4432 * arbitration of target/attribute to a different layer 4433 * of configuration. 4434 */ 4435 mvreg_write(pp, MVNETA_WIN_SIZE(0), 0xffff0000); 4436 win_enable &= ~BIT(0); 4437 win_protect = 3; 4438 } 4439 4440 mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable); 4441 mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect); 4442 } 4443 4444 /* Power up the port */ 4445 static int mvneta_port_power_up(struct mvneta_port *pp, int phy_mode) 4446 { 4447 /* MAC Cause register should be cleared */ 4448 mvreg_write(pp, MVNETA_UNIT_INTR_CAUSE, 0); 4449 4450 if (phy_mode == PHY_INTERFACE_MODE_QSGMII) 4451 mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_QSGMII_SERDES_PROTO); 4452 else if (phy_mode == PHY_INTERFACE_MODE_SGMII || 4453 phy_interface_mode_is_8023z(phy_mode)) 4454 mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_SGMII_SERDES_PROTO); 4455 else if (!phy_interface_mode_is_rgmii(phy_mode)) 4456 return -EINVAL; 4457 4458 return 0; 4459 } 4460 4461 /* Device initialization routine */ 4462 static int mvneta_probe(struct platform_device *pdev) 4463 { 4464 struct resource *res; 4465 struct device_node *dn = pdev->dev.of_node; 4466 struct device_node *bm_node; 4467 struct mvneta_port *pp; 4468 struct net_device *dev; 4469 struct phylink *phylink; 4470 struct phy *comphy; 4471 const char *dt_mac_addr; 4472 char hw_mac_addr[ETH_ALEN]; 4473 const char *mac_from; 4474 int tx_csum_limit; 4475 int phy_mode; 4476 int err; 4477 int cpu; 4478 4479 dev = devm_alloc_etherdev_mqs(&pdev->dev, sizeof(struct mvneta_port), 4480 txq_number, rxq_number); 4481 if (!dev) 4482 return -ENOMEM; 4483 4484 dev->irq = irq_of_parse_and_map(dn, 0); 4485 if (dev->irq == 0) 4486 return -EINVAL; 4487 4488 phy_mode = of_get_phy_mode(dn); 4489 if (phy_mode < 0) { 4490 dev_err(&pdev->dev, "incorrect phy-mode\n"); 4491 err = -EINVAL; 4492 goto err_free_irq; 4493 } 4494 4495 comphy = devm_of_phy_get(&pdev->dev, dn, NULL); 4496 if (comphy == ERR_PTR(-EPROBE_DEFER)) { 4497 err = -EPROBE_DEFER; 4498 goto err_free_irq; 4499 } else if (IS_ERR(comphy)) { 4500 comphy = NULL; 4501 } 4502 4503 phylink = phylink_create(dev, pdev->dev.fwnode, phy_mode, 4504 &mvneta_phylink_ops); 4505 if (IS_ERR(phylink)) { 4506 err = PTR_ERR(phylink); 4507 goto err_free_irq; 4508 } 4509 4510 dev->tx_queue_len = MVNETA_MAX_TXD; 4511 dev->watchdog_timeo = 5 * HZ; 4512 dev->netdev_ops = &mvneta_netdev_ops; 4513 4514 dev->ethtool_ops = &mvneta_eth_tool_ops; 4515 4516 pp = netdev_priv(dev); 4517 spin_lock_init(&pp->lock); 4518 pp->phylink = phylink; 4519 pp->comphy = comphy; 4520 pp->phy_interface = phy_mode; 4521 pp->dn = dn; 4522 4523 pp->rxq_def = rxq_def; 4524 pp->indir[0] = rxq_def; 4525 4526 /* Get special SoC configurations */ 4527 if (of_device_is_compatible(dn, "marvell,armada-3700-neta")) 4528 pp->neta_armada3700 = true; 4529 4530 pp->clk = devm_clk_get(&pdev->dev, "core"); 4531 if (IS_ERR(pp->clk)) 4532 pp->clk = devm_clk_get(&pdev->dev, NULL); 4533 if (IS_ERR(pp->clk)) { 4534 err = PTR_ERR(pp->clk); 4535 goto err_free_phylink; 4536 } 4537 4538 clk_prepare_enable(pp->clk); 4539 4540 pp->clk_bus = devm_clk_get(&pdev->dev, "bus"); 4541 if (!IS_ERR(pp->clk_bus)) 4542 clk_prepare_enable(pp->clk_bus); 4543 4544 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4545 pp->base = devm_ioremap_resource(&pdev->dev, res); 4546 if (IS_ERR(pp->base)) { 4547 err = PTR_ERR(pp->base); 4548 goto err_clk; 4549 } 4550 4551 /* Alloc per-cpu port structure */ 4552 pp->ports = alloc_percpu(struct mvneta_pcpu_port); 4553 if (!pp->ports) { 4554 err = -ENOMEM; 4555 goto err_clk; 4556 } 4557 4558 /* Alloc per-cpu stats */ 4559 pp->stats = netdev_alloc_pcpu_stats(struct mvneta_pcpu_stats); 4560 if (!pp->stats) { 4561 err = -ENOMEM; 4562 goto err_free_ports; 4563 } 4564 4565 dt_mac_addr = of_get_mac_address(dn); 4566 if (!IS_ERR(dt_mac_addr)) { 4567 mac_from = "device tree"; 4568 ether_addr_copy(dev->dev_addr, dt_mac_addr); 4569 } else { 4570 mvneta_get_mac_addr(pp, hw_mac_addr); 4571 if (is_valid_ether_addr(hw_mac_addr)) { 4572 mac_from = "hardware"; 4573 memcpy(dev->dev_addr, hw_mac_addr, ETH_ALEN); 4574 } else { 4575 mac_from = "random"; 4576 eth_hw_addr_random(dev); 4577 } 4578 } 4579 4580 if (!of_property_read_u32(dn, "tx-csum-limit", &tx_csum_limit)) { 4581 if (tx_csum_limit < 0 || 4582 tx_csum_limit > MVNETA_TX_CSUM_MAX_SIZE) { 4583 tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE; 4584 dev_info(&pdev->dev, 4585 "Wrong TX csum limit in DT, set to %dB\n", 4586 MVNETA_TX_CSUM_DEF_SIZE); 4587 } 4588 } else if (of_device_is_compatible(dn, "marvell,armada-370-neta")) { 4589 tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE; 4590 } else { 4591 tx_csum_limit = MVNETA_TX_CSUM_MAX_SIZE; 4592 } 4593 4594 pp->tx_csum_limit = tx_csum_limit; 4595 4596 pp->dram_target_info = mv_mbus_dram_info(); 4597 /* Armada3700 requires setting default configuration of Mbus 4598 * windows, however without using filled mbus_dram_target_info 4599 * structure. 4600 */ 4601 if (pp->dram_target_info || pp->neta_armada3700) 4602 mvneta_conf_mbus_windows(pp, pp->dram_target_info); 4603 4604 pp->tx_ring_size = MVNETA_MAX_TXD; 4605 pp->rx_ring_size = MVNETA_MAX_RXD; 4606 4607 pp->dev = dev; 4608 SET_NETDEV_DEV(dev, &pdev->dev); 4609 4610 pp->id = global_port_id++; 4611 pp->rx_offset_correction = 0; /* not relevant for SW BM */ 4612 4613 /* Obtain access to BM resources if enabled and already initialized */ 4614 bm_node = of_parse_phandle(dn, "buffer-manager", 0); 4615 if (bm_node) { 4616 pp->bm_priv = mvneta_bm_get(bm_node); 4617 if (pp->bm_priv) { 4618 err = mvneta_bm_port_init(pdev, pp); 4619 if (err < 0) { 4620 dev_info(&pdev->dev, 4621 "use SW buffer management\n"); 4622 mvneta_bm_put(pp->bm_priv); 4623 pp->bm_priv = NULL; 4624 } 4625 } 4626 /* Set RX packet offset correction for platforms, whose 4627 * NET_SKB_PAD, exceeds 64B. It should be 64B for 64-bit 4628 * platforms and 0B for 32-bit ones. 4629 */ 4630 pp->rx_offset_correction = max(0, 4631 NET_SKB_PAD - 4632 MVNETA_RX_PKT_OFFSET_CORRECTION); 4633 } 4634 of_node_put(bm_node); 4635 4636 err = mvneta_init(&pdev->dev, pp); 4637 if (err < 0) 4638 goto err_netdev; 4639 4640 err = mvneta_port_power_up(pp, phy_mode); 4641 if (err < 0) { 4642 dev_err(&pdev->dev, "can't power up port\n"); 4643 goto err_netdev; 4644 } 4645 4646 /* Armada3700 network controller does not support per-cpu 4647 * operation, so only single NAPI should be initialized. 4648 */ 4649 if (pp->neta_armada3700) { 4650 netif_napi_add(dev, &pp->napi, mvneta_poll, NAPI_POLL_WEIGHT); 4651 } else { 4652 for_each_present_cpu(cpu) { 4653 struct mvneta_pcpu_port *port = 4654 per_cpu_ptr(pp->ports, cpu); 4655 4656 netif_napi_add(dev, &port->napi, mvneta_poll, 4657 NAPI_POLL_WEIGHT); 4658 port->pp = pp; 4659 } 4660 } 4661 4662 dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4663 NETIF_F_TSO | NETIF_F_RXCSUM; 4664 dev->hw_features |= dev->features; 4665 dev->vlan_features |= dev->features; 4666 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 4667 dev->gso_max_segs = MVNETA_MAX_TSO_SEGS; 4668 4669 /* MTU range: 68 - 9676 */ 4670 dev->min_mtu = ETH_MIN_MTU; 4671 /* 9676 == 9700 - 20 and rounding to 8 */ 4672 dev->max_mtu = 9676; 4673 4674 err = register_netdev(dev); 4675 if (err < 0) { 4676 dev_err(&pdev->dev, "failed to register\n"); 4677 goto err_free_stats; 4678 } 4679 4680 netdev_info(dev, "Using %s mac address %pM\n", mac_from, 4681 dev->dev_addr); 4682 4683 platform_set_drvdata(pdev, pp->dev); 4684 4685 return 0; 4686 4687 err_netdev: 4688 unregister_netdev(dev); 4689 if (pp->bm_priv) { 4690 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 4691 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 4692 1 << pp->id); 4693 mvneta_bm_put(pp->bm_priv); 4694 } 4695 err_free_stats: 4696 free_percpu(pp->stats); 4697 err_free_ports: 4698 free_percpu(pp->ports); 4699 err_clk: 4700 clk_disable_unprepare(pp->clk_bus); 4701 clk_disable_unprepare(pp->clk); 4702 err_free_phylink: 4703 if (pp->phylink) 4704 phylink_destroy(pp->phylink); 4705 err_free_irq: 4706 irq_dispose_mapping(dev->irq); 4707 return err; 4708 } 4709 4710 /* Device removal routine */ 4711 static int mvneta_remove(struct platform_device *pdev) 4712 { 4713 struct net_device *dev = platform_get_drvdata(pdev); 4714 struct mvneta_port *pp = netdev_priv(dev); 4715 4716 unregister_netdev(dev); 4717 clk_disable_unprepare(pp->clk_bus); 4718 clk_disable_unprepare(pp->clk); 4719 free_percpu(pp->ports); 4720 free_percpu(pp->stats); 4721 irq_dispose_mapping(dev->irq); 4722 phylink_destroy(pp->phylink); 4723 4724 if (pp->bm_priv) { 4725 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 4726 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 4727 1 << pp->id); 4728 mvneta_bm_put(pp->bm_priv); 4729 } 4730 4731 return 0; 4732 } 4733 4734 #ifdef CONFIG_PM_SLEEP 4735 static int mvneta_suspend(struct device *device) 4736 { 4737 int queue; 4738 struct net_device *dev = dev_get_drvdata(device); 4739 struct mvneta_port *pp = netdev_priv(dev); 4740 4741 if (!netif_running(dev)) 4742 goto clean_exit; 4743 4744 if (!pp->neta_armada3700) { 4745 spin_lock(&pp->lock); 4746 pp->is_stopped = true; 4747 spin_unlock(&pp->lock); 4748 4749 cpuhp_state_remove_instance_nocalls(online_hpstate, 4750 &pp->node_online); 4751 cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 4752 &pp->node_dead); 4753 } 4754 4755 rtnl_lock(); 4756 mvneta_stop_dev(pp); 4757 rtnl_unlock(); 4758 4759 for (queue = 0; queue < rxq_number; queue++) { 4760 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 4761 4762 mvneta_rxq_drop_pkts(pp, rxq); 4763 } 4764 4765 for (queue = 0; queue < txq_number; queue++) { 4766 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 4767 4768 mvneta_txq_hw_deinit(pp, txq); 4769 } 4770 4771 clean_exit: 4772 netif_device_detach(dev); 4773 clk_disable_unprepare(pp->clk_bus); 4774 clk_disable_unprepare(pp->clk); 4775 4776 return 0; 4777 } 4778 4779 static int mvneta_resume(struct device *device) 4780 { 4781 struct platform_device *pdev = to_platform_device(device); 4782 struct net_device *dev = dev_get_drvdata(device); 4783 struct mvneta_port *pp = netdev_priv(dev); 4784 int err, queue; 4785 4786 clk_prepare_enable(pp->clk); 4787 if (!IS_ERR(pp->clk_bus)) 4788 clk_prepare_enable(pp->clk_bus); 4789 if (pp->dram_target_info || pp->neta_armada3700) 4790 mvneta_conf_mbus_windows(pp, pp->dram_target_info); 4791 if (pp->bm_priv) { 4792 err = mvneta_bm_port_init(pdev, pp); 4793 if (err < 0) { 4794 dev_info(&pdev->dev, "use SW buffer management\n"); 4795 pp->bm_priv = NULL; 4796 } 4797 } 4798 mvneta_defaults_set(pp); 4799 err = mvneta_port_power_up(pp, pp->phy_interface); 4800 if (err < 0) { 4801 dev_err(device, "can't power up port\n"); 4802 return err; 4803 } 4804 4805 netif_device_attach(dev); 4806 4807 if (!netif_running(dev)) 4808 return 0; 4809 4810 for (queue = 0; queue < rxq_number; queue++) { 4811 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 4812 4813 rxq->next_desc_to_proc = 0; 4814 mvneta_rxq_hw_init(pp, rxq); 4815 } 4816 4817 for (queue = 0; queue < txq_number; queue++) { 4818 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 4819 4820 txq->next_desc_to_proc = 0; 4821 mvneta_txq_hw_init(pp, txq); 4822 } 4823 4824 if (!pp->neta_armada3700) { 4825 spin_lock(&pp->lock); 4826 pp->is_stopped = false; 4827 spin_unlock(&pp->lock); 4828 cpuhp_state_add_instance_nocalls(online_hpstate, 4829 &pp->node_online); 4830 cpuhp_state_add_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 4831 &pp->node_dead); 4832 } 4833 4834 rtnl_lock(); 4835 mvneta_start_dev(pp); 4836 rtnl_unlock(); 4837 mvneta_set_rx_mode(dev); 4838 4839 return 0; 4840 } 4841 #endif 4842 4843 static SIMPLE_DEV_PM_OPS(mvneta_pm_ops, mvneta_suspend, mvneta_resume); 4844 4845 static const struct of_device_id mvneta_match[] = { 4846 { .compatible = "marvell,armada-370-neta" }, 4847 { .compatible = "marvell,armada-xp-neta" }, 4848 { .compatible = "marvell,armada-3700-neta" }, 4849 { } 4850 }; 4851 MODULE_DEVICE_TABLE(of, mvneta_match); 4852 4853 static struct platform_driver mvneta_driver = { 4854 .probe = mvneta_probe, 4855 .remove = mvneta_remove, 4856 .driver = { 4857 .name = MVNETA_DRIVER_NAME, 4858 .of_match_table = mvneta_match, 4859 .pm = &mvneta_pm_ops, 4860 }, 4861 }; 4862 4863 static int __init mvneta_driver_init(void) 4864 { 4865 int ret; 4866 4867 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/mvmeta:online", 4868 mvneta_cpu_online, 4869 mvneta_cpu_down_prepare); 4870 if (ret < 0) 4871 goto out; 4872 online_hpstate = ret; 4873 ret = cpuhp_setup_state_multi(CPUHP_NET_MVNETA_DEAD, "net/mvneta:dead", 4874 NULL, mvneta_cpu_dead); 4875 if (ret) 4876 goto err_dead; 4877 4878 ret = platform_driver_register(&mvneta_driver); 4879 if (ret) 4880 goto err; 4881 return 0; 4882 4883 err: 4884 cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD); 4885 err_dead: 4886 cpuhp_remove_multi_state(online_hpstate); 4887 out: 4888 return ret; 4889 } 4890 module_init(mvneta_driver_init); 4891 4892 static void __exit mvneta_driver_exit(void) 4893 { 4894 platform_driver_unregister(&mvneta_driver); 4895 cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD); 4896 cpuhp_remove_multi_state(online_hpstate); 4897 } 4898 module_exit(mvneta_driver_exit); 4899 4900 MODULE_DESCRIPTION("Marvell NETA Ethernet Driver - www.marvell.com"); 4901 MODULE_AUTHOR("Rami Rosen <rosenr@marvell.com>, Thomas Petazzoni <thomas.petazzoni@free-electrons.com>"); 4902 MODULE_LICENSE("GPL"); 4903 4904 module_param(rxq_number, int, 0444); 4905 module_param(txq_number, int, 0444); 4906 4907 module_param(rxq_def, int, 0444); 4908 module_param(rx_copybreak, int, 0644); 4909