1 /* 2 * Driver for Marvell NETA network card for Armada XP and Armada 370 SoCs. 3 * 4 * Copyright (C) 2012 Marvell 5 * 6 * Rami Rosen <rosenr@marvell.com> 7 * Thomas Petazzoni <thomas.petazzoni@free-electrons.com> 8 * 9 * This file is licensed under the terms of the GNU General Public 10 * License version 2. This program is licensed "as is" without any 11 * warranty of any kind, whether express or implied. 12 */ 13 14 #include <linux/clk.h> 15 #include <linux/cpu.h> 16 #include <linux/etherdevice.h> 17 #include <linux/if_vlan.h> 18 #include <linux/inetdevice.h> 19 #include <linux/interrupt.h> 20 #include <linux/io.h> 21 #include <linux/kernel.h> 22 #include <linux/mbus.h> 23 #include <linux/module.h> 24 #include <linux/netdevice.h> 25 #include <linux/of.h> 26 #include <linux/of_address.h> 27 #include <linux/of_irq.h> 28 #include <linux/of_mdio.h> 29 #include <linux/of_net.h> 30 #include <linux/phy.h> 31 #include <linux/phylink.h> 32 #include <linux/platform_device.h> 33 #include <linux/skbuff.h> 34 #include <net/hwbm.h> 35 #include "mvneta_bm.h" 36 #include <net/ip.h> 37 #include <net/ipv6.h> 38 #include <net/tso.h> 39 40 /* Registers */ 41 #define MVNETA_RXQ_CONFIG_REG(q) (0x1400 + ((q) << 2)) 42 #define MVNETA_RXQ_HW_BUF_ALLOC BIT(0) 43 #define MVNETA_RXQ_SHORT_POOL_ID_SHIFT 4 44 #define MVNETA_RXQ_SHORT_POOL_ID_MASK 0x30 45 #define MVNETA_RXQ_LONG_POOL_ID_SHIFT 6 46 #define MVNETA_RXQ_LONG_POOL_ID_MASK 0xc0 47 #define MVNETA_RXQ_PKT_OFFSET_ALL_MASK (0xf << 8) 48 #define MVNETA_RXQ_PKT_OFFSET_MASK(offs) ((offs) << 8) 49 #define MVNETA_RXQ_THRESHOLD_REG(q) (0x14c0 + ((q) << 2)) 50 #define MVNETA_RXQ_NON_OCCUPIED(v) ((v) << 16) 51 #define MVNETA_RXQ_BASE_ADDR_REG(q) (0x1480 + ((q) << 2)) 52 #define MVNETA_RXQ_SIZE_REG(q) (0x14a0 + ((q) << 2)) 53 #define MVNETA_RXQ_BUF_SIZE_SHIFT 19 54 #define MVNETA_RXQ_BUF_SIZE_MASK (0x1fff << 19) 55 #define MVNETA_RXQ_STATUS_REG(q) (0x14e0 + ((q) << 2)) 56 #define MVNETA_RXQ_OCCUPIED_ALL_MASK 0x3fff 57 #define MVNETA_RXQ_STATUS_UPDATE_REG(q) (0x1500 + ((q) << 2)) 58 #define MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT 16 59 #define MVNETA_RXQ_ADD_NON_OCCUPIED_MAX 255 60 #define MVNETA_PORT_POOL_BUFFER_SZ_REG(pool) (0x1700 + ((pool) << 2)) 61 #define MVNETA_PORT_POOL_BUFFER_SZ_SHIFT 3 62 #define MVNETA_PORT_POOL_BUFFER_SZ_MASK 0xfff8 63 #define MVNETA_PORT_RX_RESET 0x1cc0 64 #define MVNETA_PORT_RX_DMA_RESET BIT(0) 65 #define MVNETA_PHY_ADDR 0x2000 66 #define MVNETA_PHY_ADDR_MASK 0x1f 67 #define MVNETA_MBUS_RETRY 0x2010 68 #define MVNETA_UNIT_INTR_CAUSE 0x2080 69 #define MVNETA_UNIT_CONTROL 0x20B0 70 #define MVNETA_PHY_POLLING_ENABLE BIT(1) 71 #define MVNETA_WIN_BASE(w) (0x2200 + ((w) << 3)) 72 #define MVNETA_WIN_SIZE(w) (0x2204 + ((w) << 3)) 73 #define MVNETA_WIN_REMAP(w) (0x2280 + ((w) << 2)) 74 #define MVNETA_BASE_ADDR_ENABLE 0x2290 75 #define MVNETA_ACCESS_PROTECT_ENABLE 0x2294 76 #define MVNETA_PORT_CONFIG 0x2400 77 #define MVNETA_UNI_PROMISC_MODE BIT(0) 78 #define MVNETA_DEF_RXQ(q) ((q) << 1) 79 #define MVNETA_DEF_RXQ_ARP(q) ((q) << 4) 80 #define MVNETA_TX_UNSET_ERR_SUM BIT(12) 81 #define MVNETA_DEF_RXQ_TCP(q) ((q) << 16) 82 #define MVNETA_DEF_RXQ_UDP(q) ((q) << 19) 83 #define MVNETA_DEF_RXQ_BPDU(q) ((q) << 22) 84 #define MVNETA_RX_CSUM_WITH_PSEUDO_HDR BIT(25) 85 #define MVNETA_PORT_CONFIG_DEFL_VALUE(q) (MVNETA_DEF_RXQ(q) | \ 86 MVNETA_DEF_RXQ_ARP(q) | \ 87 MVNETA_DEF_RXQ_TCP(q) | \ 88 MVNETA_DEF_RXQ_UDP(q) | \ 89 MVNETA_DEF_RXQ_BPDU(q) | \ 90 MVNETA_TX_UNSET_ERR_SUM | \ 91 MVNETA_RX_CSUM_WITH_PSEUDO_HDR) 92 #define MVNETA_PORT_CONFIG_EXTEND 0x2404 93 #define MVNETA_MAC_ADDR_LOW 0x2414 94 #define MVNETA_MAC_ADDR_HIGH 0x2418 95 #define MVNETA_SDMA_CONFIG 0x241c 96 #define MVNETA_SDMA_BRST_SIZE_16 4 97 #define MVNETA_RX_BRST_SZ_MASK(burst) ((burst) << 1) 98 #define MVNETA_RX_NO_DATA_SWAP BIT(4) 99 #define MVNETA_TX_NO_DATA_SWAP BIT(5) 100 #define MVNETA_DESC_SWAP BIT(6) 101 #define MVNETA_TX_BRST_SZ_MASK(burst) ((burst) << 22) 102 #define MVNETA_PORT_STATUS 0x2444 103 #define MVNETA_TX_IN_PRGRS BIT(1) 104 #define MVNETA_TX_FIFO_EMPTY BIT(8) 105 #define MVNETA_RX_MIN_FRAME_SIZE 0x247c 106 #define MVNETA_SERDES_CFG 0x24A0 107 #define MVNETA_SGMII_SERDES_PROTO 0x0cc7 108 #define MVNETA_QSGMII_SERDES_PROTO 0x0667 109 #define MVNETA_TYPE_PRIO 0x24bc 110 #define MVNETA_FORCE_UNI BIT(21) 111 #define MVNETA_TXQ_CMD_1 0x24e4 112 #define MVNETA_TXQ_CMD 0x2448 113 #define MVNETA_TXQ_DISABLE_SHIFT 8 114 #define MVNETA_TXQ_ENABLE_MASK 0x000000ff 115 #define MVNETA_RX_DISCARD_FRAME_COUNT 0x2484 116 #define MVNETA_OVERRUN_FRAME_COUNT 0x2488 117 #define MVNETA_GMAC_CLOCK_DIVIDER 0x24f4 118 #define MVNETA_GMAC_1MS_CLOCK_ENABLE BIT(31) 119 #define MVNETA_ACC_MODE 0x2500 120 #define MVNETA_BM_ADDRESS 0x2504 121 #define MVNETA_CPU_MAP(cpu) (0x2540 + ((cpu) << 2)) 122 #define MVNETA_CPU_RXQ_ACCESS_ALL_MASK 0x000000ff 123 #define MVNETA_CPU_TXQ_ACCESS_ALL_MASK 0x0000ff00 124 #define MVNETA_CPU_RXQ_ACCESS(rxq) BIT(rxq) 125 #define MVNETA_CPU_TXQ_ACCESS(txq) BIT(txq + 8) 126 #define MVNETA_RXQ_TIME_COAL_REG(q) (0x2580 + ((q) << 2)) 127 128 /* Exception Interrupt Port/Queue Cause register 129 * 130 * Their behavior depend of the mapping done using the PCPX2Q 131 * registers. For a given CPU if the bit associated to a queue is not 132 * set, then for the register a read from this CPU will always return 133 * 0 and a write won't do anything 134 */ 135 136 #define MVNETA_INTR_NEW_CAUSE 0x25a0 137 #define MVNETA_INTR_NEW_MASK 0x25a4 138 139 /* bits 0..7 = TXQ SENT, one bit per queue. 140 * bits 8..15 = RXQ OCCUP, one bit per queue. 141 * bits 16..23 = RXQ FREE, one bit per queue. 142 * bit 29 = OLD_REG_SUM, see old reg ? 143 * bit 30 = TX_ERR_SUM, one bit for 4 ports 144 * bit 31 = MISC_SUM, one bit for 4 ports 145 */ 146 #define MVNETA_TX_INTR_MASK(nr_txqs) (((1 << nr_txqs) - 1) << 0) 147 #define MVNETA_TX_INTR_MASK_ALL (0xff << 0) 148 #define MVNETA_RX_INTR_MASK(nr_rxqs) (((1 << nr_rxqs) - 1) << 8) 149 #define MVNETA_RX_INTR_MASK_ALL (0xff << 8) 150 #define MVNETA_MISCINTR_INTR_MASK BIT(31) 151 152 #define MVNETA_INTR_OLD_CAUSE 0x25a8 153 #define MVNETA_INTR_OLD_MASK 0x25ac 154 155 /* Data Path Port/Queue Cause Register */ 156 #define MVNETA_INTR_MISC_CAUSE 0x25b0 157 #define MVNETA_INTR_MISC_MASK 0x25b4 158 159 #define MVNETA_CAUSE_PHY_STATUS_CHANGE BIT(0) 160 #define MVNETA_CAUSE_LINK_CHANGE BIT(1) 161 #define MVNETA_CAUSE_PTP BIT(4) 162 163 #define MVNETA_CAUSE_INTERNAL_ADDR_ERR BIT(7) 164 #define MVNETA_CAUSE_RX_OVERRUN BIT(8) 165 #define MVNETA_CAUSE_RX_CRC_ERROR BIT(9) 166 #define MVNETA_CAUSE_RX_LARGE_PKT BIT(10) 167 #define MVNETA_CAUSE_TX_UNDERUN BIT(11) 168 #define MVNETA_CAUSE_PRBS_ERR BIT(12) 169 #define MVNETA_CAUSE_PSC_SYNC_CHANGE BIT(13) 170 #define MVNETA_CAUSE_SERDES_SYNC_ERR BIT(14) 171 172 #define MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT 16 173 #define MVNETA_CAUSE_BMU_ALLOC_ERR_ALL_MASK (0xF << MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT) 174 #define MVNETA_CAUSE_BMU_ALLOC_ERR_MASK(pool) (1 << (MVNETA_CAUSE_BMU_ALLOC_ERR_SHIFT + (pool))) 175 176 #define MVNETA_CAUSE_TXQ_ERROR_SHIFT 24 177 #define MVNETA_CAUSE_TXQ_ERROR_ALL_MASK (0xFF << MVNETA_CAUSE_TXQ_ERROR_SHIFT) 178 #define MVNETA_CAUSE_TXQ_ERROR_MASK(q) (1 << (MVNETA_CAUSE_TXQ_ERROR_SHIFT + (q))) 179 180 #define MVNETA_INTR_ENABLE 0x25b8 181 #define MVNETA_TXQ_INTR_ENABLE_ALL_MASK 0x0000ff00 182 #define MVNETA_RXQ_INTR_ENABLE_ALL_MASK 0x000000ff 183 184 #define MVNETA_RXQ_CMD 0x2680 185 #define MVNETA_RXQ_DISABLE_SHIFT 8 186 #define MVNETA_RXQ_ENABLE_MASK 0x000000ff 187 #define MVETH_TXQ_TOKEN_COUNT_REG(q) (0x2700 + ((q) << 4)) 188 #define MVETH_TXQ_TOKEN_CFG_REG(q) (0x2704 + ((q) << 4)) 189 #define MVNETA_GMAC_CTRL_0 0x2c00 190 #define MVNETA_GMAC_MAX_RX_SIZE_SHIFT 2 191 #define MVNETA_GMAC_MAX_RX_SIZE_MASK 0x7ffc 192 #define MVNETA_GMAC0_PORT_1000BASE_X BIT(1) 193 #define MVNETA_GMAC0_PORT_ENABLE BIT(0) 194 #define MVNETA_GMAC_CTRL_2 0x2c08 195 #define MVNETA_GMAC2_INBAND_AN_ENABLE BIT(0) 196 #define MVNETA_GMAC2_PCS_ENABLE BIT(3) 197 #define MVNETA_GMAC2_PORT_RGMII BIT(4) 198 #define MVNETA_GMAC2_PORT_RESET BIT(6) 199 #define MVNETA_GMAC_STATUS 0x2c10 200 #define MVNETA_GMAC_LINK_UP BIT(0) 201 #define MVNETA_GMAC_SPEED_1000 BIT(1) 202 #define MVNETA_GMAC_SPEED_100 BIT(2) 203 #define MVNETA_GMAC_FULL_DUPLEX BIT(3) 204 #define MVNETA_GMAC_RX_FLOW_CTRL_ENABLE BIT(4) 205 #define MVNETA_GMAC_TX_FLOW_CTRL_ENABLE BIT(5) 206 #define MVNETA_GMAC_RX_FLOW_CTRL_ACTIVE BIT(6) 207 #define MVNETA_GMAC_TX_FLOW_CTRL_ACTIVE BIT(7) 208 #define MVNETA_GMAC_AN_COMPLETE BIT(11) 209 #define MVNETA_GMAC_SYNC_OK BIT(14) 210 #define MVNETA_GMAC_AUTONEG_CONFIG 0x2c0c 211 #define MVNETA_GMAC_FORCE_LINK_DOWN BIT(0) 212 #define MVNETA_GMAC_FORCE_LINK_PASS BIT(1) 213 #define MVNETA_GMAC_INBAND_AN_ENABLE BIT(2) 214 #define MVNETA_GMAC_AN_BYPASS_ENABLE BIT(3) 215 #define MVNETA_GMAC_INBAND_RESTART_AN BIT(4) 216 #define MVNETA_GMAC_CONFIG_MII_SPEED BIT(5) 217 #define MVNETA_GMAC_CONFIG_GMII_SPEED BIT(6) 218 #define MVNETA_GMAC_AN_SPEED_EN BIT(7) 219 #define MVNETA_GMAC_CONFIG_FLOW_CTRL BIT(8) 220 #define MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL BIT(9) 221 #define MVNETA_GMAC_AN_FLOW_CTRL_EN BIT(11) 222 #define MVNETA_GMAC_CONFIG_FULL_DUPLEX BIT(12) 223 #define MVNETA_GMAC_AN_DUPLEX_EN BIT(13) 224 #define MVNETA_GMAC_CTRL_4 0x2c90 225 #define MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE BIT(1) 226 #define MVNETA_MIB_COUNTERS_BASE 0x3000 227 #define MVNETA_MIB_LATE_COLLISION 0x7c 228 #define MVNETA_DA_FILT_SPEC_MCAST 0x3400 229 #define MVNETA_DA_FILT_OTH_MCAST 0x3500 230 #define MVNETA_DA_FILT_UCAST_BASE 0x3600 231 #define MVNETA_TXQ_BASE_ADDR_REG(q) (0x3c00 + ((q) << 2)) 232 #define MVNETA_TXQ_SIZE_REG(q) (0x3c20 + ((q) << 2)) 233 #define MVNETA_TXQ_SENT_THRESH_ALL_MASK 0x3fff0000 234 #define MVNETA_TXQ_SENT_THRESH_MASK(coal) ((coal) << 16) 235 #define MVNETA_TXQ_UPDATE_REG(q) (0x3c60 + ((q) << 2)) 236 #define MVNETA_TXQ_DEC_SENT_SHIFT 16 237 #define MVNETA_TXQ_DEC_SENT_MASK 0xff 238 #define MVNETA_TXQ_STATUS_REG(q) (0x3c40 + ((q) << 2)) 239 #define MVNETA_TXQ_SENT_DESC_SHIFT 16 240 #define MVNETA_TXQ_SENT_DESC_MASK 0x3fff0000 241 #define MVNETA_PORT_TX_RESET 0x3cf0 242 #define MVNETA_PORT_TX_DMA_RESET BIT(0) 243 #define MVNETA_TX_MTU 0x3e0c 244 #define MVNETA_TX_TOKEN_SIZE 0x3e14 245 #define MVNETA_TX_TOKEN_SIZE_MAX 0xffffffff 246 #define MVNETA_TXQ_TOKEN_SIZE_REG(q) (0x3e40 + ((q) << 2)) 247 #define MVNETA_TXQ_TOKEN_SIZE_MAX 0x7fffffff 248 249 #define MVNETA_LPI_CTRL_0 0x2cc0 250 #define MVNETA_LPI_CTRL_1 0x2cc4 251 #define MVNETA_LPI_REQUEST_ENABLE BIT(0) 252 #define MVNETA_LPI_CTRL_2 0x2cc8 253 #define MVNETA_LPI_STATUS 0x2ccc 254 255 #define MVNETA_CAUSE_TXQ_SENT_DESC_ALL_MASK 0xff 256 257 /* Descriptor ring Macros */ 258 #define MVNETA_QUEUE_NEXT_DESC(q, index) \ 259 (((index) < (q)->last_desc) ? ((index) + 1) : 0) 260 261 /* Various constants */ 262 263 /* Coalescing */ 264 #define MVNETA_TXDONE_COAL_PKTS 0 /* interrupt per packet */ 265 #define MVNETA_RX_COAL_PKTS 32 266 #define MVNETA_RX_COAL_USEC 100 267 268 /* The two bytes Marvell header. Either contains a special value used 269 * by Marvell switches when a specific hardware mode is enabled (not 270 * supported by this driver) or is filled automatically by zeroes on 271 * the RX side. Those two bytes being at the front of the Ethernet 272 * header, they allow to have the IP header aligned on a 4 bytes 273 * boundary automatically: the hardware skips those two bytes on its 274 * own. 275 */ 276 #define MVNETA_MH_SIZE 2 277 278 #define MVNETA_VLAN_TAG_LEN 4 279 280 #define MVNETA_TX_CSUM_DEF_SIZE 1600 281 #define MVNETA_TX_CSUM_MAX_SIZE 9800 282 #define MVNETA_ACC_MODE_EXT1 1 283 #define MVNETA_ACC_MODE_EXT2 2 284 285 #define MVNETA_MAX_DECODE_WIN 6 286 287 /* Timeout constants */ 288 #define MVNETA_TX_DISABLE_TIMEOUT_MSEC 1000 289 #define MVNETA_RX_DISABLE_TIMEOUT_MSEC 1000 290 #define MVNETA_TX_FIFO_EMPTY_TIMEOUT 10000 291 292 #define MVNETA_TX_MTU_MAX 0x3ffff 293 294 /* The RSS lookup table actually has 256 entries but we do not use 295 * them yet 296 */ 297 #define MVNETA_RSS_LU_TABLE_SIZE 1 298 299 /* Max number of Rx descriptors */ 300 #define MVNETA_MAX_RXD 512 301 302 /* Max number of Tx descriptors */ 303 #define MVNETA_MAX_TXD 1024 304 305 /* Max number of allowed TCP segments for software TSO */ 306 #define MVNETA_MAX_TSO_SEGS 100 307 308 #define MVNETA_MAX_SKB_DESCS (MVNETA_MAX_TSO_SEGS * 2 + MAX_SKB_FRAGS) 309 310 /* descriptor aligned size */ 311 #define MVNETA_DESC_ALIGNED_SIZE 32 312 313 /* Number of bytes to be taken into account by HW when putting incoming data 314 * to the buffers. It is needed in case NET_SKB_PAD exceeds maximum packet 315 * offset supported in MVNETA_RXQ_CONFIG_REG(q) registers. 316 */ 317 #define MVNETA_RX_PKT_OFFSET_CORRECTION 64 318 319 #define MVNETA_RX_PKT_SIZE(mtu) \ 320 ALIGN((mtu) + MVNETA_MH_SIZE + MVNETA_VLAN_TAG_LEN + \ 321 ETH_HLEN + ETH_FCS_LEN, \ 322 cache_line_size()) 323 324 #define IS_TSO_HEADER(txq, addr) \ 325 ((addr >= txq->tso_hdrs_phys) && \ 326 (addr < txq->tso_hdrs_phys + txq->size * TSO_HEADER_SIZE)) 327 328 #define MVNETA_RX_GET_BM_POOL_ID(rxd) \ 329 (((rxd)->status & MVNETA_RXD_BM_POOL_MASK) >> MVNETA_RXD_BM_POOL_SHIFT) 330 331 enum { 332 ETHTOOL_STAT_EEE_WAKEUP, 333 ETHTOOL_STAT_SKB_ALLOC_ERR, 334 ETHTOOL_STAT_REFILL_ERR, 335 ETHTOOL_MAX_STATS, 336 }; 337 338 struct mvneta_statistic { 339 unsigned short offset; 340 unsigned short type; 341 const char name[ETH_GSTRING_LEN]; 342 }; 343 344 #define T_REG_32 32 345 #define T_REG_64 64 346 #define T_SW 1 347 348 static const struct mvneta_statistic mvneta_statistics[] = { 349 { 0x3000, T_REG_64, "good_octets_received", }, 350 { 0x3010, T_REG_32, "good_frames_received", }, 351 { 0x3008, T_REG_32, "bad_octets_received", }, 352 { 0x3014, T_REG_32, "bad_frames_received", }, 353 { 0x3018, T_REG_32, "broadcast_frames_received", }, 354 { 0x301c, T_REG_32, "multicast_frames_received", }, 355 { 0x3050, T_REG_32, "unrec_mac_control_received", }, 356 { 0x3058, T_REG_32, "good_fc_received", }, 357 { 0x305c, T_REG_32, "bad_fc_received", }, 358 { 0x3060, T_REG_32, "undersize_received", }, 359 { 0x3064, T_REG_32, "fragments_received", }, 360 { 0x3068, T_REG_32, "oversize_received", }, 361 { 0x306c, T_REG_32, "jabber_received", }, 362 { 0x3070, T_REG_32, "mac_receive_error", }, 363 { 0x3074, T_REG_32, "bad_crc_event", }, 364 { 0x3078, T_REG_32, "collision", }, 365 { 0x307c, T_REG_32, "late_collision", }, 366 { 0x2484, T_REG_32, "rx_discard", }, 367 { 0x2488, T_REG_32, "rx_overrun", }, 368 { 0x3020, T_REG_32, "frames_64_octets", }, 369 { 0x3024, T_REG_32, "frames_65_to_127_octets", }, 370 { 0x3028, T_REG_32, "frames_128_to_255_octets", }, 371 { 0x302c, T_REG_32, "frames_256_to_511_octets", }, 372 { 0x3030, T_REG_32, "frames_512_to_1023_octets", }, 373 { 0x3034, T_REG_32, "frames_1024_to_max_octets", }, 374 { 0x3038, T_REG_64, "good_octets_sent", }, 375 { 0x3040, T_REG_32, "good_frames_sent", }, 376 { 0x3044, T_REG_32, "excessive_collision", }, 377 { 0x3048, T_REG_32, "multicast_frames_sent", }, 378 { 0x304c, T_REG_32, "broadcast_frames_sent", }, 379 { 0x3054, T_REG_32, "fc_sent", }, 380 { 0x300c, T_REG_32, "internal_mac_transmit_err", }, 381 { ETHTOOL_STAT_EEE_WAKEUP, T_SW, "eee_wakeup_errors", }, 382 { ETHTOOL_STAT_SKB_ALLOC_ERR, T_SW, "skb_alloc_errors", }, 383 { ETHTOOL_STAT_REFILL_ERR, T_SW, "refill_errors", }, 384 }; 385 386 struct mvneta_pcpu_stats { 387 struct u64_stats_sync syncp; 388 u64 rx_packets; 389 u64 rx_bytes; 390 u64 tx_packets; 391 u64 tx_bytes; 392 }; 393 394 struct mvneta_pcpu_port { 395 /* Pointer to the shared port */ 396 struct mvneta_port *pp; 397 398 /* Pointer to the CPU-local NAPI struct */ 399 struct napi_struct napi; 400 401 /* Cause of the previous interrupt */ 402 u32 cause_rx_tx; 403 }; 404 405 struct mvneta_port { 406 u8 id; 407 struct mvneta_pcpu_port __percpu *ports; 408 struct mvneta_pcpu_stats __percpu *stats; 409 410 int pkt_size; 411 void __iomem *base; 412 struct mvneta_rx_queue *rxqs; 413 struct mvneta_tx_queue *txqs; 414 struct net_device *dev; 415 struct hlist_node node_online; 416 struct hlist_node node_dead; 417 int rxq_def; 418 /* Protect the access to the percpu interrupt registers, 419 * ensuring that the configuration remains coherent. 420 */ 421 spinlock_t lock; 422 bool is_stopped; 423 424 u32 cause_rx_tx; 425 struct napi_struct napi; 426 427 /* Core clock */ 428 struct clk *clk; 429 /* AXI clock */ 430 struct clk *clk_bus; 431 u8 mcast_count[256]; 432 u16 tx_ring_size; 433 u16 rx_ring_size; 434 435 phy_interface_t phy_interface; 436 struct device_node *dn; 437 unsigned int tx_csum_limit; 438 struct phylink *phylink; 439 440 struct mvneta_bm *bm_priv; 441 struct mvneta_bm_pool *pool_long; 442 struct mvneta_bm_pool *pool_short; 443 int bm_win_id; 444 445 bool eee_enabled; 446 bool eee_active; 447 bool tx_lpi_enabled; 448 449 u64 ethtool_stats[ARRAY_SIZE(mvneta_statistics)]; 450 451 u32 indir[MVNETA_RSS_LU_TABLE_SIZE]; 452 453 /* Flags for special SoC configurations */ 454 bool neta_armada3700; 455 u16 rx_offset_correction; 456 const struct mbus_dram_target_info *dram_target_info; 457 }; 458 459 /* The mvneta_tx_desc and mvneta_rx_desc structures describe the 460 * layout of the transmit and reception DMA descriptors, and their 461 * layout is therefore defined by the hardware design 462 */ 463 464 #define MVNETA_TX_L3_OFF_SHIFT 0 465 #define MVNETA_TX_IP_HLEN_SHIFT 8 466 #define MVNETA_TX_L4_UDP BIT(16) 467 #define MVNETA_TX_L3_IP6 BIT(17) 468 #define MVNETA_TXD_IP_CSUM BIT(18) 469 #define MVNETA_TXD_Z_PAD BIT(19) 470 #define MVNETA_TXD_L_DESC BIT(20) 471 #define MVNETA_TXD_F_DESC BIT(21) 472 #define MVNETA_TXD_FLZ_DESC (MVNETA_TXD_Z_PAD | \ 473 MVNETA_TXD_L_DESC | \ 474 MVNETA_TXD_F_DESC) 475 #define MVNETA_TX_L4_CSUM_FULL BIT(30) 476 #define MVNETA_TX_L4_CSUM_NOT BIT(31) 477 478 #define MVNETA_RXD_ERR_CRC 0x0 479 #define MVNETA_RXD_BM_POOL_SHIFT 13 480 #define MVNETA_RXD_BM_POOL_MASK (BIT(13) | BIT(14)) 481 #define MVNETA_RXD_ERR_SUMMARY BIT(16) 482 #define MVNETA_RXD_ERR_OVERRUN BIT(17) 483 #define MVNETA_RXD_ERR_LEN BIT(18) 484 #define MVNETA_RXD_ERR_RESOURCE (BIT(17) | BIT(18)) 485 #define MVNETA_RXD_ERR_CODE_MASK (BIT(17) | BIT(18)) 486 #define MVNETA_RXD_L3_IP4 BIT(25) 487 #define MVNETA_RXD_LAST_DESC BIT(26) 488 #define MVNETA_RXD_FIRST_DESC BIT(27) 489 #define MVNETA_RXD_FIRST_LAST_DESC (MVNETA_RXD_FIRST_DESC | \ 490 MVNETA_RXD_LAST_DESC) 491 #define MVNETA_RXD_L4_CSUM_OK BIT(30) 492 493 #if defined(__LITTLE_ENDIAN) 494 struct mvneta_tx_desc { 495 u32 command; /* Options used by HW for packet transmitting.*/ 496 u16 reserved1; /* csum_l4 (for future use) */ 497 u16 data_size; /* Data size of transmitted packet in bytes */ 498 u32 buf_phys_addr; /* Physical addr of transmitted buffer */ 499 u32 reserved2; /* hw_cmd - (for future use, PMT) */ 500 u32 reserved3[4]; /* Reserved - (for future use) */ 501 }; 502 503 struct mvneta_rx_desc { 504 u32 status; /* Info about received packet */ 505 u16 reserved1; /* pnc_info - (for future use, PnC) */ 506 u16 data_size; /* Size of received packet in bytes */ 507 508 u32 buf_phys_addr; /* Physical address of the buffer */ 509 u32 reserved2; /* pnc_flow_id (for future use, PnC) */ 510 511 u32 buf_cookie; /* cookie for access to RX buffer in rx path */ 512 u16 reserved3; /* prefetch_cmd, for future use */ 513 u16 reserved4; /* csum_l4 - (for future use, PnC) */ 514 515 u32 reserved5; /* pnc_extra PnC (for future use, PnC) */ 516 u32 reserved6; /* hw_cmd (for future use, PnC and HWF) */ 517 }; 518 #else 519 struct mvneta_tx_desc { 520 u16 data_size; /* Data size of transmitted packet in bytes */ 521 u16 reserved1; /* csum_l4 (for future use) */ 522 u32 command; /* Options used by HW for packet transmitting.*/ 523 u32 reserved2; /* hw_cmd - (for future use, PMT) */ 524 u32 buf_phys_addr; /* Physical addr of transmitted buffer */ 525 u32 reserved3[4]; /* Reserved - (for future use) */ 526 }; 527 528 struct mvneta_rx_desc { 529 u16 data_size; /* Size of received packet in bytes */ 530 u16 reserved1; /* pnc_info - (for future use, PnC) */ 531 u32 status; /* Info about received packet */ 532 533 u32 reserved2; /* pnc_flow_id (for future use, PnC) */ 534 u32 buf_phys_addr; /* Physical address of the buffer */ 535 536 u16 reserved4; /* csum_l4 - (for future use, PnC) */ 537 u16 reserved3; /* prefetch_cmd, for future use */ 538 u32 buf_cookie; /* cookie for access to RX buffer in rx path */ 539 540 u32 reserved5; /* pnc_extra PnC (for future use, PnC) */ 541 u32 reserved6; /* hw_cmd (for future use, PnC and HWF) */ 542 }; 543 #endif 544 545 struct mvneta_tx_queue { 546 /* Number of this TX queue, in the range 0-7 */ 547 u8 id; 548 549 /* Number of TX DMA descriptors in the descriptor ring */ 550 int size; 551 552 /* Number of currently used TX DMA descriptor in the 553 * descriptor ring 554 */ 555 int count; 556 int pending; 557 int tx_stop_threshold; 558 int tx_wake_threshold; 559 560 /* Array of transmitted skb */ 561 struct sk_buff **tx_skb; 562 563 /* Index of last TX DMA descriptor that was inserted */ 564 int txq_put_index; 565 566 /* Index of the TX DMA descriptor to be cleaned up */ 567 int txq_get_index; 568 569 u32 done_pkts_coal; 570 571 /* Virtual address of the TX DMA descriptors array */ 572 struct mvneta_tx_desc *descs; 573 574 /* DMA address of the TX DMA descriptors array */ 575 dma_addr_t descs_phys; 576 577 /* Index of the last TX DMA descriptor */ 578 int last_desc; 579 580 /* Index of the next TX DMA descriptor to process */ 581 int next_desc_to_proc; 582 583 /* DMA buffers for TSO headers */ 584 char *tso_hdrs; 585 586 /* DMA address of TSO headers */ 587 dma_addr_t tso_hdrs_phys; 588 589 /* Affinity mask for CPUs*/ 590 cpumask_t affinity_mask; 591 }; 592 593 struct mvneta_rx_queue { 594 /* rx queue number, in the range 0-7 */ 595 u8 id; 596 597 /* num of rx descriptors in the rx descriptor ring */ 598 int size; 599 600 u32 pkts_coal; 601 u32 time_coal; 602 603 /* Virtual address of the RX buffer */ 604 void **buf_virt_addr; 605 606 /* Virtual address of the RX DMA descriptors array */ 607 struct mvneta_rx_desc *descs; 608 609 /* DMA address of the RX DMA descriptors array */ 610 dma_addr_t descs_phys; 611 612 /* Index of the last RX DMA descriptor */ 613 int last_desc; 614 615 /* Index of the next RX DMA descriptor to process */ 616 int next_desc_to_proc; 617 618 /* Index of first RX DMA descriptor to refill */ 619 int first_to_refill; 620 u32 refill_num; 621 622 /* pointer to uncomplete skb buffer */ 623 struct sk_buff *skb; 624 int left_size; 625 626 /* error counters */ 627 u32 skb_alloc_err; 628 u32 refill_err; 629 }; 630 631 static enum cpuhp_state online_hpstate; 632 /* The hardware supports eight (8) rx queues, but we are only allowing 633 * the first one to be used. Therefore, let's just allocate one queue. 634 */ 635 static int rxq_number = 8; 636 static int txq_number = 8; 637 638 static int rxq_def; 639 640 static int rx_copybreak __read_mostly = 256; 641 static int rx_header_size __read_mostly = 128; 642 643 /* HW BM need that each port be identify by a unique ID */ 644 static int global_port_id; 645 646 #define MVNETA_DRIVER_NAME "mvneta" 647 #define MVNETA_DRIVER_VERSION "1.0" 648 649 /* Utility/helper methods */ 650 651 /* Write helper method */ 652 static void mvreg_write(struct mvneta_port *pp, u32 offset, u32 data) 653 { 654 writel(data, pp->base + offset); 655 } 656 657 /* Read helper method */ 658 static u32 mvreg_read(struct mvneta_port *pp, u32 offset) 659 { 660 return readl(pp->base + offset); 661 } 662 663 /* Increment txq get counter */ 664 static void mvneta_txq_inc_get(struct mvneta_tx_queue *txq) 665 { 666 txq->txq_get_index++; 667 if (txq->txq_get_index == txq->size) 668 txq->txq_get_index = 0; 669 } 670 671 /* Increment txq put counter */ 672 static void mvneta_txq_inc_put(struct mvneta_tx_queue *txq) 673 { 674 txq->txq_put_index++; 675 if (txq->txq_put_index == txq->size) 676 txq->txq_put_index = 0; 677 } 678 679 680 /* Clear all MIB counters */ 681 static void mvneta_mib_counters_clear(struct mvneta_port *pp) 682 { 683 int i; 684 u32 dummy; 685 686 /* Perform dummy reads from MIB counters */ 687 for (i = 0; i < MVNETA_MIB_LATE_COLLISION; i += 4) 688 dummy = mvreg_read(pp, (MVNETA_MIB_COUNTERS_BASE + i)); 689 dummy = mvreg_read(pp, MVNETA_RX_DISCARD_FRAME_COUNT); 690 dummy = mvreg_read(pp, MVNETA_OVERRUN_FRAME_COUNT); 691 } 692 693 /* Get System Network Statistics */ 694 static void 695 mvneta_get_stats64(struct net_device *dev, 696 struct rtnl_link_stats64 *stats) 697 { 698 struct mvneta_port *pp = netdev_priv(dev); 699 unsigned int start; 700 int cpu; 701 702 for_each_possible_cpu(cpu) { 703 struct mvneta_pcpu_stats *cpu_stats; 704 u64 rx_packets; 705 u64 rx_bytes; 706 u64 tx_packets; 707 u64 tx_bytes; 708 709 cpu_stats = per_cpu_ptr(pp->stats, cpu); 710 do { 711 start = u64_stats_fetch_begin_irq(&cpu_stats->syncp); 712 rx_packets = cpu_stats->rx_packets; 713 rx_bytes = cpu_stats->rx_bytes; 714 tx_packets = cpu_stats->tx_packets; 715 tx_bytes = cpu_stats->tx_bytes; 716 } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start)); 717 718 stats->rx_packets += rx_packets; 719 stats->rx_bytes += rx_bytes; 720 stats->tx_packets += tx_packets; 721 stats->tx_bytes += tx_bytes; 722 } 723 724 stats->rx_errors = dev->stats.rx_errors; 725 stats->rx_dropped = dev->stats.rx_dropped; 726 727 stats->tx_dropped = dev->stats.tx_dropped; 728 } 729 730 /* Rx descriptors helper methods */ 731 732 /* Checks whether the RX descriptor having this status is both the first 733 * and the last descriptor for the RX packet. Each RX packet is currently 734 * received through a single RX descriptor, so not having each RX 735 * descriptor with its first and last bits set is an error 736 */ 737 static int mvneta_rxq_desc_is_first_last(u32 status) 738 { 739 return (status & MVNETA_RXD_FIRST_LAST_DESC) == 740 MVNETA_RXD_FIRST_LAST_DESC; 741 } 742 743 /* Add number of descriptors ready to receive new packets */ 744 static void mvneta_rxq_non_occup_desc_add(struct mvneta_port *pp, 745 struct mvneta_rx_queue *rxq, 746 int ndescs) 747 { 748 /* Only MVNETA_RXQ_ADD_NON_OCCUPIED_MAX (255) descriptors can 749 * be added at once 750 */ 751 while (ndescs > MVNETA_RXQ_ADD_NON_OCCUPIED_MAX) { 752 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), 753 (MVNETA_RXQ_ADD_NON_OCCUPIED_MAX << 754 MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT)); 755 ndescs -= MVNETA_RXQ_ADD_NON_OCCUPIED_MAX; 756 } 757 758 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), 759 (ndescs << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT)); 760 } 761 762 /* Get number of RX descriptors occupied by received packets */ 763 static int mvneta_rxq_busy_desc_num_get(struct mvneta_port *pp, 764 struct mvneta_rx_queue *rxq) 765 { 766 u32 val; 767 768 val = mvreg_read(pp, MVNETA_RXQ_STATUS_REG(rxq->id)); 769 return val & MVNETA_RXQ_OCCUPIED_ALL_MASK; 770 } 771 772 /* Update num of rx desc called upon return from rx path or 773 * from mvneta_rxq_drop_pkts(). 774 */ 775 static void mvneta_rxq_desc_num_update(struct mvneta_port *pp, 776 struct mvneta_rx_queue *rxq, 777 int rx_done, int rx_filled) 778 { 779 u32 val; 780 781 if ((rx_done <= 0xff) && (rx_filled <= 0xff)) { 782 val = rx_done | 783 (rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT); 784 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val); 785 return; 786 } 787 788 /* Only 255 descriptors can be added at once */ 789 while ((rx_done > 0) || (rx_filled > 0)) { 790 if (rx_done <= 0xff) { 791 val = rx_done; 792 rx_done = 0; 793 } else { 794 val = 0xff; 795 rx_done -= 0xff; 796 } 797 if (rx_filled <= 0xff) { 798 val |= rx_filled << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT; 799 rx_filled = 0; 800 } else { 801 val |= 0xff << MVNETA_RXQ_ADD_NON_OCCUPIED_SHIFT; 802 rx_filled -= 0xff; 803 } 804 mvreg_write(pp, MVNETA_RXQ_STATUS_UPDATE_REG(rxq->id), val); 805 } 806 } 807 808 /* Get pointer to next RX descriptor to be processed by SW */ 809 static struct mvneta_rx_desc * 810 mvneta_rxq_next_desc_get(struct mvneta_rx_queue *rxq) 811 { 812 int rx_desc = rxq->next_desc_to_proc; 813 814 rxq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(rxq, rx_desc); 815 prefetch(rxq->descs + rxq->next_desc_to_proc); 816 return rxq->descs + rx_desc; 817 } 818 819 /* Change maximum receive size of the port. */ 820 static void mvneta_max_rx_size_set(struct mvneta_port *pp, int max_rx_size) 821 { 822 u32 val; 823 824 val = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 825 val &= ~MVNETA_GMAC_MAX_RX_SIZE_MASK; 826 val |= ((max_rx_size - MVNETA_MH_SIZE) / 2) << 827 MVNETA_GMAC_MAX_RX_SIZE_SHIFT; 828 mvreg_write(pp, MVNETA_GMAC_CTRL_0, val); 829 } 830 831 832 /* Set rx queue offset */ 833 static void mvneta_rxq_offset_set(struct mvneta_port *pp, 834 struct mvneta_rx_queue *rxq, 835 int offset) 836 { 837 u32 val; 838 839 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 840 val &= ~MVNETA_RXQ_PKT_OFFSET_ALL_MASK; 841 842 /* Offset is in */ 843 val |= MVNETA_RXQ_PKT_OFFSET_MASK(offset >> 3); 844 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 845 } 846 847 848 /* Tx descriptors helper methods */ 849 850 /* Update HW with number of TX descriptors to be sent */ 851 static void mvneta_txq_pend_desc_add(struct mvneta_port *pp, 852 struct mvneta_tx_queue *txq, 853 int pend_desc) 854 { 855 u32 val; 856 857 pend_desc += txq->pending; 858 859 /* Only 255 Tx descriptors can be added at once */ 860 do { 861 val = min(pend_desc, 255); 862 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val); 863 pend_desc -= val; 864 } while (pend_desc > 0); 865 txq->pending = 0; 866 } 867 868 /* Get pointer to next TX descriptor to be processed (send) by HW */ 869 static struct mvneta_tx_desc * 870 mvneta_txq_next_desc_get(struct mvneta_tx_queue *txq) 871 { 872 int tx_desc = txq->next_desc_to_proc; 873 874 txq->next_desc_to_proc = MVNETA_QUEUE_NEXT_DESC(txq, tx_desc); 875 return txq->descs + tx_desc; 876 } 877 878 /* Release the last allocated TX descriptor. Useful to handle DMA 879 * mapping failures in the TX path. 880 */ 881 static void mvneta_txq_desc_put(struct mvneta_tx_queue *txq) 882 { 883 if (txq->next_desc_to_proc == 0) 884 txq->next_desc_to_proc = txq->last_desc - 1; 885 else 886 txq->next_desc_to_proc--; 887 } 888 889 /* Set rxq buf size */ 890 static void mvneta_rxq_buf_size_set(struct mvneta_port *pp, 891 struct mvneta_rx_queue *rxq, 892 int buf_size) 893 { 894 u32 val; 895 896 val = mvreg_read(pp, MVNETA_RXQ_SIZE_REG(rxq->id)); 897 898 val &= ~MVNETA_RXQ_BUF_SIZE_MASK; 899 val |= ((buf_size >> 3) << MVNETA_RXQ_BUF_SIZE_SHIFT); 900 901 mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), val); 902 } 903 904 /* Disable buffer management (BM) */ 905 static void mvneta_rxq_bm_disable(struct mvneta_port *pp, 906 struct mvneta_rx_queue *rxq) 907 { 908 u32 val; 909 910 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 911 val &= ~MVNETA_RXQ_HW_BUF_ALLOC; 912 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 913 } 914 915 /* Enable buffer management (BM) */ 916 static void mvneta_rxq_bm_enable(struct mvneta_port *pp, 917 struct mvneta_rx_queue *rxq) 918 { 919 u32 val; 920 921 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 922 val |= MVNETA_RXQ_HW_BUF_ALLOC; 923 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 924 } 925 926 /* Notify HW about port's assignment of pool for bigger packets */ 927 static void mvneta_rxq_long_pool_set(struct mvneta_port *pp, 928 struct mvneta_rx_queue *rxq) 929 { 930 u32 val; 931 932 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 933 val &= ~MVNETA_RXQ_LONG_POOL_ID_MASK; 934 val |= (pp->pool_long->id << MVNETA_RXQ_LONG_POOL_ID_SHIFT); 935 936 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 937 } 938 939 /* Notify HW about port's assignment of pool for smaller packets */ 940 static void mvneta_rxq_short_pool_set(struct mvneta_port *pp, 941 struct mvneta_rx_queue *rxq) 942 { 943 u32 val; 944 945 val = mvreg_read(pp, MVNETA_RXQ_CONFIG_REG(rxq->id)); 946 val &= ~MVNETA_RXQ_SHORT_POOL_ID_MASK; 947 val |= (pp->pool_short->id << MVNETA_RXQ_SHORT_POOL_ID_SHIFT); 948 949 mvreg_write(pp, MVNETA_RXQ_CONFIG_REG(rxq->id), val); 950 } 951 952 /* Set port's receive buffer size for assigned BM pool */ 953 static inline void mvneta_bm_pool_bufsize_set(struct mvneta_port *pp, 954 int buf_size, 955 u8 pool_id) 956 { 957 u32 val; 958 959 if (!IS_ALIGNED(buf_size, 8)) { 960 dev_warn(pp->dev->dev.parent, 961 "illegal buf_size value %d, round to %d\n", 962 buf_size, ALIGN(buf_size, 8)); 963 buf_size = ALIGN(buf_size, 8); 964 } 965 966 val = mvreg_read(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id)); 967 val |= buf_size & MVNETA_PORT_POOL_BUFFER_SZ_MASK; 968 mvreg_write(pp, MVNETA_PORT_POOL_BUFFER_SZ_REG(pool_id), val); 969 } 970 971 /* Configure MBUS window in order to enable access BM internal SRAM */ 972 static int mvneta_mbus_io_win_set(struct mvneta_port *pp, u32 base, u32 wsize, 973 u8 target, u8 attr) 974 { 975 u32 win_enable, win_protect; 976 int i; 977 978 win_enable = mvreg_read(pp, MVNETA_BASE_ADDR_ENABLE); 979 980 if (pp->bm_win_id < 0) { 981 /* Find first not occupied window */ 982 for (i = 0; i < MVNETA_MAX_DECODE_WIN; i++) { 983 if (win_enable & (1 << i)) { 984 pp->bm_win_id = i; 985 break; 986 } 987 } 988 if (i == MVNETA_MAX_DECODE_WIN) 989 return -ENOMEM; 990 } else { 991 i = pp->bm_win_id; 992 } 993 994 mvreg_write(pp, MVNETA_WIN_BASE(i), 0); 995 mvreg_write(pp, MVNETA_WIN_SIZE(i), 0); 996 997 if (i < 4) 998 mvreg_write(pp, MVNETA_WIN_REMAP(i), 0); 999 1000 mvreg_write(pp, MVNETA_WIN_BASE(i), (base & 0xffff0000) | 1001 (attr << 8) | target); 1002 1003 mvreg_write(pp, MVNETA_WIN_SIZE(i), (wsize - 1) & 0xffff0000); 1004 1005 win_protect = mvreg_read(pp, MVNETA_ACCESS_PROTECT_ENABLE); 1006 win_protect |= 3 << (2 * i); 1007 mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect); 1008 1009 win_enable &= ~(1 << i); 1010 mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable); 1011 1012 return 0; 1013 } 1014 1015 static int mvneta_bm_port_mbus_init(struct mvneta_port *pp) 1016 { 1017 u32 wsize; 1018 u8 target, attr; 1019 int err; 1020 1021 /* Get BM window information */ 1022 err = mvebu_mbus_get_io_win_info(pp->bm_priv->bppi_phys_addr, &wsize, 1023 &target, &attr); 1024 if (err < 0) 1025 return err; 1026 1027 pp->bm_win_id = -1; 1028 1029 /* Open NETA -> BM window */ 1030 err = mvneta_mbus_io_win_set(pp, pp->bm_priv->bppi_phys_addr, wsize, 1031 target, attr); 1032 if (err < 0) { 1033 netdev_info(pp->dev, "fail to configure mbus window to BM\n"); 1034 return err; 1035 } 1036 return 0; 1037 } 1038 1039 /* Assign and initialize pools for port. In case of fail 1040 * buffer manager will remain disabled for current port. 1041 */ 1042 static int mvneta_bm_port_init(struct platform_device *pdev, 1043 struct mvneta_port *pp) 1044 { 1045 struct device_node *dn = pdev->dev.of_node; 1046 u32 long_pool_id, short_pool_id; 1047 1048 if (!pp->neta_armada3700) { 1049 int ret; 1050 1051 ret = mvneta_bm_port_mbus_init(pp); 1052 if (ret) 1053 return ret; 1054 } 1055 1056 if (of_property_read_u32(dn, "bm,pool-long", &long_pool_id)) { 1057 netdev_info(pp->dev, "missing long pool id\n"); 1058 return -EINVAL; 1059 } 1060 1061 /* Create port's long pool depending on mtu */ 1062 pp->pool_long = mvneta_bm_pool_use(pp->bm_priv, long_pool_id, 1063 MVNETA_BM_LONG, pp->id, 1064 MVNETA_RX_PKT_SIZE(pp->dev->mtu)); 1065 if (!pp->pool_long) { 1066 netdev_info(pp->dev, "fail to obtain long pool for port\n"); 1067 return -ENOMEM; 1068 } 1069 1070 pp->pool_long->port_map |= 1 << pp->id; 1071 1072 mvneta_bm_pool_bufsize_set(pp, pp->pool_long->buf_size, 1073 pp->pool_long->id); 1074 1075 /* If short pool id is not defined, assume using single pool */ 1076 if (of_property_read_u32(dn, "bm,pool-short", &short_pool_id)) 1077 short_pool_id = long_pool_id; 1078 1079 /* Create port's short pool */ 1080 pp->pool_short = mvneta_bm_pool_use(pp->bm_priv, short_pool_id, 1081 MVNETA_BM_SHORT, pp->id, 1082 MVNETA_BM_SHORT_PKT_SIZE); 1083 if (!pp->pool_short) { 1084 netdev_info(pp->dev, "fail to obtain short pool for port\n"); 1085 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 1086 return -ENOMEM; 1087 } 1088 1089 if (short_pool_id != long_pool_id) { 1090 pp->pool_short->port_map |= 1 << pp->id; 1091 mvneta_bm_pool_bufsize_set(pp, pp->pool_short->buf_size, 1092 pp->pool_short->id); 1093 } 1094 1095 return 0; 1096 } 1097 1098 /* Update settings of a pool for bigger packets */ 1099 static void mvneta_bm_update_mtu(struct mvneta_port *pp, int mtu) 1100 { 1101 struct mvneta_bm_pool *bm_pool = pp->pool_long; 1102 struct hwbm_pool *hwbm_pool = &bm_pool->hwbm_pool; 1103 int num; 1104 1105 /* Release all buffers from long pool */ 1106 mvneta_bm_bufs_free(pp->bm_priv, bm_pool, 1 << pp->id); 1107 if (hwbm_pool->buf_num) { 1108 WARN(1, "cannot free all buffers in pool %d\n", 1109 bm_pool->id); 1110 goto bm_mtu_err; 1111 } 1112 1113 bm_pool->pkt_size = MVNETA_RX_PKT_SIZE(mtu); 1114 bm_pool->buf_size = MVNETA_RX_BUF_SIZE(bm_pool->pkt_size); 1115 hwbm_pool->frag_size = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) + 1116 SKB_DATA_ALIGN(MVNETA_RX_BUF_SIZE(bm_pool->pkt_size)); 1117 1118 /* Fill entire long pool */ 1119 num = hwbm_pool_add(hwbm_pool, hwbm_pool->size, GFP_ATOMIC); 1120 if (num != hwbm_pool->size) { 1121 WARN(1, "pool %d: %d of %d allocated\n", 1122 bm_pool->id, num, hwbm_pool->size); 1123 goto bm_mtu_err; 1124 } 1125 mvneta_bm_pool_bufsize_set(pp, bm_pool->buf_size, bm_pool->id); 1126 1127 return; 1128 1129 bm_mtu_err: 1130 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 1131 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 1 << pp->id); 1132 1133 pp->bm_priv = NULL; 1134 mvreg_write(pp, MVNETA_ACC_MODE, MVNETA_ACC_MODE_EXT1); 1135 netdev_info(pp->dev, "fail to update MTU, fall back to software BM\n"); 1136 } 1137 1138 /* Start the Ethernet port RX and TX activity */ 1139 static void mvneta_port_up(struct mvneta_port *pp) 1140 { 1141 int queue; 1142 u32 q_map; 1143 1144 /* Enable all initialized TXs. */ 1145 q_map = 0; 1146 for (queue = 0; queue < txq_number; queue++) { 1147 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 1148 if (txq->descs) 1149 q_map |= (1 << queue); 1150 } 1151 mvreg_write(pp, MVNETA_TXQ_CMD, q_map); 1152 1153 q_map = 0; 1154 /* Enable all initialized RXQs. */ 1155 for (queue = 0; queue < rxq_number; queue++) { 1156 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 1157 1158 if (rxq->descs) 1159 q_map |= (1 << queue); 1160 } 1161 mvreg_write(pp, MVNETA_RXQ_CMD, q_map); 1162 } 1163 1164 /* Stop the Ethernet port activity */ 1165 static void mvneta_port_down(struct mvneta_port *pp) 1166 { 1167 u32 val; 1168 int count; 1169 1170 /* Stop Rx port activity. Check port Rx activity. */ 1171 val = mvreg_read(pp, MVNETA_RXQ_CMD) & MVNETA_RXQ_ENABLE_MASK; 1172 1173 /* Issue stop command for active channels only */ 1174 if (val != 0) 1175 mvreg_write(pp, MVNETA_RXQ_CMD, 1176 val << MVNETA_RXQ_DISABLE_SHIFT); 1177 1178 /* Wait for all Rx activity to terminate. */ 1179 count = 0; 1180 do { 1181 if (count++ >= MVNETA_RX_DISABLE_TIMEOUT_MSEC) { 1182 netdev_warn(pp->dev, 1183 "TIMEOUT for RX stopped ! rx_queue_cmd: 0x%08x\n", 1184 val); 1185 break; 1186 } 1187 mdelay(1); 1188 1189 val = mvreg_read(pp, MVNETA_RXQ_CMD); 1190 } while (val & MVNETA_RXQ_ENABLE_MASK); 1191 1192 /* Stop Tx port activity. Check port Tx activity. Issue stop 1193 * command for active channels only 1194 */ 1195 val = (mvreg_read(pp, MVNETA_TXQ_CMD)) & MVNETA_TXQ_ENABLE_MASK; 1196 1197 if (val != 0) 1198 mvreg_write(pp, MVNETA_TXQ_CMD, 1199 (val << MVNETA_TXQ_DISABLE_SHIFT)); 1200 1201 /* Wait for all Tx activity to terminate. */ 1202 count = 0; 1203 do { 1204 if (count++ >= MVNETA_TX_DISABLE_TIMEOUT_MSEC) { 1205 netdev_warn(pp->dev, 1206 "TIMEOUT for TX stopped status=0x%08x\n", 1207 val); 1208 break; 1209 } 1210 mdelay(1); 1211 1212 /* Check TX Command reg that all Txqs are stopped */ 1213 val = mvreg_read(pp, MVNETA_TXQ_CMD); 1214 1215 } while (val & MVNETA_TXQ_ENABLE_MASK); 1216 1217 /* Double check to verify that TX FIFO is empty */ 1218 count = 0; 1219 do { 1220 if (count++ >= MVNETA_TX_FIFO_EMPTY_TIMEOUT) { 1221 netdev_warn(pp->dev, 1222 "TX FIFO empty timeout status=0x%08x\n", 1223 val); 1224 break; 1225 } 1226 mdelay(1); 1227 1228 val = mvreg_read(pp, MVNETA_PORT_STATUS); 1229 } while (!(val & MVNETA_TX_FIFO_EMPTY) && 1230 (val & MVNETA_TX_IN_PRGRS)); 1231 1232 udelay(200); 1233 } 1234 1235 /* Enable the port by setting the port enable bit of the MAC control register */ 1236 static void mvneta_port_enable(struct mvneta_port *pp) 1237 { 1238 u32 val; 1239 1240 /* Enable port */ 1241 val = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 1242 val |= MVNETA_GMAC0_PORT_ENABLE; 1243 mvreg_write(pp, MVNETA_GMAC_CTRL_0, val); 1244 } 1245 1246 /* Disable the port and wait for about 200 usec before retuning */ 1247 static void mvneta_port_disable(struct mvneta_port *pp) 1248 { 1249 u32 val; 1250 1251 /* Reset the Enable bit in the Serial Control Register */ 1252 val = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 1253 val &= ~MVNETA_GMAC0_PORT_ENABLE; 1254 mvreg_write(pp, MVNETA_GMAC_CTRL_0, val); 1255 1256 udelay(200); 1257 } 1258 1259 /* Multicast tables methods */ 1260 1261 /* Set all entries in Unicast MAC Table; queue==-1 means reject all */ 1262 static void mvneta_set_ucast_table(struct mvneta_port *pp, int queue) 1263 { 1264 int offset; 1265 u32 val; 1266 1267 if (queue == -1) { 1268 val = 0; 1269 } else { 1270 val = 0x1 | (queue << 1); 1271 val |= (val << 24) | (val << 16) | (val << 8); 1272 } 1273 1274 for (offset = 0; offset <= 0xc; offset += 4) 1275 mvreg_write(pp, MVNETA_DA_FILT_UCAST_BASE + offset, val); 1276 } 1277 1278 /* Set all entries in Special Multicast MAC Table; queue==-1 means reject all */ 1279 static void mvneta_set_special_mcast_table(struct mvneta_port *pp, int queue) 1280 { 1281 int offset; 1282 u32 val; 1283 1284 if (queue == -1) { 1285 val = 0; 1286 } else { 1287 val = 0x1 | (queue << 1); 1288 val |= (val << 24) | (val << 16) | (val << 8); 1289 } 1290 1291 for (offset = 0; offset <= 0xfc; offset += 4) 1292 mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + offset, val); 1293 1294 } 1295 1296 /* Set all entries in Other Multicast MAC Table. queue==-1 means reject all */ 1297 static void mvneta_set_other_mcast_table(struct mvneta_port *pp, int queue) 1298 { 1299 int offset; 1300 u32 val; 1301 1302 if (queue == -1) { 1303 memset(pp->mcast_count, 0, sizeof(pp->mcast_count)); 1304 val = 0; 1305 } else { 1306 memset(pp->mcast_count, 1, sizeof(pp->mcast_count)); 1307 val = 0x1 | (queue << 1); 1308 val |= (val << 24) | (val << 16) | (val << 8); 1309 } 1310 1311 for (offset = 0; offset <= 0xfc; offset += 4) 1312 mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + offset, val); 1313 } 1314 1315 static void mvneta_percpu_unmask_interrupt(void *arg) 1316 { 1317 struct mvneta_port *pp = arg; 1318 1319 /* All the queue are unmasked, but actually only the ones 1320 * mapped to this CPU will be unmasked 1321 */ 1322 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 1323 MVNETA_RX_INTR_MASK_ALL | 1324 MVNETA_TX_INTR_MASK_ALL | 1325 MVNETA_MISCINTR_INTR_MASK); 1326 } 1327 1328 static void mvneta_percpu_mask_interrupt(void *arg) 1329 { 1330 struct mvneta_port *pp = arg; 1331 1332 /* All the queue are masked, but actually only the ones 1333 * mapped to this CPU will be masked 1334 */ 1335 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0); 1336 mvreg_write(pp, MVNETA_INTR_OLD_MASK, 0); 1337 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 0); 1338 } 1339 1340 static void mvneta_percpu_clear_intr_cause(void *arg) 1341 { 1342 struct mvneta_port *pp = arg; 1343 1344 /* All the queue are cleared, but actually only the ones 1345 * mapped to this CPU will be cleared 1346 */ 1347 mvreg_write(pp, MVNETA_INTR_NEW_CAUSE, 0); 1348 mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0); 1349 mvreg_write(pp, MVNETA_INTR_OLD_CAUSE, 0); 1350 } 1351 1352 /* This method sets defaults to the NETA port: 1353 * Clears interrupt Cause and Mask registers. 1354 * Clears all MAC tables. 1355 * Sets defaults to all registers. 1356 * Resets RX and TX descriptor rings. 1357 * Resets PHY. 1358 * This method can be called after mvneta_port_down() to return the port 1359 * settings to defaults. 1360 */ 1361 static void mvneta_defaults_set(struct mvneta_port *pp) 1362 { 1363 int cpu; 1364 int queue; 1365 u32 val; 1366 int max_cpu = num_present_cpus(); 1367 1368 /* Clear all Cause registers */ 1369 on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true); 1370 1371 /* Mask all interrupts */ 1372 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 1373 mvreg_write(pp, MVNETA_INTR_ENABLE, 0); 1374 1375 /* Enable MBUS Retry bit16 */ 1376 mvreg_write(pp, MVNETA_MBUS_RETRY, 0x20); 1377 1378 /* Set CPU queue access map. CPUs are assigned to the RX and 1379 * TX queues modulo their number. If there is only one TX 1380 * queue then it is assigned to the CPU associated to the 1381 * default RX queue. 1382 */ 1383 for_each_present_cpu(cpu) { 1384 int rxq_map = 0, txq_map = 0; 1385 int rxq, txq; 1386 if (!pp->neta_armada3700) { 1387 for (rxq = 0; rxq < rxq_number; rxq++) 1388 if ((rxq % max_cpu) == cpu) 1389 rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq); 1390 1391 for (txq = 0; txq < txq_number; txq++) 1392 if ((txq % max_cpu) == cpu) 1393 txq_map |= MVNETA_CPU_TXQ_ACCESS(txq); 1394 1395 /* With only one TX queue we configure a special case 1396 * which will allow to get all the irq on a single 1397 * CPU 1398 */ 1399 if (txq_number == 1) 1400 txq_map = (cpu == pp->rxq_def) ? 1401 MVNETA_CPU_TXQ_ACCESS(1) : 0; 1402 1403 } else { 1404 txq_map = MVNETA_CPU_TXQ_ACCESS_ALL_MASK; 1405 rxq_map = MVNETA_CPU_RXQ_ACCESS_ALL_MASK; 1406 } 1407 1408 mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map); 1409 } 1410 1411 /* Reset RX and TX DMAs */ 1412 mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET); 1413 mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET); 1414 1415 /* Disable Legacy WRR, Disable EJP, Release from reset */ 1416 mvreg_write(pp, MVNETA_TXQ_CMD_1, 0); 1417 for (queue = 0; queue < txq_number; queue++) { 1418 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(queue), 0); 1419 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(queue), 0); 1420 } 1421 1422 mvreg_write(pp, MVNETA_PORT_TX_RESET, 0); 1423 mvreg_write(pp, MVNETA_PORT_RX_RESET, 0); 1424 1425 /* Set Port Acceleration Mode */ 1426 if (pp->bm_priv) 1427 /* HW buffer management + legacy parser */ 1428 val = MVNETA_ACC_MODE_EXT2; 1429 else 1430 /* SW buffer management + legacy parser */ 1431 val = MVNETA_ACC_MODE_EXT1; 1432 mvreg_write(pp, MVNETA_ACC_MODE, val); 1433 1434 if (pp->bm_priv) 1435 mvreg_write(pp, MVNETA_BM_ADDRESS, pp->bm_priv->bppi_phys_addr); 1436 1437 /* Update val of portCfg register accordingly with all RxQueue types */ 1438 val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def); 1439 mvreg_write(pp, MVNETA_PORT_CONFIG, val); 1440 1441 val = 0; 1442 mvreg_write(pp, MVNETA_PORT_CONFIG_EXTEND, val); 1443 mvreg_write(pp, MVNETA_RX_MIN_FRAME_SIZE, 64); 1444 1445 /* Build PORT_SDMA_CONFIG_REG */ 1446 val = 0; 1447 1448 /* Default burst size */ 1449 val |= MVNETA_TX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16); 1450 val |= MVNETA_RX_BRST_SZ_MASK(MVNETA_SDMA_BRST_SIZE_16); 1451 val |= MVNETA_RX_NO_DATA_SWAP | MVNETA_TX_NO_DATA_SWAP; 1452 1453 #if defined(__BIG_ENDIAN) 1454 val |= MVNETA_DESC_SWAP; 1455 #endif 1456 1457 /* Assign port SDMA configuration */ 1458 mvreg_write(pp, MVNETA_SDMA_CONFIG, val); 1459 1460 /* Disable PHY polling in hardware, since we're using the 1461 * kernel phylib to do this. 1462 */ 1463 val = mvreg_read(pp, MVNETA_UNIT_CONTROL); 1464 val &= ~MVNETA_PHY_POLLING_ENABLE; 1465 mvreg_write(pp, MVNETA_UNIT_CONTROL, val); 1466 1467 mvneta_set_ucast_table(pp, -1); 1468 mvneta_set_special_mcast_table(pp, -1); 1469 mvneta_set_other_mcast_table(pp, -1); 1470 1471 /* Set port interrupt enable register - default enable all */ 1472 mvreg_write(pp, MVNETA_INTR_ENABLE, 1473 (MVNETA_RXQ_INTR_ENABLE_ALL_MASK 1474 | MVNETA_TXQ_INTR_ENABLE_ALL_MASK)); 1475 1476 mvneta_mib_counters_clear(pp); 1477 } 1478 1479 /* Set max sizes for tx queues */ 1480 static void mvneta_txq_max_tx_size_set(struct mvneta_port *pp, int max_tx_size) 1481 1482 { 1483 u32 val, size, mtu; 1484 int queue; 1485 1486 mtu = max_tx_size * 8; 1487 if (mtu > MVNETA_TX_MTU_MAX) 1488 mtu = MVNETA_TX_MTU_MAX; 1489 1490 /* Set MTU */ 1491 val = mvreg_read(pp, MVNETA_TX_MTU); 1492 val &= ~MVNETA_TX_MTU_MAX; 1493 val |= mtu; 1494 mvreg_write(pp, MVNETA_TX_MTU, val); 1495 1496 /* TX token size and all TXQs token size must be larger that MTU */ 1497 val = mvreg_read(pp, MVNETA_TX_TOKEN_SIZE); 1498 1499 size = val & MVNETA_TX_TOKEN_SIZE_MAX; 1500 if (size < mtu) { 1501 size = mtu; 1502 val &= ~MVNETA_TX_TOKEN_SIZE_MAX; 1503 val |= size; 1504 mvreg_write(pp, MVNETA_TX_TOKEN_SIZE, val); 1505 } 1506 for (queue = 0; queue < txq_number; queue++) { 1507 val = mvreg_read(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue)); 1508 1509 size = val & MVNETA_TXQ_TOKEN_SIZE_MAX; 1510 if (size < mtu) { 1511 size = mtu; 1512 val &= ~MVNETA_TXQ_TOKEN_SIZE_MAX; 1513 val |= size; 1514 mvreg_write(pp, MVNETA_TXQ_TOKEN_SIZE_REG(queue), val); 1515 } 1516 } 1517 } 1518 1519 /* Set unicast address */ 1520 static void mvneta_set_ucast_addr(struct mvneta_port *pp, u8 last_nibble, 1521 int queue) 1522 { 1523 unsigned int unicast_reg; 1524 unsigned int tbl_offset; 1525 unsigned int reg_offset; 1526 1527 /* Locate the Unicast table entry */ 1528 last_nibble = (0xf & last_nibble); 1529 1530 /* offset from unicast tbl base */ 1531 tbl_offset = (last_nibble / 4) * 4; 1532 1533 /* offset within the above reg */ 1534 reg_offset = last_nibble % 4; 1535 1536 unicast_reg = mvreg_read(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset)); 1537 1538 if (queue == -1) { 1539 /* Clear accepts frame bit at specified unicast DA tbl entry */ 1540 unicast_reg &= ~(0xff << (8 * reg_offset)); 1541 } else { 1542 unicast_reg &= ~(0xff << (8 * reg_offset)); 1543 unicast_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset)); 1544 } 1545 1546 mvreg_write(pp, (MVNETA_DA_FILT_UCAST_BASE + tbl_offset), unicast_reg); 1547 } 1548 1549 /* Set mac address */ 1550 static void mvneta_mac_addr_set(struct mvneta_port *pp, unsigned char *addr, 1551 int queue) 1552 { 1553 unsigned int mac_h; 1554 unsigned int mac_l; 1555 1556 if (queue != -1) { 1557 mac_l = (addr[4] << 8) | (addr[5]); 1558 mac_h = (addr[0] << 24) | (addr[1] << 16) | 1559 (addr[2] << 8) | (addr[3] << 0); 1560 1561 mvreg_write(pp, MVNETA_MAC_ADDR_LOW, mac_l); 1562 mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, mac_h); 1563 } 1564 1565 /* Accept frames of this address */ 1566 mvneta_set_ucast_addr(pp, addr[5], queue); 1567 } 1568 1569 /* Set the number of packets that will be received before RX interrupt 1570 * will be generated by HW. 1571 */ 1572 static void mvneta_rx_pkts_coal_set(struct mvneta_port *pp, 1573 struct mvneta_rx_queue *rxq, u32 value) 1574 { 1575 mvreg_write(pp, MVNETA_RXQ_THRESHOLD_REG(rxq->id), 1576 value | MVNETA_RXQ_NON_OCCUPIED(0)); 1577 } 1578 1579 /* Set the time delay in usec before RX interrupt will be generated by 1580 * HW. 1581 */ 1582 static void mvneta_rx_time_coal_set(struct mvneta_port *pp, 1583 struct mvneta_rx_queue *rxq, u32 value) 1584 { 1585 u32 val; 1586 unsigned long clk_rate; 1587 1588 clk_rate = clk_get_rate(pp->clk); 1589 val = (clk_rate / 1000000) * value; 1590 1591 mvreg_write(pp, MVNETA_RXQ_TIME_COAL_REG(rxq->id), val); 1592 } 1593 1594 /* Set threshold for TX_DONE pkts coalescing */ 1595 static void mvneta_tx_done_pkts_coal_set(struct mvneta_port *pp, 1596 struct mvneta_tx_queue *txq, u32 value) 1597 { 1598 u32 val; 1599 1600 val = mvreg_read(pp, MVNETA_TXQ_SIZE_REG(txq->id)); 1601 1602 val &= ~MVNETA_TXQ_SENT_THRESH_ALL_MASK; 1603 val |= MVNETA_TXQ_SENT_THRESH_MASK(value); 1604 1605 mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), val); 1606 } 1607 1608 /* Handle rx descriptor fill by setting buf_cookie and buf_phys_addr */ 1609 static void mvneta_rx_desc_fill(struct mvneta_rx_desc *rx_desc, 1610 u32 phys_addr, void *virt_addr, 1611 struct mvneta_rx_queue *rxq) 1612 { 1613 int i; 1614 1615 rx_desc->buf_phys_addr = phys_addr; 1616 i = rx_desc - rxq->descs; 1617 rxq->buf_virt_addr[i] = virt_addr; 1618 } 1619 1620 /* Decrement sent descriptors counter */ 1621 static void mvneta_txq_sent_desc_dec(struct mvneta_port *pp, 1622 struct mvneta_tx_queue *txq, 1623 int sent_desc) 1624 { 1625 u32 val; 1626 1627 /* Only 255 TX descriptors can be updated at once */ 1628 while (sent_desc > 0xff) { 1629 val = 0xff << MVNETA_TXQ_DEC_SENT_SHIFT; 1630 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val); 1631 sent_desc = sent_desc - 0xff; 1632 } 1633 1634 val = sent_desc << MVNETA_TXQ_DEC_SENT_SHIFT; 1635 mvreg_write(pp, MVNETA_TXQ_UPDATE_REG(txq->id), val); 1636 } 1637 1638 /* Get number of TX descriptors already sent by HW */ 1639 static int mvneta_txq_sent_desc_num_get(struct mvneta_port *pp, 1640 struct mvneta_tx_queue *txq) 1641 { 1642 u32 val; 1643 int sent_desc; 1644 1645 val = mvreg_read(pp, MVNETA_TXQ_STATUS_REG(txq->id)); 1646 sent_desc = (val & MVNETA_TXQ_SENT_DESC_MASK) >> 1647 MVNETA_TXQ_SENT_DESC_SHIFT; 1648 1649 return sent_desc; 1650 } 1651 1652 /* Get number of sent descriptors and decrement counter. 1653 * The number of sent descriptors is returned. 1654 */ 1655 static int mvneta_txq_sent_desc_proc(struct mvneta_port *pp, 1656 struct mvneta_tx_queue *txq) 1657 { 1658 int sent_desc; 1659 1660 /* Get number of sent descriptors */ 1661 sent_desc = mvneta_txq_sent_desc_num_get(pp, txq); 1662 1663 /* Decrement sent descriptors counter */ 1664 if (sent_desc) 1665 mvneta_txq_sent_desc_dec(pp, txq, sent_desc); 1666 1667 return sent_desc; 1668 } 1669 1670 /* Set TXQ descriptors fields relevant for CSUM calculation */ 1671 static u32 mvneta_txq_desc_csum(int l3_offs, int l3_proto, 1672 int ip_hdr_len, int l4_proto) 1673 { 1674 u32 command; 1675 1676 /* Fields: L3_offset, IP_hdrlen, L3_type, G_IPv4_chk, 1677 * G_L4_chk, L4_type; required only for checksum 1678 * calculation 1679 */ 1680 command = l3_offs << MVNETA_TX_L3_OFF_SHIFT; 1681 command |= ip_hdr_len << MVNETA_TX_IP_HLEN_SHIFT; 1682 1683 if (l3_proto == htons(ETH_P_IP)) 1684 command |= MVNETA_TXD_IP_CSUM; 1685 else 1686 command |= MVNETA_TX_L3_IP6; 1687 1688 if (l4_proto == IPPROTO_TCP) 1689 command |= MVNETA_TX_L4_CSUM_FULL; 1690 else if (l4_proto == IPPROTO_UDP) 1691 command |= MVNETA_TX_L4_UDP | MVNETA_TX_L4_CSUM_FULL; 1692 else 1693 command |= MVNETA_TX_L4_CSUM_NOT; 1694 1695 return command; 1696 } 1697 1698 1699 /* Display more error info */ 1700 static void mvneta_rx_error(struct mvneta_port *pp, 1701 struct mvneta_rx_desc *rx_desc) 1702 { 1703 u32 status = rx_desc->status; 1704 1705 switch (status & MVNETA_RXD_ERR_CODE_MASK) { 1706 case MVNETA_RXD_ERR_CRC: 1707 netdev_err(pp->dev, "bad rx status %08x (crc error), size=%d\n", 1708 status, rx_desc->data_size); 1709 break; 1710 case MVNETA_RXD_ERR_OVERRUN: 1711 netdev_err(pp->dev, "bad rx status %08x (overrun error), size=%d\n", 1712 status, rx_desc->data_size); 1713 break; 1714 case MVNETA_RXD_ERR_LEN: 1715 netdev_err(pp->dev, "bad rx status %08x (max frame length error), size=%d\n", 1716 status, rx_desc->data_size); 1717 break; 1718 case MVNETA_RXD_ERR_RESOURCE: 1719 netdev_err(pp->dev, "bad rx status %08x (resource error), size=%d\n", 1720 status, rx_desc->data_size); 1721 break; 1722 } 1723 } 1724 1725 /* Handle RX checksum offload based on the descriptor's status */ 1726 static void mvneta_rx_csum(struct mvneta_port *pp, u32 status, 1727 struct sk_buff *skb) 1728 { 1729 if ((pp->dev->features & NETIF_F_RXCSUM) && 1730 (status & MVNETA_RXD_L3_IP4) && 1731 (status & MVNETA_RXD_L4_CSUM_OK)) { 1732 skb->csum = 0; 1733 skb->ip_summed = CHECKSUM_UNNECESSARY; 1734 return; 1735 } 1736 1737 skb->ip_summed = CHECKSUM_NONE; 1738 } 1739 1740 /* Return tx queue pointer (find last set bit) according to <cause> returned 1741 * form tx_done reg. <cause> must not be null. The return value is always a 1742 * valid queue for matching the first one found in <cause>. 1743 */ 1744 static struct mvneta_tx_queue *mvneta_tx_done_policy(struct mvneta_port *pp, 1745 u32 cause) 1746 { 1747 int queue = fls(cause) - 1; 1748 1749 return &pp->txqs[queue]; 1750 } 1751 1752 /* Free tx queue skbuffs */ 1753 static void mvneta_txq_bufs_free(struct mvneta_port *pp, 1754 struct mvneta_tx_queue *txq, int num, 1755 struct netdev_queue *nq) 1756 { 1757 unsigned int bytes_compl = 0, pkts_compl = 0; 1758 int i; 1759 1760 for (i = 0; i < num; i++) { 1761 struct mvneta_tx_desc *tx_desc = txq->descs + 1762 txq->txq_get_index; 1763 struct sk_buff *skb = txq->tx_skb[txq->txq_get_index]; 1764 1765 if (skb) { 1766 bytes_compl += skb->len; 1767 pkts_compl++; 1768 } 1769 1770 mvneta_txq_inc_get(txq); 1771 1772 if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr)) 1773 dma_unmap_single(pp->dev->dev.parent, 1774 tx_desc->buf_phys_addr, 1775 tx_desc->data_size, DMA_TO_DEVICE); 1776 if (!skb) 1777 continue; 1778 dev_kfree_skb_any(skb); 1779 } 1780 1781 netdev_tx_completed_queue(nq, pkts_compl, bytes_compl); 1782 } 1783 1784 /* Handle end of transmission */ 1785 static void mvneta_txq_done(struct mvneta_port *pp, 1786 struct mvneta_tx_queue *txq) 1787 { 1788 struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id); 1789 int tx_done; 1790 1791 tx_done = mvneta_txq_sent_desc_proc(pp, txq); 1792 if (!tx_done) 1793 return; 1794 1795 mvneta_txq_bufs_free(pp, txq, tx_done, nq); 1796 1797 txq->count -= tx_done; 1798 1799 if (netif_tx_queue_stopped(nq)) { 1800 if (txq->count <= txq->tx_wake_threshold) 1801 netif_tx_wake_queue(nq); 1802 } 1803 } 1804 1805 /* Refill processing for SW buffer management */ 1806 /* Allocate page per descriptor */ 1807 static int mvneta_rx_refill(struct mvneta_port *pp, 1808 struct mvneta_rx_desc *rx_desc, 1809 struct mvneta_rx_queue *rxq, 1810 gfp_t gfp_mask) 1811 { 1812 dma_addr_t phys_addr; 1813 struct page *page; 1814 1815 page = __dev_alloc_page(gfp_mask); 1816 if (!page) 1817 return -ENOMEM; 1818 1819 /* map page for use */ 1820 phys_addr = dma_map_page(pp->dev->dev.parent, page, 0, PAGE_SIZE, 1821 DMA_FROM_DEVICE); 1822 if (unlikely(dma_mapping_error(pp->dev->dev.parent, phys_addr))) { 1823 __free_page(page); 1824 return -ENOMEM; 1825 } 1826 1827 phys_addr += pp->rx_offset_correction; 1828 mvneta_rx_desc_fill(rx_desc, phys_addr, page, rxq); 1829 return 0; 1830 } 1831 1832 /* Handle tx checksum */ 1833 static u32 mvneta_skb_tx_csum(struct mvneta_port *pp, struct sk_buff *skb) 1834 { 1835 if (skb->ip_summed == CHECKSUM_PARTIAL) { 1836 int ip_hdr_len = 0; 1837 __be16 l3_proto = vlan_get_protocol(skb); 1838 u8 l4_proto; 1839 1840 if (l3_proto == htons(ETH_P_IP)) { 1841 struct iphdr *ip4h = ip_hdr(skb); 1842 1843 /* Calculate IPv4 checksum and L4 checksum */ 1844 ip_hdr_len = ip4h->ihl; 1845 l4_proto = ip4h->protocol; 1846 } else if (l3_proto == htons(ETH_P_IPV6)) { 1847 struct ipv6hdr *ip6h = ipv6_hdr(skb); 1848 1849 /* Read l4_protocol from one of IPv6 extra headers */ 1850 if (skb_network_header_len(skb) > 0) 1851 ip_hdr_len = (skb_network_header_len(skb) >> 2); 1852 l4_proto = ip6h->nexthdr; 1853 } else 1854 return MVNETA_TX_L4_CSUM_NOT; 1855 1856 return mvneta_txq_desc_csum(skb_network_offset(skb), 1857 l3_proto, ip_hdr_len, l4_proto); 1858 } 1859 1860 return MVNETA_TX_L4_CSUM_NOT; 1861 } 1862 1863 /* Drop packets received by the RXQ and free buffers */ 1864 static void mvneta_rxq_drop_pkts(struct mvneta_port *pp, 1865 struct mvneta_rx_queue *rxq) 1866 { 1867 int rx_done, i; 1868 1869 rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); 1870 if (rx_done) 1871 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); 1872 1873 if (pp->bm_priv) { 1874 for (i = 0; i < rx_done; i++) { 1875 struct mvneta_rx_desc *rx_desc = 1876 mvneta_rxq_next_desc_get(rxq); 1877 u8 pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc); 1878 struct mvneta_bm_pool *bm_pool; 1879 1880 bm_pool = &pp->bm_priv->bm_pools[pool_id]; 1881 /* Return dropped buffer to the pool */ 1882 mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool, 1883 rx_desc->buf_phys_addr); 1884 } 1885 return; 1886 } 1887 1888 for (i = 0; i < rxq->size; i++) { 1889 struct mvneta_rx_desc *rx_desc = rxq->descs + i; 1890 void *data = rxq->buf_virt_addr[i]; 1891 if (!data || !(rx_desc->buf_phys_addr)) 1892 continue; 1893 1894 dma_unmap_page(pp->dev->dev.parent, rx_desc->buf_phys_addr, 1895 PAGE_SIZE, DMA_FROM_DEVICE); 1896 __free_page(data); 1897 } 1898 } 1899 1900 static inline 1901 int mvneta_rx_refill_queue(struct mvneta_port *pp, struct mvneta_rx_queue *rxq) 1902 { 1903 struct mvneta_rx_desc *rx_desc; 1904 int curr_desc = rxq->first_to_refill; 1905 int i; 1906 1907 for (i = 0; (i < rxq->refill_num) && (i < 64); i++) { 1908 rx_desc = rxq->descs + curr_desc; 1909 if (!(rx_desc->buf_phys_addr)) { 1910 if (mvneta_rx_refill(pp, rx_desc, rxq, GFP_ATOMIC)) { 1911 pr_err("Can't refill queue %d. Done %d from %d\n", 1912 rxq->id, i, rxq->refill_num); 1913 rxq->refill_err++; 1914 break; 1915 } 1916 } 1917 curr_desc = MVNETA_QUEUE_NEXT_DESC(rxq, curr_desc); 1918 } 1919 rxq->refill_num -= i; 1920 rxq->first_to_refill = curr_desc; 1921 1922 return i; 1923 } 1924 1925 /* Main rx processing when using software buffer management */ 1926 static int mvneta_rx_swbm(struct napi_struct *napi, 1927 struct mvneta_port *pp, int budget, 1928 struct mvneta_rx_queue *rxq) 1929 { 1930 struct net_device *dev = pp->dev; 1931 int rx_todo, rx_proc; 1932 int refill = 0; 1933 u32 rcvd_pkts = 0; 1934 u32 rcvd_bytes = 0; 1935 1936 /* Get number of received packets */ 1937 rx_todo = mvneta_rxq_busy_desc_num_get(pp, rxq); 1938 rx_proc = 0; 1939 1940 /* Fairness NAPI loop */ 1941 while ((rcvd_pkts < budget) && (rx_proc < rx_todo)) { 1942 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); 1943 unsigned char *data; 1944 struct page *page; 1945 dma_addr_t phys_addr; 1946 u32 rx_status, index; 1947 int rx_bytes, skb_size, copy_size; 1948 int frag_num, frag_size, frag_offset; 1949 1950 index = rx_desc - rxq->descs; 1951 page = (struct page *)rxq->buf_virt_addr[index]; 1952 data = page_address(page); 1953 /* Prefetch header */ 1954 prefetch(data); 1955 1956 phys_addr = rx_desc->buf_phys_addr; 1957 rx_status = rx_desc->status; 1958 rx_proc++; 1959 rxq->refill_num++; 1960 1961 if (rx_status & MVNETA_RXD_FIRST_DESC) { 1962 /* Check errors only for FIRST descriptor */ 1963 if (rx_status & MVNETA_RXD_ERR_SUMMARY) { 1964 mvneta_rx_error(pp, rx_desc); 1965 dev->stats.rx_errors++; 1966 /* leave the descriptor untouched */ 1967 continue; 1968 } 1969 rx_bytes = rx_desc->data_size - 1970 (ETH_FCS_LEN + MVNETA_MH_SIZE); 1971 1972 /* Allocate small skb for each new packet */ 1973 skb_size = max(rx_copybreak, rx_header_size); 1974 rxq->skb = netdev_alloc_skb_ip_align(dev, skb_size); 1975 if (unlikely(!rxq->skb)) { 1976 netdev_err(dev, 1977 "Can't allocate skb on queue %d\n", 1978 rxq->id); 1979 dev->stats.rx_dropped++; 1980 rxq->skb_alloc_err++; 1981 continue; 1982 } 1983 copy_size = min(skb_size, rx_bytes); 1984 1985 /* Copy data from buffer to SKB, skip Marvell header */ 1986 memcpy(rxq->skb->data, data + MVNETA_MH_SIZE, 1987 copy_size); 1988 skb_put(rxq->skb, copy_size); 1989 rxq->left_size = rx_bytes - copy_size; 1990 1991 mvneta_rx_csum(pp, rx_status, rxq->skb); 1992 if (rxq->left_size == 0) { 1993 int size = copy_size + MVNETA_MH_SIZE; 1994 1995 dma_sync_single_range_for_cpu(dev->dev.parent, 1996 phys_addr, 0, 1997 size, 1998 DMA_FROM_DEVICE); 1999 2000 /* leave the descriptor and buffer untouched */ 2001 } else { 2002 /* refill descriptor with new buffer later */ 2003 rx_desc->buf_phys_addr = 0; 2004 2005 frag_num = 0; 2006 frag_offset = copy_size + MVNETA_MH_SIZE; 2007 frag_size = min(rxq->left_size, 2008 (int)(PAGE_SIZE - frag_offset)); 2009 skb_add_rx_frag(rxq->skb, frag_num, page, 2010 frag_offset, frag_size, 2011 PAGE_SIZE); 2012 dma_unmap_page(dev->dev.parent, phys_addr, 2013 PAGE_SIZE, DMA_FROM_DEVICE); 2014 rxq->left_size -= frag_size; 2015 } 2016 } else { 2017 /* Middle or Last descriptor */ 2018 if (unlikely(!rxq->skb)) { 2019 pr_debug("no skb for rx_status 0x%x\n", 2020 rx_status); 2021 continue; 2022 } 2023 if (!rxq->left_size) { 2024 /* last descriptor has only FCS */ 2025 /* and can be discarded */ 2026 dma_sync_single_range_for_cpu(dev->dev.parent, 2027 phys_addr, 0, 2028 ETH_FCS_LEN, 2029 DMA_FROM_DEVICE); 2030 /* leave the descriptor and buffer untouched */ 2031 } else { 2032 /* refill descriptor with new buffer later */ 2033 rx_desc->buf_phys_addr = 0; 2034 2035 frag_num = skb_shinfo(rxq->skb)->nr_frags; 2036 frag_offset = 0; 2037 frag_size = min(rxq->left_size, 2038 (int)(PAGE_SIZE - frag_offset)); 2039 skb_add_rx_frag(rxq->skb, frag_num, page, 2040 frag_offset, frag_size, 2041 PAGE_SIZE); 2042 2043 dma_unmap_page(dev->dev.parent, phys_addr, 2044 PAGE_SIZE, DMA_FROM_DEVICE); 2045 2046 rxq->left_size -= frag_size; 2047 } 2048 } /* Middle or Last descriptor */ 2049 2050 if (!(rx_status & MVNETA_RXD_LAST_DESC)) 2051 /* no last descriptor this time */ 2052 continue; 2053 2054 if (rxq->left_size) { 2055 pr_err("get last desc, but left_size (%d) != 0\n", 2056 rxq->left_size); 2057 dev_kfree_skb_any(rxq->skb); 2058 rxq->left_size = 0; 2059 rxq->skb = NULL; 2060 continue; 2061 } 2062 rcvd_pkts++; 2063 rcvd_bytes += rxq->skb->len; 2064 2065 /* Linux processing */ 2066 rxq->skb->protocol = eth_type_trans(rxq->skb, dev); 2067 2068 napi_gro_receive(napi, rxq->skb); 2069 2070 /* clean uncomplete skb pointer in queue */ 2071 rxq->skb = NULL; 2072 rxq->left_size = 0; 2073 } 2074 2075 if (rcvd_pkts) { 2076 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2077 2078 u64_stats_update_begin(&stats->syncp); 2079 stats->rx_packets += rcvd_pkts; 2080 stats->rx_bytes += rcvd_bytes; 2081 u64_stats_update_end(&stats->syncp); 2082 } 2083 2084 /* return some buffers to hardware queue, one at a time is too slow */ 2085 refill = mvneta_rx_refill_queue(pp, rxq); 2086 2087 /* Update rxq management counters */ 2088 mvneta_rxq_desc_num_update(pp, rxq, rx_proc, refill); 2089 2090 return rcvd_pkts; 2091 } 2092 2093 /* Main rx processing when using hardware buffer management */ 2094 static int mvneta_rx_hwbm(struct napi_struct *napi, 2095 struct mvneta_port *pp, int rx_todo, 2096 struct mvneta_rx_queue *rxq) 2097 { 2098 struct net_device *dev = pp->dev; 2099 int rx_done; 2100 u32 rcvd_pkts = 0; 2101 u32 rcvd_bytes = 0; 2102 2103 /* Get number of received packets */ 2104 rx_done = mvneta_rxq_busy_desc_num_get(pp, rxq); 2105 2106 if (rx_todo > rx_done) 2107 rx_todo = rx_done; 2108 2109 rx_done = 0; 2110 2111 /* Fairness NAPI loop */ 2112 while (rx_done < rx_todo) { 2113 struct mvneta_rx_desc *rx_desc = mvneta_rxq_next_desc_get(rxq); 2114 struct mvneta_bm_pool *bm_pool = NULL; 2115 struct sk_buff *skb; 2116 unsigned char *data; 2117 dma_addr_t phys_addr; 2118 u32 rx_status, frag_size; 2119 int rx_bytes, err; 2120 u8 pool_id; 2121 2122 rx_done++; 2123 rx_status = rx_desc->status; 2124 rx_bytes = rx_desc->data_size - (ETH_FCS_LEN + MVNETA_MH_SIZE); 2125 data = (u8 *)(uintptr_t)rx_desc->buf_cookie; 2126 phys_addr = rx_desc->buf_phys_addr; 2127 pool_id = MVNETA_RX_GET_BM_POOL_ID(rx_desc); 2128 bm_pool = &pp->bm_priv->bm_pools[pool_id]; 2129 2130 if (!mvneta_rxq_desc_is_first_last(rx_status) || 2131 (rx_status & MVNETA_RXD_ERR_SUMMARY)) { 2132 err_drop_frame_ret_pool: 2133 /* Return the buffer to the pool */ 2134 mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool, 2135 rx_desc->buf_phys_addr); 2136 err_drop_frame: 2137 dev->stats.rx_errors++; 2138 mvneta_rx_error(pp, rx_desc); 2139 /* leave the descriptor untouched */ 2140 continue; 2141 } 2142 2143 if (rx_bytes <= rx_copybreak) { 2144 /* better copy a small frame and not unmap the DMA region */ 2145 skb = netdev_alloc_skb_ip_align(dev, rx_bytes); 2146 if (unlikely(!skb)) 2147 goto err_drop_frame_ret_pool; 2148 2149 dma_sync_single_range_for_cpu(dev->dev.parent, 2150 rx_desc->buf_phys_addr, 2151 MVNETA_MH_SIZE + NET_SKB_PAD, 2152 rx_bytes, 2153 DMA_FROM_DEVICE); 2154 skb_put_data(skb, data + MVNETA_MH_SIZE + NET_SKB_PAD, 2155 rx_bytes); 2156 2157 skb->protocol = eth_type_trans(skb, dev); 2158 mvneta_rx_csum(pp, rx_status, skb); 2159 napi_gro_receive(napi, skb); 2160 2161 rcvd_pkts++; 2162 rcvd_bytes += rx_bytes; 2163 2164 /* Return the buffer to the pool */ 2165 mvneta_bm_pool_put_bp(pp->bm_priv, bm_pool, 2166 rx_desc->buf_phys_addr); 2167 2168 /* leave the descriptor and buffer untouched */ 2169 continue; 2170 } 2171 2172 /* Refill processing */ 2173 err = hwbm_pool_refill(&bm_pool->hwbm_pool, GFP_ATOMIC); 2174 if (err) { 2175 netdev_err(dev, "Linux processing - Can't refill\n"); 2176 rxq->refill_err++; 2177 goto err_drop_frame_ret_pool; 2178 } 2179 2180 frag_size = bm_pool->hwbm_pool.frag_size; 2181 2182 skb = build_skb(data, frag_size > PAGE_SIZE ? 0 : frag_size); 2183 2184 /* After refill old buffer has to be unmapped regardless 2185 * the skb is successfully built or not. 2186 */ 2187 dma_unmap_single(&pp->bm_priv->pdev->dev, phys_addr, 2188 bm_pool->buf_size, DMA_FROM_DEVICE); 2189 if (!skb) 2190 goto err_drop_frame; 2191 2192 rcvd_pkts++; 2193 rcvd_bytes += rx_bytes; 2194 2195 /* Linux processing */ 2196 skb_reserve(skb, MVNETA_MH_SIZE + NET_SKB_PAD); 2197 skb_put(skb, rx_bytes); 2198 2199 skb->protocol = eth_type_trans(skb, dev); 2200 2201 mvneta_rx_csum(pp, rx_status, skb); 2202 2203 napi_gro_receive(napi, skb); 2204 } 2205 2206 if (rcvd_pkts) { 2207 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2208 2209 u64_stats_update_begin(&stats->syncp); 2210 stats->rx_packets += rcvd_pkts; 2211 stats->rx_bytes += rcvd_bytes; 2212 u64_stats_update_end(&stats->syncp); 2213 } 2214 2215 /* Update rxq management counters */ 2216 mvneta_rxq_desc_num_update(pp, rxq, rx_done, rx_done); 2217 2218 return rx_done; 2219 } 2220 2221 static inline void 2222 mvneta_tso_put_hdr(struct sk_buff *skb, 2223 struct mvneta_port *pp, struct mvneta_tx_queue *txq) 2224 { 2225 struct mvneta_tx_desc *tx_desc; 2226 int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2227 2228 txq->tx_skb[txq->txq_put_index] = NULL; 2229 tx_desc = mvneta_txq_next_desc_get(txq); 2230 tx_desc->data_size = hdr_len; 2231 tx_desc->command = mvneta_skb_tx_csum(pp, skb); 2232 tx_desc->command |= MVNETA_TXD_F_DESC; 2233 tx_desc->buf_phys_addr = txq->tso_hdrs_phys + 2234 txq->txq_put_index * TSO_HEADER_SIZE; 2235 mvneta_txq_inc_put(txq); 2236 } 2237 2238 static inline int 2239 mvneta_tso_put_data(struct net_device *dev, struct mvneta_tx_queue *txq, 2240 struct sk_buff *skb, char *data, int size, 2241 bool last_tcp, bool is_last) 2242 { 2243 struct mvneta_tx_desc *tx_desc; 2244 2245 tx_desc = mvneta_txq_next_desc_get(txq); 2246 tx_desc->data_size = size; 2247 tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, data, 2248 size, DMA_TO_DEVICE); 2249 if (unlikely(dma_mapping_error(dev->dev.parent, 2250 tx_desc->buf_phys_addr))) { 2251 mvneta_txq_desc_put(txq); 2252 return -ENOMEM; 2253 } 2254 2255 tx_desc->command = 0; 2256 txq->tx_skb[txq->txq_put_index] = NULL; 2257 2258 if (last_tcp) { 2259 /* last descriptor in the TCP packet */ 2260 tx_desc->command = MVNETA_TXD_L_DESC; 2261 2262 /* last descriptor in SKB */ 2263 if (is_last) 2264 txq->tx_skb[txq->txq_put_index] = skb; 2265 } 2266 mvneta_txq_inc_put(txq); 2267 return 0; 2268 } 2269 2270 static int mvneta_tx_tso(struct sk_buff *skb, struct net_device *dev, 2271 struct mvneta_tx_queue *txq) 2272 { 2273 int total_len, data_left; 2274 int desc_count = 0; 2275 struct mvneta_port *pp = netdev_priv(dev); 2276 struct tso_t tso; 2277 int hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 2278 int i; 2279 2280 /* Count needed descriptors */ 2281 if ((txq->count + tso_count_descs(skb)) >= txq->size) 2282 return 0; 2283 2284 if (skb_headlen(skb) < (skb_transport_offset(skb) + tcp_hdrlen(skb))) { 2285 pr_info("*** Is this even possible???!?!?\n"); 2286 return 0; 2287 } 2288 2289 /* Initialize the TSO handler, and prepare the first payload */ 2290 tso_start(skb, &tso); 2291 2292 total_len = skb->len - hdr_len; 2293 while (total_len > 0) { 2294 char *hdr; 2295 2296 data_left = min_t(int, skb_shinfo(skb)->gso_size, total_len); 2297 total_len -= data_left; 2298 desc_count++; 2299 2300 /* prepare packet headers: MAC + IP + TCP */ 2301 hdr = txq->tso_hdrs + txq->txq_put_index * TSO_HEADER_SIZE; 2302 tso_build_hdr(skb, hdr, &tso, data_left, total_len == 0); 2303 2304 mvneta_tso_put_hdr(skb, pp, txq); 2305 2306 while (data_left > 0) { 2307 int size; 2308 desc_count++; 2309 2310 size = min_t(int, tso.size, data_left); 2311 2312 if (mvneta_tso_put_data(dev, txq, skb, 2313 tso.data, size, 2314 size == data_left, 2315 total_len == 0)) 2316 goto err_release; 2317 data_left -= size; 2318 2319 tso_build_data(skb, &tso, size); 2320 } 2321 } 2322 2323 return desc_count; 2324 2325 err_release: 2326 /* Release all used data descriptors; header descriptors must not 2327 * be DMA-unmapped. 2328 */ 2329 for (i = desc_count - 1; i >= 0; i--) { 2330 struct mvneta_tx_desc *tx_desc = txq->descs + i; 2331 if (!IS_TSO_HEADER(txq, tx_desc->buf_phys_addr)) 2332 dma_unmap_single(pp->dev->dev.parent, 2333 tx_desc->buf_phys_addr, 2334 tx_desc->data_size, 2335 DMA_TO_DEVICE); 2336 mvneta_txq_desc_put(txq); 2337 } 2338 return 0; 2339 } 2340 2341 /* Handle tx fragmentation processing */ 2342 static int mvneta_tx_frag_process(struct mvneta_port *pp, struct sk_buff *skb, 2343 struct mvneta_tx_queue *txq) 2344 { 2345 struct mvneta_tx_desc *tx_desc; 2346 int i, nr_frags = skb_shinfo(skb)->nr_frags; 2347 2348 for (i = 0; i < nr_frags; i++) { 2349 skb_frag_t *frag = &skb_shinfo(skb)->frags[i]; 2350 void *addr = page_address(frag->page.p) + frag->page_offset; 2351 2352 tx_desc = mvneta_txq_next_desc_get(txq); 2353 tx_desc->data_size = frag->size; 2354 2355 tx_desc->buf_phys_addr = 2356 dma_map_single(pp->dev->dev.parent, addr, 2357 tx_desc->data_size, DMA_TO_DEVICE); 2358 2359 if (dma_mapping_error(pp->dev->dev.parent, 2360 tx_desc->buf_phys_addr)) { 2361 mvneta_txq_desc_put(txq); 2362 goto error; 2363 } 2364 2365 if (i == nr_frags - 1) { 2366 /* Last descriptor */ 2367 tx_desc->command = MVNETA_TXD_L_DESC | MVNETA_TXD_Z_PAD; 2368 txq->tx_skb[txq->txq_put_index] = skb; 2369 } else { 2370 /* Descriptor in the middle: Not First, Not Last */ 2371 tx_desc->command = 0; 2372 txq->tx_skb[txq->txq_put_index] = NULL; 2373 } 2374 mvneta_txq_inc_put(txq); 2375 } 2376 2377 return 0; 2378 2379 error: 2380 /* Release all descriptors that were used to map fragments of 2381 * this packet, as well as the corresponding DMA mappings 2382 */ 2383 for (i = i - 1; i >= 0; i--) { 2384 tx_desc = txq->descs + i; 2385 dma_unmap_single(pp->dev->dev.parent, 2386 tx_desc->buf_phys_addr, 2387 tx_desc->data_size, 2388 DMA_TO_DEVICE); 2389 mvneta_txq_desc_put(txq); 2390 } 2391 2392 return -ENOMEM; 2393 } 2394 2395 /* Main tx processing */ 2396 static netdev_tx_t mvneta_tx(struct sk_buff *skb, struct net_device *dev) 2397 { 2398 struct mvneta_port *pp = netdev_priv(dev); 2399 u16 txq_id = skb_get_queue_mapping(skb); 2400 struct mvneta_tx_queue *txq = &pp->txqs[txq_id]; 2401 struct mvneta_tx_desc *tx_desc; 2402 int len = skb->len; 2403 int frags = 0; 2404 u32 tx_cmd; 2405 2406 if (!netif_running(dev)) 2407 goto out; 2408 2409 if (skb_is_gso(skb)) { 2410 frags = mvneta_tx_tso(skb, dev, txq); 2411 goto out; 2412 } 2413 2414 frags = skb_shinfo(skb)->nr_frags + 1; 2415 2416 /* Get a descriptor for the first part of the packet */ 2417 tx_desc = mvneta_txq_next_desc_get(txq); 2418 2419 tx_cmd = mvneta_skb_tx_csum(pp, skb); 2420 2421 tx_desc->data_size = skb_headlen(skb); 2422 2423 tx_desc->buf_phys_addr = dma_map_single(dev->dev.parent, skb->data, 2424 tx_desc->data_size, 2425 DMA_TO_DEVICE); 2426 if (unlikely(dma_mapping_error(dev->dev.parent, 2427 tx_desc->buf_phys_addr))) { 2428 mvneta_txq_desc_put(txq); 2429 frags = 0; 2430 goto out; 2431 } 2432 2433 if (frags == 1) { 2434 /* First and Last descriptor */ 2435 tx_cmd |= MVNETA_TXD_FLZ_DESC; 2436 tx_desc->command = tx_cmd; 2437 txq->tx_skb[txq->txq_put_index] = skb; 2438 mvneta_txq_inc_put(txq); 2439 } else { 2440 /* First but not Last */ 2441 tx_cmd |= MVNETA_TXD_F_DESC; 2442 txq->tx_skb[txq->txq_put_index] = NULL; 2443 mvneta_txq_inc_put(txq); 2444 tx_desc->command = tx_cmd; 2445 /* Continue with other skb fragments */ 2446 if (mvneta_tx_frag_process(pp, skb, txq)) { 2447 dma_unmap_single(dev->dev.parent, 2448 tx_desc->buf_phys_addr, 2449 tx_desc->data_size, 2450 DMA_TO_DEVICE); 2451 mvneta_txq_desc_put(txq); 2452 frags = 0; 2453 goto out; 2454 } 2455 } 2456 2457 out: 2458 if (frags > 0) { 2459 struct mvneta_pcpu_stats *stats = this_cpu_ptr(pp->stats); 2460 struct netdev_queue *nq = netdev_get_tx_queue(dev, txq_id); 2461 2462 netdev_tx_sent_queue(nq, len); 2463 2464 txq->count += frags; 2465 if (txq->count >= txq->tx_stop_threshold) 2466 netif_tx_stop_queue(nq); 2467 2468 if (!skb->xmit_more || netif_xmit_stopped(nq) || 2469 txq->pending + frags > MVNETA_TXQ_DEC_SENT_MASK) 2470 mvneta_txq_pend_desc_add(pp, txq, frags); 2471 else 2472 txq->pending += frags; 2473 2474 u64_stats_update_begin(&stats->syncp); 2475 stats->tx_packets++; 2476 stats->tx_bytes += len; 2477 u64_stats_update_end(&stats->syncp); 2478 } else { 2479 dev->stats.tx_dropped++; 2480 dev_kfree_skb_any(skb); 2481 } 2482 2483 return NETDEV_TX_OK; 2484 } 2485 2486 2487 /* Free tx resources, when resetting a port */ 2488 static void mvneta_txq_done_force(struct mvneta_port *pp, 2489 struct mvneta_tx_queue *txq) 2490 2491 { 2492 struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id); 2493 int tx_done = txq->count; 2494 2495 mvneta_txq_bufs_free(pp, txq, tx_done, nq); 2496 2497 /* reset txq */ 2498 txq->count = 0; 2499 txq->txq_put_index = 0; 2500 txq->txq_get_index = 0; 2501 } 2502 2503 /* Handle tx done - called in softirq context. The <cause_tx_done> argument 2504 * must be a valid cause according to MVNETA_TXQ_INTR_MASK_ALL. 2505 */ 2506 static void mvneta_tx_done_gbe(struct mvneta_port *pp, u32 cause_tx_done) 2507 { 2508 struct mvneta_tx_queue *txq; 2509 struct netdev_queue *nq; 2510 int cpu = smp_processor_id(); 2511 2512 while (cause_tx_done) { 2513 txq = mvneta_tx_done_policy(pp, cause_tx_done); 2514 2515 nq = netdev_get_tx_queue(pp->dev, txq->id); 2516 __netif_tx_lock(nq, cpu); 2517 2518 if (txq->count) 2519 mvneta_txq_done(pp, txq); 2520 2521 __netif_tx_unlock(nq); 2522 cause_tx_done &= ~((1 << txq->id)); 2523 } 2524 } 2525 2526 /* Compute crc8 of the specified address, using a unique algorithm , 2527 * according to hw spec, different than generic crc8 algorithm 2528 */ 2529 static int mvneta_addr_crc(unsigned char *addr) 2530 { 2531 int crc = 0; 2532 int i; 2533 2534 for (i = 0; i < ETH_ALEN; i++) { 2535 int j; 2536 2537 crc = (crc ^ addr[i]) << 8; 2538 for (j = 7; j >= 0; j--) { 2539 if (crc & (0x100 << j)) 2540 crc ^= 0x107 << j; 2541 } 2542 } 2543 2544 return crc; 2545 } 2546 2547 /* This method controls the net device special MAC multicast support. 2548 * The Special Multicast Table for MAC addresses supports MAC of the form 2549 * 0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF). 2550 * The MAC DA[7:0] bits are used as a pointer to the Special Multicast 2551 * Table entries in the DA-Filter table. This method set the Special 2552 * Multicast Table appropriate entry. 2553 */ 2554 static void mvneta_set_special_mcast_addr(struct mvneta_port *pp, 2555 unsigned char last_byte, 2556 int queue) 2557 { 2558 unsigned int smc_table_reg; 2559 unsigned int tbl_offset; 2560 unsigned int reg_offset; 2561 2562 /* Register offset from SMC table base */ 2563 tbl_offset = (last_byte / 4); 2564 /* Entry offset within the above reg */ 2565 reg_offset = last_byte % 4; 2566 2567 smc_table_reg = mvreg_read(pp, (MVNETA_DA_FILT_SPEC_MCAST 2568 + tbl_offset * 4)); 2569 2570 if (queue == -1) 2571 smc_table_reg &= ~(0xff << (8 * reg_offset)); 2572 else { 2573 smc_table_reg &= ~(0xff << (8 * reg_offset)); 2574 smc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset)); 2575 } 2576 2577 mvreg_write(pp, MVNETA_DA_FILT_SPEC_MCAST + tbl_offset * 4, 2578 smc_table_reg); 2579 } 2580 2581 /* This method controls the network device Other MAC multicast support. 2582 * The Other Multicast Table is used for multicast of another type. 2583 * A CRC-8 is used as an index to the Other Multicast Table entries 2584 * in the DA-Filter table. 2585 * The method gets the CRC-8 value from the calling routine and 2586 * sets the Other Multicast Table appropriate entry according to the 2587 * specified CRC-8 . 2588 */ 2589 static void mvneta_set_other_mcast_addr(struct mvneta_port *pp, 2590 unsigned char crc8, 2591 int queue) 2592 { 2593 unsigned int omc_table_reg; 2594 unsigned int tbl_offset; 2595 unsigned int reg_offset; 2596 2597 tbl_offset = (crc8 / 4) * 4; /* Register offset from OMC table base */ 2598 reg_offset = crc8 % 4; /* Entry offset within the above reg */ 2599 2600 omc_table_reg = mvreg_read(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset); 2601 2602 if (queue == -1) { 2603 /* Clear accepts frame bit at specified Other DA table entry */ 2604 omc_table_reg &= ~(0xff << (8 * reg_offset)); 2605 } else { 2606 omc_table_reg &= ~(0xff << (8 * reg_offset)); 2607 omc_table_reg |= ((0x01 | (queue << 1)) << (8 * reg_offset)); 2608 } 2609 2610 mvreg_write(pp, MVNETA_DA_FILT_OTH_MCAST + tbl_offset, omc_table_reg); 2611 } 2612 2613 /* The network device supports multicast using two tables: 2614 * 1) Special Multicast Table for MAC addresses of the form 2615 * 0x01-00-5E-00-00-XX (where XX is between 0x00 and 0xFF). 2616 * The MAC DA[7:0] bits are used as a pointer to the Special Multicast 2617 * Table entries in the DA-Filter table. 2618 * 2) Other Multicast Table for multicast of another type. A CRC-8 value 2619 * is used as an index to the Other Multicast Table entries in the 2620 * DA-Filter table. 2621 */ 2622 static int mvneta_mcast_addr_set(struct mvneta_port *pp, unsigned char *p_addr, 2623 int queue) 2624 { 2625 unsigned char crc_result = 0; 2626 2627 if (memcmp(p_addr, "\x01\x00\x5e\x00\x00", 5) == 0) { 2628 mvneta_set_special_mcast_addr(pp, p_addr[5], queue); 2629 return 0; 2630 } 2631 2632 crc_result = mvneta_addr_crc(p_addr); 2633 if (queue == -1) { 2634 if (pp->mcast_count[crc_result] == 0) { 2635 netdev_info(pp->dev, "No valid Mcast for crc8=0x%02x\n", 2636 crc_result); 2637 return -EINVAL; 2638 } 2639 2640 pp->mcast_count[crc_result]--; 2641 if (pp->mcast_count[crc_result] != 0) { 2642 netdev_info(pp->dev, 2643 "After delete there are %d valid Mcast for crc8=0x%02x\n", 2644 pp->mcast_count[crc_result], crc_result); 2645 return -EINVAL; 2646 } 2647 } else 2648 pp->mcast_count[crc_result]++; 2649 2650 mvneta_set_other_mcast_addr(pp, crc_result, queue); 2651 2652 return 0; 2653 } 2654 2655 /* Configure Fitering mode of Ethernet port */ 2656 static void mvneta_rx_unicast_promisc_set(struct mvneta_port *pp, 2657 int is_promisc) 2658 { 2659 u32 port_cfg_reg, val; 2660 2661 port_cfg_reg = mvreg_read(pp, MVNETA_PORT_CONFIG); 2662 2663 val = mvreg_read(pp, MVNETA_TYPE_PRIO); 2664 2665 /* Set / Clear UPM bit in port configuration register */ 2666 if (is_promisc) { 2667 /* Accept all Unicast addresses */ 2668 port_cfg_reg |= MVNETA_UNI_PROMISC_MODE; 2669 val |= MVNETA_FORCE_UNI; 2670 mvreg_write(pp, MVNETA_MAC_ADDR_LOW, 0xffff); 2671 mvreg_write(pp, MVNETA_MAC_ADDR_HIGH, 0xffffffff); 2672 } else { 2673 /* Reject all Unicast addresses */ 2674 port_cfg_reg &= ~MVNETA_UNI_PROMISC_MODE; 2675 val &= ~MVNETA_FORCE_UNI; 2676 } 2677 2678 mvreg_write(pp, MVNETA_PORT_CONFIG, port_cfg_reg); 2679 mvreg_write(pp, MVNETA_TYPE_PRIO, val); 2680 } 2681 2682 /* register unicast and multicast addresses */ 2683 static void mvneta_set_rx_mode(struct net_device *dev) 2684 { 2685 struct mvneta_port *pp = netdev_priv(dev); 2686 struct netdev_hw_addr *ha; 2687 2688 if (dev->flags & IFF_PROMISC) { 2689 /* Accept all: Multicast + Unicast */ 2690 mvneta_rx_unicast_promisc_set(pp, 1); 2691 mvneta_set_ucast_table(pp, pp->rxq_def); 2692 mvneta_set_special_mcast_table(pp, pp->rxq_def); 2693 mvneta_set_other_mcast_table(pp, pp->rxq_def); 2694 } else { 2695 /* Accept single Unicast */ 2696 mvneta_rx_unicast_promisc_set(pp, 0); 2697 mvneta_set_ucast_table(pp, -1); 2698 mvneta_mac_addr_set(pp, dev->dev_addr, pp->rxq_def); 2699 2700 if (dev->flags & IFF_ALLMULTI) { 2701 /* Accept all multicast */ 2702 mvneta_set_special_mcast_table(pp, pp->rxq_def); 2703 mvneta_set_other_mcast_table(pp, pp->rxq_def); 2704 } else { 2705 /* Accept only initialized multicast */ 2706 mvneta_set_special_mcast_table(pp, -1); 2707 mvneta_set_other_mcast_table(pp, -1); 2708 2709 if (!netdev_mc_empty(dev)) { 2710 netdev_for_each_mc_addr(ha, dev) { 2711 mvneta_mcast_addr_set(pp, ha->addr, 2712 pp->rxq_def); 2713 } 2714 } 2715 } 2716 } 2717 } 2718 2719 /* Interrupt handling - the callback for request_irq() */ 2720 static irqreturn_t mvneta_isr(int irq, void *dev_id) 2721 { 2722 struct mvneta_port *pp = (struct mvneta_port *)dev_id; 2723 2724 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 0); 2725 napi_schedule(&pp->napi); 2726 2727 return IRQ_HANDLED; 2728 } 2729 2730 /* Interrupt handling - the callback for request_percpu_irq() */ 2731 static irqreturn_t mvneta_percpu_isr(int irq, void *dev_id) 2732 { 2733 struct mvneta_pcpu_port *port = (struct mvneta_pcpu_port *)dev_id; 2734 2735 disable_percpu_irq(port->pp->dev->irq); 2736 napi_schedule(&port->napi); 2737 2738 return IRQ_HANDLED; 2739 } 2740 2741 static void mvneta_link_change(struct mvneta_port *pp) 2742 { 2743 u32 gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS); 2744 2745 phylink_mac_change(pp->phylink, !!(gmac_stat & MVNETA_GMAC_LINK_UP)); 2746 } 2747 2748 /* NAPI handler 2749 * Bits 0 - 7 of the causeRxTx register indicate that are transmitted 2750 * packets on the corresponding TXQ (Bit 0 is for TX queue 1). 2751 * Bits 8 -15 of the cause Rx Tx register indicate that are received 2752 * packets on the corresponding RXQ (Bit 8 is for RX queue 0). 2753 * Each CPU has its own causeRxTx register 2754 */ 2755 static int mvneta_poll(struct napi_struct *napi, int budget) 2756 { 2757 int rx_done = 0; 2758 u32 cause_rx_tx; 2759 int rx_queue; 2760 struct mvneta_port *pp = netdev_priv(napi->dev); 2761 struct mvneta_pcpu_port *port = this_cpu_ptr(pp->ports); 2762 2763 if (!netif_running(pp->dev)) { 2764 napi_complete(napi); 2765 return rx_done; 2766 } 2767 2768 /* Read cause register */ 2769 cause_rx_tx = mvreg_read(pp, MVNETA_INTR_NEW_CAUSE); 2770 if (cause_rx_tx & MVNETA_MISCINTR_INTR_MASK) { 2771 u32 cause_misc = mvreg_read(pp, MVNETA_INTR_MISC_CAUSE); 2772 2773 mvreg_write(pp, MVNETA_INTR_MISC_CAUSE, 0); 2774 2775 if (cause_misc & (MVNETA_CAUSE_PHY_STATUS_CHANGE | 2776 MVNETA_CAUSE_LINK_CHANGE)) 2777 mvneta_link_change(pp); 2778 } 2779 2780 /* Release Tx descriptors */ 2781 if (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL) { 2782 mvneta_tx_done_gbe(pp, (cause_rx_tx & MVNETA_TX_INTR_MASK_ALL)); 2783 cause_rx_tx &= ~MVNETA_TX_INTR_MASK_ALL; 2784 } 2785 2786 /* For the case where the last mvneta_poll did not process all 2787 * RX packets 2788 */ 2789 rx_queue = fls(((cause_rx_tx >> 8) & 0xff)); 2790 2791 cause_rx_tx |= pp->neta_armada3700 ? pp->cause_rx_tx : 2792 port->cause_rx_tx; 2793 2794 if (rx_queue) { 2795 rx_queue = rx_queue - 1; 2796 if (pp->bm_priv) 2797 rx_done = mvneta_rx_hwbm(napi, pp, budget, 2798 &pp->rxqs[rx_queue]); 2799 else 2800 rx_done = mvneta_rx_swbm(napi, pp, budget, 2801 &pp->rxqs[rx_queue]); 2802 } 2803 2804 if (rx_done < budget) { 2805 cause_rx_tx = 0; 2806 napi_complete_done(napi, rx_done); 2807 2808 if (pp->neta_armada3700) { 2809 unsigned long flags; 2810 2811 local_irq_save(flags); 2812 mvreg_write(pp, MVNETA_INTR_NEW_MASK, 2813 MVNETA_RX_INTR_MASK(rxq_number) | 2814 MVNETA_TX_INTR_MASK(txq_number) | 2815 MVNETA_MISCINTR_INTR_MASK); 2816 local_irq_restore(flags); 2817 } else { 2818 enable_percpu_irq(pp->dev->irq, 0); 2819 } 2820 } 2821 2822 if (pp->neta_armada3700) 2823 pp->cause_rx_tx = cause_rx_tx; 2824 else 2825 port->cause_rx_tx = cause_rx_tx; 2826 2827 return rx_done; 2828 } 2829 2830 /* Handle rxq fill: allocates rxq skbs; called when initializing a port */ 2831 static int mvneta_rxq_fill(struct mvneta_port *pp, struct mvneta_rx_queue *rxq, 2832 int num) 2833 { 2834 int i; 2835 2836 for (i = 0; i < num; i++) { 2837 memset(rxq->descs + i, 0, sizeof(struct mvneta_rx_desc)); 2838 if (mvneta_rx_refill(pp, rxq->descs + i, rxq, 2839 GFP_KERNEL) != 0) { 2840 netdev_err(pp->dev, 2841 "%s:rxq %d, %d of %d buffs filled\n", 2842 __func__, rxq->id, i, num); 2843 break; 2844 } 2845 } 2846 2847 /* Add this number of RX descriptors as non occupied (ready to 2848 * get packets) 2849 */ 2850 mvneta_rxq_non_occup_desc_add(pp, rxq, i); 2851 2852 return i; 2853 } 2854 2855 /* Free all packets pending transmit from all TXQs and reset TX port */ 2856 static void mvneta_tx_reset(struct mvneta_port *pp) 2857 { 2858 int queue; 2859 2860 /* free the skb's in the tx ring */ 2861 for (queue = 0; queue < txq_number; queue++) 2862 mvneta_txq_done_force(pp, &pp->txqs[queue]); 2863 2864 mvreg_write(pp, MVNETA_PORT_TX_RESET, MVNETA_PORT_TX_DMA_RESET); 2865 mvreg_write(pp, MVNETA_PORT_TX_RESET, 0); 2866 } 2867 2868 static void mvneta_rx_reset(struct mvneta_port *pp) 2869 { 2870 mvreg_write(pp, MVNETA_PORT_RX_RESET, MVNETA_PORT_RX_DMA_RESET); 2871 mvreg_write(pp, MVNETA_PORT_RX_RESET, 0); 2872 } 2873 2874 /* Rx/Tx queue initialization/cleanup methods */ 2875 2876 static int mvneta_rxq_sw_init(struct mvneta_port *pp, 2877 struct mvneta_rx_queue *rxq) 2878 { 2879 rxq->size = pp->rx_ring_size; 2880 2881 /* Allocate memory for RX descriptors */ 2882 rxq->descs = dma_alloc_coherent(pp->dev->dev.parent, 2883 rxq->size * MVNETA_DESC_ALIGNED_SIZE, 2884 &rxq->descs_phys, GFP_KERNEL); 2885 if (!rxq->descs) 2886 return -ENOMEM; 2887 2888 rxq->last_desc = rxq->size - 1; 2889 2890 return 0; 2891 } 2892 2893 static void mvneta_rxq_hw_init(struct mvneta_port *pp, 2894 struct mvneta_rx_queue *rxq) 2895 { 2896 /* Set Rx descriptors queue starting address */ 2897 mvreg_write(pp, MVNETA_RXQ_BASE_ADDR_REG(rxq->id), rxq->descs_phys); 2898 mvreg_write(pp, MVNETA_RXQ_SIZE_REG(rxq->id), rxq->size); 2899 2900 /* Set coalescing pkts and time */ 2901 mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal); 2902 mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal); 2903 2904 if (!pp->bm_priv) { 2905 /* Set Offset */ 2906 mvneta_rxq_offset_set(pp, rxq, 0); 2907 mvneta_rxq_buf_size_set(pp, rxq, PAGE_SIZE < SZ_64K ? 2908 PAGE_SIZE : 2909 MVNETA_RX_BUF_SIZE(pp->pkt_size)); 2910 mvneta_rxq_bm_disable(pp, rxq); 2911 mvneta_rxq_fill(pp, rxq, rxq->size); 2912 } else { 2913 /* Set Offset */ 2914 mvneta_rxq_offset_set(pp, rxq, 2915 NET_SKB_PAD - pp->rx_offset_correction); 2916 2917 mvneta_rxq_bm_enable(pp, rxq); 2918 /* Fill RXQ with buffers from RX pool */ 2919 mvneta_rxq_long_pool_set(pp, rxq); 2920 mvneta_rxq_short_pool_set(pp, rxq); 2921 mvneta_rxq_non_occup_desc_add(pp, rxq, rxq->size); 2922 } 2923 } 2924 2925 /* Create a specified RX queue */ 2926 static int mvneta_rxq_init(struct mvneta_port *pp, 2927 struct mvneta_rx_queue *rxq) 2928 2929 { 2930 int ret; 2931 2932 ret = mvneta_rxq_sw_init(pp, rxq); 2933 if (ret < 0) 2934 return ret; 2935 2936 mvneta_rxq_hw_init(pp, rxq); 2937 2938 return 0; 2939 } 2940 2941 /* Cleanup Rx queue */ 2942 static void mvneta_rxq_deinit(struct mvneta_port *pp, 2943 struct mvneta_rx_queue *rxq) 2944 { 2945 mvneta_rxq_drop_pkts(pp, rxq); 2946 2947 if (rxq->skb) 2948 dev_kfree_skb_any(rxq->skb); 2949 2950 if (rxq->descs) 2951 dma_free_coherent(pp->dev->dev.parent, 2952 rxq->size * MVNETA_DESC_ALIGNED_SIZE, 2953 rxq->descs, 2954 rxq->descs_phys); 2955 2956 rxq->descs = NULL; 2957 rxq->last_desc = 0; 2958 rxq->next_desc_to_proc = 0; 2959 rxq->descs_phys = 0; 2960 rxq->first_to_refill = 0; 2961 rxq->refill_num = 0; 2962 rxq->skb = NULL; 2963 rxq->left_size = 0; 2964 } 2965 2966 static int mvneta_txq_sw_init(struct mvneta_port *pp, 2967 struct mvneta_tx_queue *txq) 2968 { 2969 int cpu; 2970 2971 txq->size = pp->tx_ring_size; 2972 2973 /* A queue must always have room for at least one skb. 2974 * Therefore, stop the queue when the free entries reaches 2975 * the maximum number of descriptors per skb. 2976 */ 2977 txq->tx_stop_threshold = txq->size - MVNETA_MAX_SKB_DESCS; 2978 txq->tx_wake_threshold = txq->tx_stop_threshold / 2; 2979 2980 /* Allocate memory for TX descriptors */ 2981 txq->descs = dma_alloc_coherent(pp->dev->dev.parent, 2982 txq->size * MVNETA_DESC_ALIGNED_SIZE, 2983 &txq->descs_phys, GFP_KERNEL); 2984 if (!txq->descs) 2985 return -ENOMEM; 2986 2987 txq->last_desc = txq->size - 1; 2988 2989 txq->tx_skb = kmalloc_array(txq->size, sizeof(*txq->tx_skb), 2990 GFP_KERNEL); 2991 if (!txq->tx_skb) { 2992 dma_free_coherent(pp->dev->dev.parent, 2993 txq->size * MVNETA_DESC_ALIGNED_SIZE, 2994 txq->descs, txq->descs_phys); 2995 return -ENOMEM; 2996 } 2997 2998 /* Allocate DMA buffers for TSO MAC/IP/TCP headers */ 2999 txq->tso_hdrs = dma_alloc_coherent(pp->dev->dev.parent, 3000 txq->size * TSO_HEADER_SIZE, 3001 &txq->tso_hdrs_phys, GFP_KERNEL); 3002 if (!txq->tso_hdrs) { 3003 kfree(txq->tx_skb); 3004 dma_free_coherent(pp->dev->dev.parent, 3005 txq->size * MVNETA_DESC_ALIGNED_SIZE, 3006 txq->descs, txq->descs_phys); 3007 return -ENOMEM; 3008 } 3009 3010 /* Setup XPS mapping */ 3011 if (txq_number > 1) 3012 cpu = txq->id % num_present_cpus(); 3013 else 3014 cpu = pp->rxq_def % num_present_cpus(); 3015 cpumask_set_cpu(cpu, &txq->affinity_mask); 3016 netif_set_xps_queue(pp->dev, &txq->affinity_mask, txq->id); 3017 3018 return 0; 3019 } 3020 3021 static void mvneta_txq_hw_init(struct mvneta_port *pp, 3022 struct mvneta_tx_queue *txq) 3023 { 3024 /* Set maximum bandwidth for enabled TXQs */ 3025 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0x03ffffff); 3026 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0x3fffffff); 3027 3028 /* Set Tx descriptors queue starting address */ 3029 mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), txq->descs_phys); 3030 mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), txq->size); 3031 3032 mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal); 3033 } 3034 3035 /* Create and initialize a tx queue */ 3036 static int mvneta_txq_init(struct mvneta_port *pp, 3037 struct mvneta_tx_queue *txq) 3038 { 3039 int ret; 3040 3041 ret = mvneta_txq_sw_init(pp, txq); 3042 if (ret < 0) 3043 return ret; 3044 3045 mvneta_txq_hw_init(pp, txq); 3046 3047 return 0; 3048 } 3049 3050 /* Free allocated resources when mvneta_txq_init() fails to allocate memory*/ 3051 static void mvneta_txq_sw_deinit(struct mvneta_port *pp, 3052 struct mvneta_tx_queue *txq) 3053 { 3054 struct netdev_queue *nq = netdev_get_tx_queue(pp->dev, txq->id); 3055 3056 kfree(txq->tx_skb); 3057 3058 if (txq->tso_hdrs) 3059 dma_free_coherent(pp->dev->dev.parent, 3060 txq->size * TSO_HEADER_SIZE, 3061 txq->tso_hdrs, txq->tso_hdrs_phys); 3062 if (txq->descs) 3063 dma_free_coherent(pp->dev->dev.parent, 3064 txq->size * MVNETA_DESC_ALIGNED_SIZE, 3065 txq->descs, txq->descs_phys); 3066 3067 netdev_tx_reset_queue(nq); 3068 3069 txq->descs = NULL; 3070 txq->last_desc = 0; 3071 txq->next_desc_to_proc = 0; 3072 txq->descs_phys = 0; 3073 } 3074 3075 static void mvneta_txq_hw_deinit(struct mvneta_port *pp, 3076 struct mvneta_tx_queue *txq) 3077 { 3078 /* Set minimum bandwidth for disabled TXQs */ 3079 mvreg_write(pp, MVETH_TXQ_TOKEN_CFG_REG(txq->id), 0); 3080 mvreg_write(pp, MVETH_TXQ_TOKEN_COUNT_REG(txq->id), 0); 3081 3082 /* Set Tx descriptors queue starting address and size */ 3083 mvreg_write(pp, MVNETA_TXQ_BASE_ADDR_REG(txq->id), 0); 3084 mvreg_write(pp, MVNETA_TXQ_SIZE_REG(txq->id), 0); 3085 } 3086 3087 static void mvneta_txq_deinit(struct mvneta_port *pp, 3088 struct mvneta_tx_queue *txq) 3089 { 3090 mvneta_txq_sw_deinit(pp, txq); 3091 mvneta_txq_hw_deinit(pp, txq); 3092 } 3093 3094 /* Cleanup all Tx queues */ 3095 static void mvneta_cleanup_txqs(struct mvneta_port *pp) 3096 { 3097 int queue; 3098 3099 for (queue = 0; queue < txq_number; queue++) 3100 mvneta_txq_deinit(pp, &pp->txqs[queue]); 3101 } 3102 3103 /* Cleanup all Rx queues */ 3104 static void mvneta_cleanup_rxqs(struct mvneta_port *pp) 3105 { 3106 int queue; 3107 3108 for (queue = 0; queue < rxq_number; queue++) 3109 mvneta_rxq_deinit(pp, &pp->rxqs[queue]); 3110 } 3111 3112 3113 /* Init all Rx queues */ 3114 static int mvneta_setup_rxqs(struct mvneta_port *pp) 3115 { 3116 int queue; 3117 3118 for (queue = 0; queue < rxq_number; queue++) { 3119 int err = mvneta_rxq_init(pp, &pp->rxqs[queue]); 3120 3121 if (err) { 3122 netdev_err(pp->dev, "%s: can't create rxq=%d\n", 3123 __func__, queue); 3124 mvneta_cleanup_rxqs(pp); 3125 return err; 3126 } 3127 } 3128 3129 return 0; 3130 } 3131 3132 /* Init all tx queues */ 3133 static int mvneta_setup_txqs(struct mvneta_port *pp) 3134 { 3135 int queue; 3136 3137 for (queue = 0; queue < txq_number; queue++) { 3138 int err = mvneta_txq_init(pp, &pp->txqs[queue]); 3139 if (err) { 3140 netdev_err(pp->dev, "%s: can't create txq=%d\n", 3141 __func__, queue); 3142 mvneta_cleanup_txqs(pp); 3143 return err; 3144 } 3145 } 3146 3147 return 0; 3148 } 3149 3150 static void mvneta_start_dev(struct mvneta_port *pp) 3151 { 3152 int cpu; 3153 3154 mvneta_max_rx_size_set(pp, pp->pkt_size); 3155 mvneta_txq_max_tx_size_set(pp, pp->pkt_size); 3156 3157 /* start the Rx/Tx activity */ 3158 mvneta_port_enable(pp); 3159 3160 if (!pp->neta_armada3700) { 3161 /* Enable polling on the port */ 3162 for_each_online_cpu(cpu) { 3163 struct mvneta_pcpu_port *port = 3164 per_cpu_ptr(pp->ports, cpu); 3165 3166 napi_enable(&port->napi); 3167 } 3168 } else { 3169 napi_enable(&pp->napi); 3170 } 3171 3172 /* Unmask interrupts. It has to be done from each CPU */ 3173 on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true); 3174 3175 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 3176 MVNETA_CAUSE_PHY_STATUS_CHANGE | 3177 MVNETA_CAUSE_LINK_CHANGE); 3178 3179 phylink_start(pp->phylink); 3180 netif_tx_start_all_queues(pp->dev); 3181 } 3182 3183 static void mvneta_stop_dev(struct mvneta_port *pp) 3184 { 3185 unsigned int cpu; 3186 3187 phylink_stop(pp->phylink); 3188 3189 if (!pp->neta_armada3700) { 3190 for_each_online_cpu(cpu) { 3191 struct mvneta_pcpu_port *port = 3192 per_cpu_ptr(pp->ports, cpu); 3193 3194 napi_disable(&port->napi); 3195 } 3196 } else { 3197 napi_disable(&pp->napi); 3198 } 3199 3200 netif_carrier_off(pp->dev); 3201 3202 mvneta_port_down(pp); 3203 netif_tx_stop_all_queues(pp->dev); 3204 3205 /* Stop the port activity */ 3206 mvneta_port_disable(pp); 3207 3208 /* Clear all ethernet port interrupts */ 3209 on_each_cpu(mvneta_percpu_clear_intr_cause, pp, true); 3210 3211 /* Mask all ethernet port interrupts */ 3212 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3213 3214 mvneta_tx_reset(pp); 3215 mvneta_rx_reset(pp); 3216 } 3217 3218 static void mvneta_percpu_enable(void *arg) 3219 { 3220 struct mvneta_port *pp = arg; 3221 3222 enable_percpu_irq(pp->dev->irq, IRQ_TYPE_NONE); 3223 } 3224 3225 static void mvneta_percpu_disable(void *arg) 3226 { 3227 struct mvneta_port *pp = arg; 3228 3229 disable_percpu_irq(pp->dev->irq); 3230 } 3231 3232 /* Change the device mtu */ 3233 static int mvneta_change_mtu(struct net_device *dev, int mtu) 3234 { 3235 struct mvneta_port *pp = netdev_priv(dev); 3236 int ret; 3237 3238 if (!IS_ALIGNED(MVNETA_RX_PKT_SIZE(mtu), 8)) { 3239 netdev_info(dev, "Illegal MTU value %d, rounding to %d\n", 3240 mtu, ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8)); 3241 mtu = ALIGN(MVNETA_RX_PKT_SIZE(mtu), 8); 3242 } 3243 3244 dev->mtu = mtu; 3245 3246 if (!netif_running(dev)) { 3247 if (pp->bm_priv) 3248 mvneta_bm_update_mtu(pp, mtu); 3249 3250 netdev_update_features(dev); 3251 return 0; 3252 } 3253 3254 /* The interface is running, so we have to force a 3255 * reallocation of the queues 3256 */ 3257 mvneta_stop_dev(pp); 3258 on_each_cpu(mvneta_percpu_disable, pp, true); 3259 3260 mvneta_cleanup_txqs(pp); 3261 mvneta_cleanup_rxqs(pp); 3262 3263 if (pp->bm_priv) 3264 mvneta_bm_update_mtu(pp, mtu); 3265 3266 pp->pkt_size = MVNETA_RX_PKT_SIZE(dev->mtu); 3267 3268 ret = mvneta_setup_rxqs(pp); 3269 if (ret) { 3270 netdev_err(dev, "unable to setup rxqs after MTU change\n"); 3271 return ret; 3272 } 3273 3274 ret = mvneta_setup_txqs(pp); 3275 if (ret) { 3276 netdev_err(dev, "unable to setup txqs after MTU change\n"); 3277 return ret; 3278 } 3279 3280 on_each_cpu(mvneta_percpu_enable, pp, true); 3281 mvneta_start_dev(pp); 3282 3283 netdev_update_features(dev); 3284 3285 return 0; 3286 } 3287 3288 static netdev_features_t mvneta_fix_features(struct net_device *dev, 3289 netdev_features_t features) 3290 { 3291 struct mvneta_port *pp = netdev_priv(dev); 3292 3293 if (pp->tx_csum_limit && dev->mtu > pp->tx_csum_limit) { 3294 features &= ~(NETIF_F_IP_CSUM | NETIF_F_TSO); 3295 netdev_info(dev, 3296 "Disable IP checksum for MTU greater than %dB\n", 3297 pp->tx_csum_limit); 3298 } 3299 3300 return features; 3301 } 3302 3303 /* Get mac address */ 3304 static void mvneta_get_mac_addr(struct mvneta_port *pp, unsigned char *addr) 3305 { 3306 u32 mac_addr_l, mac_addr_h; 3307 3308 mac_addr_l = mvreg_read(pp, MVNETA_MAC_ADDR_LOW); 3309 mac_addr_h = mvreg_read(pp, MVNETA_MAC_ADDR_HIGH); 3310 addr[0] = (mac_addr_h >> 24) & 0xFF; 3311 addr[1] = (mac_addr_h >> 16) & 0xFF; 3312 addr[2] = (mac_addr_h >> 8) & 0xFF; 3313 addr[3] = mac_addr_h & 0xFF; 3314 addr[4] = (mac_addr_l >> 8) & 0xFF; 3315 addr[5] = mac_addr_l & 0xFF; 3316 } 3317 3318 /* Handle setting mac address */ 3319 static int mvneta_set_mac_addr(struct net_device *dev, void *addr) 3320 { 3321 struct mvneta_port *pp = netdev_priv(dev); 3322 struct sockaddr *sockaddr = addr; 3323 int ret; 3324 3325 ret = eth_prepare_mac_addr_change(dev, addr); 3326 if (ret < 0) 3327 return ret; 3328 /* Remove previous address table entry */ 3329 mvneta_mac_addr_set(pp, dev->dev_addr, -1); 3330 3331 /* Set new addr in hw */ 3332 mvneta_mac_addr_set(pp, sockaddr->sa_data, pp->rxq_def); 3333 3334 eth_commit_mac_addr_change(dev, addr); 3335 return 0; 3336 } 3337 3338 static void mvneta_validate(struct net_device *ndev, unsigned long *supported, 3339 struct phylink_link_state *state) 3340 { 3341 __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, }; 3342 3343 /* We only support QSGMII, SGMII, 802.3z and RGMII modes */ 3344 if (state->interface != PHY_INTERFACE_MODE_NA && 3345 state->interface != PHY_INTERFACE_MODE_QSGMII && 3346 state->interface != PHY_INTERFACE_MODE_SGMII && 3347 !phy_interface_mode_is_8023z(state->interface) && 3348 !phy_interface_mode_is_rgmii(state->interface)) { 3349 bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS); 3350 return; 3351 } 3352 3353 /* Allow all the expected bits */ 3354 phylink_set(mask, Autoneg); 3355 phylink_set_port_modes(mask); 3356 3357 /* Asymmetric pause is unsupported */ 3358 phylink_set(mask, Pause); 3359 3360 /* Half-duplex at speeds higher than 100Mbit is unsupported */ 3361 phylink_set(mask, 1000baseT_Full); 3362 phylink_set(mask, 1000baseX_Full); 3363 3364 if (!phy_interface_mode_is_8023z(state->interface)) { 3365 /* 10M and 100M are only supported in non-802.3z mode */ 3366 phylink_set(mask, 10baseT_Half); 3367 phylink_set(mask, 10baseT_Full); 3368 phylink_set(mask, 100baseT_Half); 3369 phylink_set(mask, 100baseT_Full); 3370 } 3371 3372 bitmap_and(supported, supported, mask, 3373 __ETHTOOL_LINK_MODE_MASK_NBITS); 3374 bitmap_and(state->advertising, state->advertising, mask, 3375 __ETHTOOL_LINK_MODE_MASK_NBITS); 3376 } 3377 3378 static int mvneta_mac_link_state(struct net_device *ndev, 3379 struct phylink_link_state *state) 3380 { 3381 struct mvneta_port *pp = netdev_priv(ndev); 3382 u32 gmac_stat; 3383 3384 gmac_stat = mvreg_read(pp, MVNETA_GMAC_STATUS); 3385 3386 if (gmac_stat & MVNETA_GMAC_SPEED_1000) 3387 state->speed = SPEED_1000; 3388 else if (gmac_stat & MVNETA_GMAC_SPEED_100) 3389 state->speed = SPEED_100; 3390 else 3391 state->speed = SPEED_10; 3392 3393 state->an_complete = !!(gmac_stat & MVNETA_GMAC_AN_COMPLETE); 3394 state->link = !!(gmac_stat & MVNETA_GMAC_LINK_UP); 3395 state->duplex = !!(gmac_stat & MVNETA_GMAC_FULL_DUPLEX); 3396 3397 state->pause = 0; 3398 if (gmac_stat & MVNETA_GMAC_RX_FLOW_CTRL_ENABLE) 3399 state->pause |= MLO_PAUSE_RX; 3400 if (gmac_stat & MVNETA_GMAC_TX_FLOW_CTRL_ENABLE) 3401 state->pause |= MLO_PAUSE_TX; 3402 3403 return 1; 3404 } 3405 3406 static void mvneta_mac_an_restart(struct net_device *ndev) 3407 { 3408 struct mvneta_port *pp = netdev_priv(ndev); 3409 u32 gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3410 3411 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, 3412 gmac_an | MVNETA_GMAC_INBAND_RESTART_AN); 3413 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, 3414 gmac_an & ~MVNETA_GMAC_INBAND_RESTART_AN); 3415 } 3416 3417 static void mvneta_mac_config(struct net_device *ndev, unsigned int mode, 3418 const struct phylink_link_state *state) 3419 { 3420 struct mvneta_port *pp = netdev_priv(ndev); 3421 u32 new_ctrl0, gmac_ctrl0 = mvreg_read(pp, MVNETA_GMAC_CTRL_0); 3422 u32 new_ctrl2, gmac_ctrl2 = mvreg_read(pp, MVNETA_GMAC_CTRL_2); 3423 u32 new_ctrl4, gmac_ctrl4 = mvreg_read(pp, MVNETA_GMAC_CTRL_4); 3424 u32 new_clk, gmac_clk = mvreg_read(pp, MVNETA_GMAC_CLOCK_DIVIDER); 3425 u32 new_an, gmac_an = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3426 3427 new_ctrl0 = gmac_ctrl0 & ~MVNETA_GMAC0_PORT_1000BASE_X; 3428 new_ctrl2 = gmac_ctrl2 & ~(MVNETA_GMAC2_INBAND_AN_ENABLE | 3429 MVNETA_GMAC2_PORT_RESET); 3430 new_ctrl4 = gmac_ctrl4 & ~(MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE); 3431 new_clk = gmac_clk & ~MVNETA_GMAC_1MS_CLOCK_ENABLE; 3432 new_an = gmac_an & ~(MVNETA_GMAC_INBAND_AN_ENABLE | 3433 MVNETA_GMAC_INBAND_RESTART_AN | 3434 MVNETA_GMAC_CONFIG_MII_SPEED | 3435 MVNETA_GMAC_CONFIG_GMII_SPEED | 3436 MVNETA_GMAC_AN_SPEED_EN | 3437 MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL | 3438 MVNETA_GMAC_CONFIG_FLOW_CTRL | 3439 MVNETA_GMAC_AN_FLOW_CTRL_EN | 3440 MVNETA_GMAC_CONFIG_FULL_DUPLEX | 3441 MVNETA_GMAC_AN_DUPLEX_EN); 3442 3443 /* Even though it might look weird, when we're configured in 3444 * SGMII or QSGMII mode, the RGMII bit needs to be set. 3445 */ 3446 new_ctrl2 |= MVNETA_GMAC2_PORT_RGMII; 3447 3448 if (state->interface == PHY_INTERFACE_MODE_QSGMII || 3449 state->interface == PHY_INTERFACE_MODE_SGMII || 3450 phy_interface_mode_is_8023z(state->interface)) 3451 new_ctrl2 |= MVNETA_GMAC2_PCS_ENABLE; 3452 3453 if (phylink_test(state->advertising, Pause)) 3454 new_an |= MVNETA_GMAC_ADVERT_SYM_FLOW_CTRL; 3455 if (state->pause & MLO_PAUSE_TXRX_MASK) 3456 new_an |= MVNETA_GMAC_CONFIG_FLOW_CTRL; 3457 3458 if (!phylink_autoneg_inband(mode)) { 3459 /* Phy or fixed speed */ 3460 if (state->duplex) 3461 new_an |= MVNETA_GMAC_CONFIG_FULL_DUPLEX; 3462 3463 if (state->speed == SPEED_1000 || state->speed == SPEED_2500) 3464 new_an |= MVNETA_GMAC_CONFIG_GMII_SPEED; 3465 else if (state->speed == SPEED_100) 3466 new_an |= MVNETA_GMAC_CONFIG_MII_SPEED; 3467 } else if (state->interface == PHY_INTERFACE_MODE_SGMII) { 3468 /* SGMII mode receives the state from the PHY */ 3469 new_ctrl2 |= MVNETA_GMAC2_INBAND_AN_ENABLE; 3470 new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE; 3471 new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN | 3472 MVNETA_GMAC_FORCE_LINK_PASS)) | 3473 MVNETA_GMAC_INBAND_AN_ENABLE | 3474 MVNETA_GMAC_AN_SPEED_EN | 3475 MVNETA_GMAC_AN_DUPLEX_EN; 3476 } else { 3477 /* 802.3z negotiation - only 1000base-X */ 3478 new_ctrl0 |= MVNETA_GMAC0_PORT_1000BASE_X; 3479 new_clk |= MVNETA_GMAC_1MS_CLOCK_ENABLE; 3480 new_an = (new_an & ~(MVNETA_GMAC_FORCE_LINK_DOWN | 3481 MVNETA_GMAC_FORCE_LINK_PASS)) | 3482 MVNETA_GMAC_INBAND_AN_ENABLE | 3483 MVNETA_GMAC_CONFIG_GMII_SPEED | 3484 /* The MAC only supports FD mode */ 3485 MVNETA_GMAC_CONFIG_FULL_DUPLEX; 3486 3487 if (state->pause & MLO_PAUSE_AN && state->an_enabled) 3488 new_an |= MVNETA_GMAC_AN_FLOW_CTRL_EN; 3489 } 3490 3491 /* Armada 370 documentation says we can only change the port mode 3492 * and in-band enable when the link is down, so force it down 3493 * while making these changes. We also do this for GMAC_CTRL2 */ 3494 if ((new_ctrl0 ^ gmac_ctrl0) & MVNETA_GMAC0_PORT_1000BASE_X || 3495 (new_ctrl2 ^ gmac_ctrl2) & MVNETA_GMAC2_INBAND_AN_ENABLE || 3496 (new_an ^ gmac_an) & MVNETA_GMAC_INBAND_AN_ENABLE) { 3497 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, 3498 (gmac_an & ~MVNETA_GMAC_FORCE_LINK_PASS) | 3499 MVNETA_GMAC_FORCE_LINK_DOWN); 3500 } 3501 3502 /* When at 2.5G, the link partner can send frames with shortened 3503 * preambles. 3504 */ 3505 if (state->speed == SPEED_2500) 3506 new_ctrl4 |= MVNETA_GMAC4_SHORT_PREAMBLE_ENABLE; 3507 3508 if (new_ctrl0 != gmac_ctrl0) 3509 mvreg_write(pp, MVNETA_GMAC_CTRL_0, new_ctrl0); 3510 if (new_ctrl2 != gmac_ctrl2) 3511 mvreg_write(pp, MVNETA_GMAC_CTRL_2, new_ctrl2); 3512 if (new_ctrl4 != gmac_ctrl4) 3513 mvreg_write(pp, MVNETA_GMAC_CTRL_4, new_ctrl4); 3514 if (new_clk != gmac_clk) 3515 mvreg_write(pp, MVNETA_GMAC_CLOCK_DIVIDER, new_clk); 3516 if (new_an != gmac_an) 3517 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, new_an); 3518 3519 if (gmac_ctrl2 & MVNETA_GMAC2_PORT_RESET) { 3520 while ((mvreg_read(pp, MVNETA_GMAC_CTRL_2) & 3521 MVNETA_GMAC2_PORT_RESET) != 0) 3522 continue; 3523 } 3524 } 3525 3526 static void mvneta_set_eee(struct mvneta_port *pp, bool enable) 3527 { 3528 u32 lpi_ctl1; 3529 3530 lpi_ctl1 = mvreg_read(pp, MVNETA_LPI_CTRL_1); 3531 if (enable) 3532 lpi_ctl1 |= MVNETA_LPI_REQUEST_ENABLE; 3533 else 3534 lpi_ctl1 &= ~MVNETA_LPI_REQUEST_ENABLE; 3535 mvreg_write(pp, MVNETA_LPI_CTRL_1, lpi_ctl1); 3536 } 3537 3538 static void mvneta_mac_link_down(struct net_device *ndev, unsigned int mode, 3539 phy_interface_t interface) 3540 { 3541 struct mvneta_port *pp = netdev_priv(ndev); 3542 u32 val; 3543 3544 mvneta_port_down(pp); 3545 3546 if (!phylink_autoneg_inband(mode)) { 3547 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3548 val &= ~MVNETA_GMAC_FORCE_LINK_PASS; 3549 val |= MVNETA_GMAC_FORCE_LINK_DOWN; 3550 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); 3551 } 3552 3553 pp->eee_active = false; 3554 mvneta_set_eee(pp, false); 3555 } 3556 3557 static void mvneta_mac_link_up(struct net_device *ndev, unsigned int mode, 3558 phy_interface_t interface, 3559 struct phy_device *phy) 3560 { 3561 struct mvneta_port *pp = netdev_priv(ndev); 3562 u32 val; 3563 3564 if (!phylink_autoneg_inband(mode)) { 3565 val = mvreg_read(pp, MVNETA_GMAC_AUTONEG_CONFIG); 3566 val &= ~MVNETA_GMAC_FORCE_LINK_DOWN; 3567 val |= MVNETA_GMAC_FORCE_LINK_PASS; 3568 mvreg_write(pp, MVNETA_GMAC_AUTONEG_CONFIG, val); 3569 } 3570 3571 mvneta_port_up(pp); 3572 3573 if (phy && pp->eee_enabled) { 3574 pp->eee_active = phy_init_eee(phy, 0) >= 0; 3575 mvneta_set_eee(pp, pp->eee_active && pp->tx_lpi_enabled); 3576 } 3577 } 3578 3579 static const struct phylink_mac_ops mvneta_phylink_ops = { 3580 .validate = mvneta_validate, 3581 .mac_link_state = mvneta_mac_link_state, 3582 .mac_an_restart = mvneta_mac_an_restart, 3583 .mac_config = mvneta_mac_config, 3584 .mac_link_down = mvneta_mac_link_down, 3585 .mac_link_up = mvneta_mac_link_up, 3586 }; 3587 3588 static int mvneta_mdio_probe(struct mvneta_port *pp) 3589 { 3590 struct ethtool_wolinfo wol = { .cmd = ETHTOOL_GWOL }; 3591 int err = phylink_of_phy_connect(pp->phylink, pp->dn, 0); 3592 3593 if (err) 3594 netdev_err(pp->dev, "could not attach PHY: %d\n", err); 3595 3596 phylink_ethtool_get_wol(pp->phylink, &wol); 3597 device_set_wakeup_capable(&pp->dev->dev, !!wol.supported); 3598 3599 return err; 3600 } 3601 3602 static void mvneta_mdio_remove(struct mvneta_port *pp) 3603 { 3604 phylink_disconnect_phy(pp->phylink); 3605 } 3606 3607 /* Electing a CPU must be done in an atomic way: it should be done 3608 * after or before the removal/insertion of a CPU and this function is 3609 * not reentrant. 3610 */ 3611 static void mvneta_percpu_elect(struct mvneta_port *pp) 3612 { 3613 int elected_cpu = 0, max_cpu, cpu, i = 0; 3614 3615 /* Use the cpu associated to the rxq when it is online, in all 3616 * the other cases, use the cpu 0 which can't be offline. 3617 */ 3618 if (cpu_online(pp->rxq_def)) 3619 elected_cpu = pp->rxq_def; 3620 3621 max_cpu = num_present_cpus(); 3622 3623 for_each_online_cpu(cpu) { 3624 int rxq_map = 0, txq_map = 0; 3625 int rxq; 3626 3627 for (rxq = 0; rxq < rxq_number; rxq++) 3628 if ((rxq % max_cpu) == cpu) 3629 rxq_map |= MVNETA_CPU_RXQ_ACCESS(rxq); 3630 3631 if (cpu == elected_cpu) 3632 /* Map the default receive queue queue to the 3633 * elected CPU 3634 */ 3635 rxq_map |= MVNETA_CPU_RXQ_ACCESS(pp->rxq_def); 3636 3637 /* We update the TX queue map only if we have one 3638 * queue. In this case we associate the TX queue to 3639 * the CPU bound to the default RX queue 3640 */ 3641 if (txq_number == 1) 3642 txq_map = (cpu == elected_cpu) ? 3643 MVNETA_CPU_TXQ_ACCESS(1) : 0; 3644 else 3645 txq_map = mvreg_read(pp, MVNETA_CPU_MAP(cpu)) & 3646 MVNETA_CPU_TXQ_ACCESS_ALL_MASK; 3647 3648 mvreg_write(pp, MVNETA_CPU_MAP(cpu), rxq_map | txq_map); 3649 3650 /* Update the interrupt mask on each CPU according the 3651 * new mapping 3652 */ 3653 smp_call_function_single(cpu, mvneta_percpu_unmask_interrupt, 3654 pp, true); 3655 i++; 3656 3657 } 3658 }; 3659 3660 static int mvneta_cpu_online(unsigned int cpu, struct hlist_node *node) 3661 { 3662 int other_cpu; 3663 struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port, 3664 node_online); 3665 struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); 3666 3667 3668 spin_lock(&pp->lock); 3669 /* 3670 * Configuring the driver for a new CPU while the driver is 3671 * stopping is racy, so just avoid it. 3672 */ 3673 if (pp->is_stopped) { 3674 spin_unlock(&pp->lock); 3675 return 0; 3676 } 3677 netif_tx_stop_all_queues(pp->dev); 3678 3679 /* 3680 * We have to synchronise on tha napi of each CPU except the one 3681 * just being woken up 3682 */ 3683 for_each_online_cpu(other_cpu) { 3684 if (other_cpu != cpu) { 3685 struct mvneta_pcpu_port *other_port = 3686 per_cpu_ptr(pp->ports, other_cpu); 3687 3688 napi_synchronize(&other_port->napi); 3689 } 3690 } 3691 3692 /* Mask all ethernet port interrupts */ 3693 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3694 napi_enable(&port->napi); 3695 3696 /* 3697 * Enable per-CPU interrupts on the CPU that is 3698 * brought up. 3699 */ 3700 mvneta_percpu_enable(pp); 3701 3702 /* 3703 * Enable per-CPU interrupt on the one CPU we care 3704 * about. 3705 */ 3706 mvneta_percpu_elect(pp); 3707 3708 /* Unmask all ethernet port interrupts */ 3709 on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true); 3710 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 3711 MVNETA_CAUSE_PHY_STATUS_CHANGE | 3712 MVNETA_CAUSE_LINK_CHANGE); 3713 netif_tx_start_all_queues(pp->dev); 3714 spin_unlock(&pp->lock); 3715 return 0; 3716 } 3717 3718 static int mvneta_cpu_down_prepare(unsigned int cpu, struct hlist_node *node) 3719 { 3720 struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port, 3721 node_online); 3722 struct mvneta_pcpu_port *port = per_cpu_ptr(pp->ports, cpu); 3723 3724 /* 3725 * Thanks to this lock we are sure that any pending cpu election is 3726 * done. 3727 */ 3728 spin_lock(&pp->lock); 3729 /* Mask all ethernet port interrupts */ 3730 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 3731 spin_unlock(&pp->lock); 3732 3733 napi_synchronize(&port->napi); 3734 napi_disable(&port->napi); 3735 /* Disable per-CPU interrupts on the CPU that is brought down. */ 3736 mvneta_percpu_disable(pp); 3737 return 0; 3738 } 3739 3740 static int mvneta_cpu_dead(unsigned int cpu, struct hlist_node *node) 3741 { 3742 struct mvneta_port *pp = hlist_entry_safe(node, struct mvneta_port, 3743 node_dead); 3744 3745 /* Check if a new CPU must be elected now this on is down */ 3746 spin_lock(&pp->lock); 3747 mvneta_percpu_elect(pp); 3748 spin_unlock(&pp->lock); 3749 /* Unmask all ethernet port interrupts */ 3750 on_each_cpu(mvneta_percpu_unmask_interrupt, pp, true); 3751 mvreg_write(pp, MVNETA_INTR_MISC_MASK, 3752 MVNETA_CAUSE_PHY_STATUS_CHANGE | 3753 MVNETA_CAUSE_LINK_CHANGE); 3754 netif_tx_start_all_queues(pp->dev); 3755 return 0; 3756 } 3757 3758 static int mvneta_open(struct net_device *dev) 3759 { 3760 struct mvneta_port *pp = netdev_priv(dev); 3761 int ret; 3762 3763 pp->pkt_size = MVNETA_RX_PKT_SIZE(pp->dev->mtu); 3764 3765 ret = mvneta_setup_rxqs(pp); 3766 if (ret) 3767 return ret; 3768 3769 ret = mvneta_setup_txqs(pp); 3770 if (ret) 3771 goto err_cleanup_rxqs; 3772 3773 /* Connect to port interrupt line */ 3774 if (pp->neta_armada3700) 3775 ret = request_irq(pp->dev->irq, mvneta_isr, 0, 3776 dev->name, pp); 3777 else 3778 ret = request_percpu_irq(pp->dev->irq, mvneta_percpu_isr, 3779 dev->name, pp->ports); 3780 if (ret) { 3781 netdev_err(pp->dev, "cannot request irq %d\n", pp->dev->irq); 3782 goto err_cleanup_txqs; 3783 } 3784 3785 if (!pp->neta_armada3700) { 3786 /* Enable per-CPU interrupt on all the CPU to handle our RX 3787 * queue interrupts 3788 */ 3789 on_each_cpu(mvneta_percpu_enable, pp, true); 3790 3791 pp->is_stopped = false; 3792 /* Register a CPU notifier to handle the case where our CPU 3793 * might be taken offline. 3794 */ 3795 ret = cpuhp_state_add_instance_nocalls(online_hpstate, 3796 &pp->node_online); 3797 if (ret) 3798 goto err_free_irq; 3799 3800 ret = cpuhp_state_add_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 3801 &pp->node_dead); 3802 if (ret) 3803 goto err_free_online_hp; 3804 } 3805 3806 ret = mvneta_mdio_probe(pp); 3807 if (ret < 0) { 3808 netdev_err(dev, "cannot probe MDIO bus\n"); 3809 goto err_free_dead_hp; 3810 } 3811 3812 mvneta_start_dev(pp); 3813 3814 return 0; 3815 3816 err_free_dead_hp: 3817 if (!pp->neta_armada3700) 3818 cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 3819 &pp->node_dead); 3820 err_free_online_hp: 3821 if (!pp->neta_armada3700) 3822 cpuhp_state_remove_instance_nocalls(online_hpstate, 3823 &pp->node_online); 3824 err_free_irq: 3825 if (pp->neta_armada3700) { 3826 free_irq(pp->dev->irq, pp); 3827 } else { 3828 on_each_cpu(mvneta_percpu_disable, pp, true); 3829 free_percpu_irq(pp->dev->irq, pp->ports); 3830 } 3831 err_cleanup_txqs: 3832 mvneta_cleanup_txqs(pp); 3833 err_cleanup_rxqs: 3834 mvneta_cleanup_rxqs(pp); 3835 return ret; 3836 } 3837 3838 /* Stop the port, free port interrupt line */ 3839 static int mvneta_stop(struct net_device *dev) 3840 { 3841 struct mvneta_port *pp = netdev_priv(dev); 3842 3843 if (!pp->neta_armada3700) { 3844 /* Inform that we are stopping so we don't want to setup the 3845 * driver for new CPUs in the notifiers. The code of the 3846 * notifier for CPU online is protected by the same spinlock, 3847 * so when we get the lock, the notifer work is done. 3848 */ 3849 spin_lock(&pp->lock); 3850 pp->is_stopped = true; 3851 spin_unlock(&pp->lock); 3852 3853 mvneta_stop_dev(pp); 3854 mvneta_mdio_remove(pp); 3855 3856 cpuhp_state_remove_instance_nocalls(online_hpstate, 3857 &pp->node_online); 3858 cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 3859 &pp->node_dead); 3860 on_each_cpu(mvneta_percpu_disable, pp, true); 3861 free_percpu_irq(dev->irq, pp->ports); 3862 } else { 3863 mvneta_stop_dev(pp); 3864 mvneta_mdio_remove(pp); 3865 free_irq(dev->irq, pp); 3866 } 3867 3868 mvneta_cleanup_rxqs(pp); 3869 mvneta_cleanup_txqs(pp); 3870 3871 return 0; 3872 } 3873 3874 static int mvneta_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 3875 { 3876 struct mvneta_port *pp = netdev_priv(dev); 3877 3878 return phylink_mii_ioctl(pp->phylink, ifr, cmd); 3879 } 3880 3881 /* Ethtool methods */ 3882 3883 /* Set link ksettings (phy address, speed) for ethtools */ 3884 static int 3885 mvneta_ethtool_set_link_ksettings(struct net_device *ndev, 3886 const struct ethtool_link_ksettings *cmd) 3887 { 3888 struct mvneta_port *pp = netdev_priv(ndev); 3889 3890 return phylink_ethtool_ksettings_set(pp->phylink, cmd); 3891 } 3892 3893 /* Get link ksettings for ethtools */ 3894 static int 3895 mvneta_ethtool_get_link_ksettings(struct net_device *ndev, 3896 struct ethtool_link_ksettings *cmd) 3897 { 3898 struct mvneta_port *pp = netdev_priv(ndev); 3899 3900 return phylink_ethtool_ksettings_get(pp->phylink, cmd); 3901 } 3902 3903 static int mvneta_ethtool_nway_reset(struct net_device *dev) 3904 { 3905 struct mvneta_port *pp = netdev_priv(dev); 3906 3907 return phylink_ethtool_nway_reset(pp->phylink); 3908 } 3909 3910 /* Set interrupt coalescing for ethtools */ 3911 static int mvneta_ethtool_set_coalesce(struct net_device *dev, 3912 struct ethtool_coalesce *c) 3913 { 3914 struct mvneta_port *pp = netdev_priv(dev); 3915 int queue; 3916 3917 for (queue = 0; queue < rxq_number; queue++) { 3918 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 3919 rxq->time_coal = c->rx_coalesce_usecs; 3920 rxq->pkts_coal = c->rx_max_coalesced_frames; 3921 mvneta_rx_pkts_coal_set(pp, rxq, rxq->pkts_coal); 3922 mvneta_rx_time_coal_set(pp, rxq, rxq->time_coal); 3923 } 3924 3925 for (queue = 0; queue < txq_number; queue++) { 3926 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 3927 txq->done_pkts_coal = c->tx_max_coalesced_frames; 3928 mvneta_tx_done_pkts_coal_set(pp, txq, txq->done_pkts_coal); 3929 } 3930 3931 return 0; 3932 } 3933 3934 /* get coalescing for ethtools */ 3935 static int mvneta_ethtool_get_coalesce(struct net_device *dev, 3936 struct ethtool_coalesce *c) 3937 { 3938 struct mvneta_port *pp = netdev_priv(dev); 3939 3940 c->rx_coalesce_usecs = pp->rxqs[0].time_coal; 3941 c->rx_max_coalesced_frames = pp->rxqs[0].pkts_coal; 3942 3943 c->tx_max_coalesced_frames = pp->txqs[0].done_pkts_coal; 3944 return 0; 3945 } 3946 3947 3948 static void mvneta_ethtool_get_drvinfo(struct net_device *dev, 3949 struct ethtool_drvinfo *drvinfo) 3950 { 3951 strlcpy(drvinfo->driver, MVNETA_DRIVER_NAME, 3952 sizeof(drvinfo->driver)); 3953 strlcpy(drvinfo->version, MVNETA_DRIVER_VERSION, 3954 sizeof(drvinfo->version)); 3955 strlcpy(drvinfo->bus_info, dev_name(&dev->dev), 3956 sizeof(drvinfo->bus_info)); 3957 } 3958 3959 3960 static void mvneta_ethtool_get_ringparam(struct net_device *netdev, 3961 struct ethtool_ringparam *ring) 3962 { 3963 struct mvneta_port *pp = netdev_priv(netdev); 3964 3965 ring->rx_max_pending = MVNETA_MAX_RXD; 3966 ring->tx_max_pending = MVNETA_MAX_TXD; 3967 ring->rx_pending = pp->rx_ring_size; 3968 ring->tx_pending = pp->tx_ring_size; 3969 } 3970 3971 static int mvneta_ethtool_set_ringparam(struct net_device *dev, 3972 struct ethtool_ringparam *ring) 3973 { 3974 struct mvneta_port *pp = netdev_priv(dev); 3975 3976 if ((ring->rx_pending == 0) || (ring->tx_pending == 0)) 3977 return -EINVAL; 3978 pp->rx_ring_size = ring->rx_pending < MVNETA_MAX_RXD ? 3979 ring->rx_pending : MVNETA_MAX_RXD; 3980 3981 pp->tx_ring_size = clamp_t(u16, ring->tx_pending, 3982 MVNETA_MAX_SKB_DESCS * 2, MVNETA_MAX_TXD); 3983 if (pp->tx_ring_size != ring->tx_pending) 3984 netdev_warn(dev, "TX queue size set to %u (requested %u)\n", 3985 pp->tx_ring_size, ring->tx_pending); 3986 3987 if (netif_running(dev)) { 3988 mvneta_stop(dev); 3989 if (mvneta_open(dev)) { 3990 netdev_err(dev, 3991 "error on opening device after ring param change\n"); 3992 return -ENOMEM; 3993 } 3994 } 3995 3996 return 0; 3997 } 3998 3999 static void mvneta_ethtool_get_pauseparam(struct net_device *dev, 4000 struct ethtool_pauseparam *pause) 4001 { 4002 struct mvneta_port *pp = netdev_priv(dev); 4003 4004 phylink_ethtool_get_pauseparam(pp->phylink, pause); 4005 } 4006 4007 static int mvneta_ethtool_set_pauseparam(struct net_device *dev, 4008 struct ethtool_pauseparam *pause) 4009 { 4010 struct mvneta_port *pp = netdev_priv(dev); 4011 4012 return phylink_ethtool_set_pauseparam(pp->phylink, pause); 4013 } 4014 4015 static void mvneta_ethtool_get_strings(struct net_device *netdev, u32 sset, 4016 u8 *data) 4017 { 4018 if (sset == ETH_SS_STATS) { 4019 int i; 4020 4021 for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) 4022 memcpy(data + i * ETH_GSTRING_LEN, 4023 mvneta_statistics[i].name, ETH_GSTRING_LEN); 4024 } 4025 } 4026 4027 static void mvneta_ethtool_update_stats(struct mvneta_port *pp) 4028 { 4029 const struct mvneta_statistic *s; 4030 void __iomem *base = pp->base; 4031 u32 high, low; 4032 u64 val; 4033 int i; 4034 4035 for (i = 0, s = mvneta_statistics; 4036 s < mvneta_statistics + ARRAY_SIZE(mvneta_statistics); 4037 s++, i++) { 4038 val = 0; 4039 4040 switch (s->type) { 4041 case T_REG_32: 4042 val = readl_relaxed(base + s->offset); 4043 break; 4044 case T_REG_64: 4045 /* Docs say to read low 32-bit then high */ 4046 low = readl_relaxed(base + s->offset); 4047 high = readl_relaxed(base + s->offset + 4); 4048 val = (u64)high << 32 | low; 4049 break; 4050 case T_SW: 4051 switch (s->offset) { 4052 case ETHTOOL_STAT_EEE_WAKEUP: 4053 val = phylink_get_eee_err(pp->phylink); 4054 break; 4055 case ETHTOOL_STAT_SKB_ALLOC_ERR: 4056 val = pp->rxqs[0].skb_alloc_err; 4057 break; 4058 case ETHTOOL_STAT_REFILL_ERR: 4059 val = pp->rxqs[0].refill_err; 4060 break; 4061 } 4062 break; 4063 } 4064 4065 pp->ethtool_stats[i] += val; 4066 } 4067 } 4068 4069 static void mvneta_ethtool_get_stats(struct net_device *dev, 4070 struct ethtool_stats *stats, u64 *data) 4071 { 4072 struct mvneta_port *pp = netdev_priv(dev); 4073 int i; 4074 4075 mvneta_ethtool_update_stats(pp); 4076 4077 for (i = 0; i < ARRAY_SIZE(mvneta_statistics); i++) 4078 *data++ = pp->ethtool_stats[i]; 4079 } 4080 4081 static int mvneta_ethtool_get_sset_count(struct net_device *dev, int sset) 4082 { 4083 if (sset == ETH_SS_STATS) 4084 return ARRAY_SIZE(mvneta_statistics); 4085 return -EOPNOTSUPP; 4086 } 4087 4088 static u32 mvneta_ethtool_get_rxfh_indir_size(struct net_device *dev) 4089 { 4090 return MVNETA_RSS_LU_TABLE_SIZE; 4091 } 4092 4093 static int mvneta_ethtool_get_rxnfc(struct net_device *dev, 4094 struct ethtool_rxnfc *info, 4095 u32 *rules __always_unused) 4096 { 4097 switch (info->cmd) { 4098 case ETHTOOL_GRXRINGS: 4099 info->data = rxq_number; 4100 return 0; 4101 case ETHTOOL_GRXFH: 4102 return -EOPNOTSUPP; 4103 default: 4104 return -EOPNOTSUPP; 4105 } 4106 } 4107 4108 static int mvneta_config_rss(struct mvneta_port *pp) 4109 { 4110 int cpu; 4111 u32 val; 4112 4113 netif_tx_stop_all_queues(pp->dev); 4114 4115 on_each_cpu(mvneta_percpu_mask_interrupt, pp, true); 4116 4117 if (!pp->neta_armada3700) { 4118 /* We have to synchronise on the napi of each CPU */ 4119 for_each_online_cpu(cpu) { 4120 struct mvneta_pcpu_port *pcpu_port = 4121 per_cpu_ptr(pp->ports, cpu); 4122 4123 napi_synchronize(&pcpu_port->napi); 4124 napi_disable(&pcpu_port->napi); 4125 } 4126 } else { 4127 napi_synchronize(&pp->napi); 4128 napi_disable(&pp->napi); 4129 } 4130 4131 pp->rxq_def = pp->indir[0]; 4132 4133 /* Update unicast mapping */ 4134 mvneta_set_rx_mode(pp->dev); 4135 4136 /* Update val of portCfg register accordingly with all RxQueue types */ 4137 val = MVNETA_PORT_CONFIG_DEFL_VALUE(pp->rxq_def); 4138 mvreg_write(pp, MVNETA_PORT_CONFIG, val); 4139 4140 /* Update the elected CPU matching the new rxq_def */ 4141 spin_lock(&pp->lock); 4142 mvneta_percpu_elect(pp); 4143 spin_unlock(&pp->lock); 4144 4145 if (!pp->neta_armada3700) { 4146 /* We have to synchronise on the napi of each CPU */ 4147 for_each_online_cpu(cpu) { 4148 struct mvneta_pcpu_port *pcpu_port = 4149 per_cpu_ptr(pp->ports, cpu); 4150 4151 napi_enable(&pcpu_port->napi); 4152 } 4153 } else { 4154 napi_enable(&pp->napi); 4155 } 4156 4157 netif_tx_start_all_queues(pp->dev); 4158 4159 return 0; 4160 } 4161 4162 static int mvneta_ethtool_set_rxfh(struct net_device *dev, const u32 *indir, 4163 const u8 *key, const u8 hfunc) 4164 { 4165 struct mvneta_port *pp = netdev_priv(dev); 4166 4167 /* Current code for Armada 3700 doesn't support RSS features yet */ 4168 if (pp->neta_armada3700) 4169 return -EOPNOTSUPP; 4170 4171 /* We require at least one supported parameter to be changed 4172 * and no change in any of the unsupported parameters 4173 */ 4174 if (key || 4175 (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP)) 4176 return -EOPNOTSUPP; 4177 4178 if (!indir) 4179 return 0; 4180 4181 memcpy(pp->indir, indir, MVNETA_RSS_LU_TABLE_SIZE); 4182 4183 return mvneta_config_rss(pp); 4184 } 4185 4186 static int mvneta_ethtool_get_rxfh(struct net_device *dev, u32 *indir, u8 *key, 4187 u8 *hfunc) 4188 { 4189 struct mvneta_port *pp = netdev_priv(dev); 4190 4191 /* Current code for Armada 3700 doesn't support RSS features yet */ 4192 if (pp->neta_armada3700) 4193 return -EOPNOTSUPP; 4194 4195 if (hfunc) 4196 *hfunc = ETH_RSS_HASH_TOP; 4197 4198 if (!indir) 4199 return 0; 4200 4201 memcpy(indir, pp->indir, MVNETA_RSS_LU_TABLE_SIZE); 4202 4203 return 0; 4204 } 4205 4206 static void mvneta_ethtool_get_wol(struct net_device *dev, 4207 struct ethtool_wolinfo *wol) 4208 { 4209 struct mvneta_port *pp = netdev_priv(dev); 4210 4211 phylink_ethtool_get_wol(pp->phylink, wol); 4212 } 4213 4214 static int mvneta_ethtool_set_wol(struct net_device *dev, 4215 struct ethtool_wolinfo *wol) 4216 { 4217 struct mvneta_port *pp = netdev_priv(dev); 4218 int ret; 4219 4220 ret = phylink_ethtool_set_wol(pp->phylink, wol); 4221 if (!ret) 4222 device_set_wakeup_enable(&dev->dev, !!wol->wolopts); 4223 4224 return ret; 4225 } 4226 4227 static int mvneta_ethtool_get_eee(struct net_device *dev, 4228 struct ethtool_eee *eee) 4229 { 4230 struct mvneta_port *pp = netdev_priv(dev); 4231 u32 lpi_ctl0; 4232 4233 lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0); 4234 4235 eee->eee_enabled = pp->eee_enabled; 4236 eee->eee_active = pp->eee_active; 4237 eee->tx_lpi_enabled = pp->tx_lpi_enabled; 4238 eee->tx_lpi_timer = (lpi_ctl0) >> 8; // * scale; 4239 4240 return phylink_ethtool_get_eee(pp->phylink, eee); 4241 } 4242 4243 static int mvneta_ethtool_set_eee(struct net_device *dev, 4244 struct ethtool_eee *eee) 4245 { 4246 struct mvneta_port *pp = netdev_priv(dev); 4247 u32 lpi_ctl0; 4248 4249 /* The Armada 37x documents do not give limits for this other than 4250 * it being an 8-bit register. */ 4251 if (eee->tx_lpi_enabled && eee->tx_lpi_timer > 255) 4252 return -EINVAL; 4253 4254 lpi_ctl0 = mvreg_read(pp, MVNETA_LPI_CTRL_0); 4255 lpi_ctl0 &= ~(0xff << 8); 4256 lpi_ctl0 |= eee->tx_lpi_timer << 8; 4257 mvreg_write(pp, MVNETA_LPI_CTRL_0, lpi_ctl0); 4258 4259 pp->eee_enabled = eee->eee_enabled; 4260 pp->tx_lpi_enabled = eee->tx_lpi_enabled; 4261 4262 mvneta_set_eee(pp, eee->tx_lpi_enabled && eee->eee_enabled); 4263 4264 return phylink_ethtool_set_eee(pp->phylink, eee); 4265 } 4266 4267 static const struct net_device_ops mvneta_netdev_ops = { 4268 .ndo_open = mvneta_open, 4269 .ndo_stop = mvneta_stop, 4270 .ndo_start_xmit = mvneta_tx, 4271 .ndo_set_rx_mode = mvneta_set_rx_mode, 4272 .ndo_set_mac_address = mvneta_set_mac_addr, 4273 .ndo_change_mtu = mvneta_change_mtu, 4274 .ndo_fix_features = mvneta_fix_features, 4275 .ndo_get_stats64 = mvneta_get_stats64, 4276 .ndo_do_ioctl = mvneta_ioctl, 4277 }; 4278 4279 static const struct ethtool_ops mvneta_eth_tool_ops = { 4280 .nway_reset = mvneta_ethtool_nway_reset, 4281 .get_link = ethtool_op_get_link, 4282 .set_coalesce = mvneta_ethtool_set_coalesce, 4283 .get_coalesce = mvneta_ethtool_get_coalesce, 4284 .get_drvinfo = mvneta_ethtool_get_drvinfo, 4285 .get_ringparam = mvneta_ethtool_get_ringparam, 4286 .set_ringparam = mvneta_ethtool_set_ringparam, 4287 .get_pauseparam = mvneta_ethtool_get_pauseparam, 4288 .set_pauseparam = mvneta_ethtool_set_pauseparam, 4289 .get_strings = mvneta_ethtool_get_strings, 4290 .get_ethtool_stats = mvneta_ethtool_get_stats, 4291 .get_sset_count = mvneta_ethtool_get_sset_count, 4292 .get_rxfh_indir_size = mvneta_ethtool_get_rxfh_indir_size, 4293 .get_rxnfc = mvneta_ethtool_get_rxnfc, 4294 .get_rxfh = mvneta_ethtool_get_rxfh, 4295 .set_rxfh = mvneta_ethtool_set_rxfh, 4296 .get_link_ksettings = mvneta_ethtool_get_link_ksettings, 4297 .set_link_ksettings = mvneta_ethtool_set_link_ksettings, 4298 .get_wol = mvneta_ethtool_get_wol, 4299 .set_wol = mvneta_ethtool_set_wol, 4300 .get_eee = mvneta_ethtool_get_eee, 4301 .set_eee = mvneta_ethtool_set_eee, 4302 }; 4303 4304 /* Initialize hw */ 4305 static int mvneta_init(struct device *dev, struct mvneta_port *pp) 4306 { 4307 int queue; 4308 4309 /* Disable port */ 4310 mvneta_port_disable(pp); 4311 4312 /* Set port default values */ 4313 mvneta_defaults_set(pp); 4314 4315 pp->txqs = devm_kcalloc(dev, txq_number, sizeof(*pp->txqs), GFP_KERNEL); 4316 if (!pp->txqs) 4317 return -ENOMEM; 4318 4319 /* Initialize TX descriptor rings */ 4320 for (queue = 0; queue < txq_number; queue++) { 4321 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 4322 txq->id = queue; 4323 txq->size = pp->tx_ring_size; 4324 txq->done_pkts_coal = MVNETA_TXDONE_COAL_PKTS; 4325 } 4326 4327 pp->rxqs = devm_kcalloc(dev, rxq_number, sizeof(*pp->rxqs), GFP_KERNEL); 4328 if (!pp->rxqs) 4329 return -ENOMEM; 4330 4331 /* Create Rx descriptor rings */ 4332 for (queue = 0; queue < rxq_number; queue++) { 4333 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 4334 rxq->id = queue; 4335 rxq->size = pp->rx_ring_size; 4336 rxq->pkts_coal = MVNETA_RX_COAL_PKTS; 4337 rxq->time_coal = MVNETA_RX_COAL_USEC; 4338 rxq->buf_virt_addr 4339 = devm_kmalloc_array(pp->dev->dev.parent, 4340 rxq->size, 4341 sizeof(*rxq->buf_virt_addr), 4342 GFP_KERNEL); 4343 if (!rxq->buf_virt_addr) 4344 return -ENOMEM; 4345 } 4346 4347 return 0; 4348 } 4349 4350 /* platform glue : initialize decoding windows */ 4351 static void mvneta_conf_mbus_windows(struct mvneta_port *pp, 4352 const struct mbus_dram_target_info *dram) 4353 { 4354 u32 win_enable; 4355 u32 win_protect; 4356 int i; 4357 4358 for (i = 0; i < 6; i++) { 4359 mvreg_write(pp, MVNETA_WIN_BASE(i), 0); 4360 mvreg_write(pp, MVNETA_WIN_SIZE(i), 0); 4361 4362 if (i < 4) 4363 mvreg_write(pp, MVNETA_WIN_REMAP(i), 0); 4364 } 4365 4366 win_enable = 0x3f; 4367 win_protect = 0; 4368 4369 if (dram) { 4370 for (i = 0; i < dram->num_cs; i++) { 4371 const struct mbus_dram_window *cs = dram->cs + i; 4372 4373 mvreg_write(pp, MVNETA_WIN_BASE(i), 4374 (cs->base & 0xffff0000) | 4375 (cs->mbus_attr << 8) | 4376 dram->mbus_dram_target_id); 4377 4378 mvreg_write(pp, MVNETA_WIN_SIZE(i), 4379 (cs->size - 1) & 0xffff0000); 4380 4381 win_enable &= ~(1 << i); 4382 win_protect |= 3 << (2 * i); 4383 } 4384 } else { 4385 /* For Armada3700 open default 4GB Mbus window, leaving 4386 * arbitration of target/attribute to a different layer 4387 * of configuration. 4388 */ 4389 mvreg_write(pp, MVNETA_WIN_SIZE(0), 0xffff0000); 4390 win_enable &= ~BIT(0); 4391 win_protect = 3; 4392 } 4393 4394 mvreg_write(pp, MVNETA_BASE_ADDR_ENABLE, win_enable); 4395 mvreg_write(pp, MVNETA_ACCESS_PROTECT_ENABLE, win_protect); 4396 } 4397 4398 /* Power up the port */ 4399 static int mvneta_port_power_up(struct mvneta_port *pp, int phy_mode) 4400 { 4401 /* MAC Cause register should be cleared */ 4402 mvreg_write(pp, MVNETA_UNIT_INTR_CAUSE, 0); 4403 4404 if (phy_mode == PHY_INTERFACE_MODE_QSGMII) 4405 mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_QSGMII_SERDES_PROTO); 4406 else if (phy_mode == PHY_INTERFACE_MODE_SGMII || 4407 phy_mode == PHY_INTERFACE_MODE_1000BASEX) 4408 mvreg_write(pp, MVNETA_SERDES_CFG, MVNETA_SGMII_SERDES_PROTO); 4409 else if (!phy_interface_mode_is_rgmii(phy_mode)) 4410 return -EINVAL; 4411 4412 return 0; 4413 } 4414 4415 /* Device initialization routine */ 4416 static int mvneta_probe(struct platform_device *pdev) 4417 { 4418 struct resource *res; 4419 struct device_node *dn = pdev->dev.of_node; 4420 struct device_node *bm_node; 4421 struct mvneta_port *pp; 4422 struct net_device *dev; 4423 struct phylink *phylink; 4424 const char *dt_mac_addr; 4425 char hw_mac_addr[ETH_ALEN]; 4426 const char *mac_from; 4427 int tx_csum_limit; 4428 int phy_mode; 4429 int err; 4430 int cpu; 4431 4432 dev = alloc_etherdev_mqs(sizeof(struct mvneta_port), txq_number, rxq_number); 4433 if (!dev) 4434 return -ENOMEM; 4435 4436 dev->irq = irq_of_parse_and_map(dn, 0); 4437 if (dev->irq == 0) { 4438 err = -EINVAL; 4439 goto err_free_netdev; 4440 } 4441 4442 phy_mode = of_get_phy_mode(dn); 4443 if (phy_mode < 0) { 4444 dev_err(&pdev->dev, "incorrect phy-mode\n"); 4445 err = -EINVAL; 4446 goto err_free_irq; 4447 } 4448 4449 phylink = phylink_create(dev, pdev->dev.fwnode, phy_mode, 4450 &mvneta_phylink_ops); 4451 if (IS_ERR(phylink)) { 4452 err = PTR_ERR(phylink); 4453 goto err_free_irq; 4454 } 4455 4456 dev->tx_queue_len = MVNETA_MAX_TXD; 4457 dev->watchdog_timeo = 5 * HZ; 4458 dev->netdev_ops = &mvneta_netdev_ops; 4459 4460 dev->ethtool_ops = &mvneta_eth_tool_ops; 4461 4462 pp = netdev_priv(dev); 4463 spin_lock_init(&pp->lock); 4464 pp->phylink = phylink; 4465 pp->phy_interface = phy_mode; 4466 pp->dn = dn; 4467 4468 pp->rxq_def = rxq_def; 4469 pp->indir[0] = rxq_def; 4470 4471 /* Get special SoC configurations */ 4472 if (of_device_is_compatible(dn, "marvell,armada-3700-neta")) 4473 pp->neta_armada3700 = true; 4474 4475 pp->clk = devm_clk_get(&pdev->dev, "core"); 4476 if (IS_ERR(pp->clk)) 4477 pp->clk = devm_clk_get(&pdev->dev, NULL); 4478 if (IS_ERR(pp->clk)) { 4479 err = PTR_ERR(pp->clk); 4480 goto err_free_phylink; 4481 } 4482 4483 clk_prepare_enable(pp->clk); 4484 4485 pp->clk_bus = devm_clk_get(&pdev->dev, "bus"); 4486 if (!IS_ERR(pp->clk_bus)) 4487 clk_prepare_enable(pp->clk_bus); 4488 4489 res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 4490 pp->base = devm_ioremap_resource(&pdev->dev, res); 4491 if (IS_ERR(pp->base)) { 4492 err = PTR_ERR(pp->base); 4493 goto err_clk; 4494 } 4495 4496 /* Alloc per-cpu port structure */ 4497 pp->ports = alloc_percpu(struct mvneta_pcpu_port); 4498 if (!pp->ports) { 4499 err = -ENOMEM; 4500 goto err_clk; 4501 } 4502 4503 /* Alloc per-cpu stats */ 4504 pp->stats = netdev_alloc_pcpu_stats(struct mvneta_pcpu_stats); 4505 if (!pp->stats) { 4506 err = -ENOMEM; 4507 goto err_free_ports; 4508 } 4509 4510 dt_mac_addr = of_get_mac_address(dn); 4511 if (dt_mac_addr) { 4512 mac_from = "device tree"; 4513 memcpy(dev->dev_addr, dt_mac_addr, ETH_ALEN); 4514 } else { 4515 mvneta_get_mac_addr(pp, hw_mac_addr); 4516 if (is_valid_ether_addr(hw_mac_addr)) { 4517 mac_from = "hardware"; 4518 memcpy(dev->dev_addr, hw_mac_addr, ETH_ALEN); 4519 } else { 4520 mac_from = "random"; 4521 eth_hw_addr_random(dev); 4522 } 4523 } 4524 4525 if (!of_property_read_u32(dn, "tx-csum-limit", &tx_csum_limit)) { 4526 if (tx_csum_limit < 0 || 4527 tx_csum_limit > MVNETA_TX_CSUM_MAX_SIZE) { 4528 tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE; 4529 dev_info(&pdev->dev, 4530 "Wrong TX csum limit in DT, set to %dB\n", 4531 MVNETA_TX_CSUM_DEF_SIZE); 4532 } 4533 } else if (of_device_is_compatible(dn, "marvell,armada-370-neta")) { 4534 tx_csum_limit = MVNETA_TX_CSUM_DEF_SIZE; 4535 } else { 4536 tx_csum_limit = MVNETA_TX_CSUM_MAX_SIZE; 4537 } 4538 4539 pp->tx_csum_limit = tx_csum_limit; 4540 4541 pp->dram_target_info = mv_mbus_dram_info(); 4542 /* Armada3700 requires setting default configuration of Mbus 4543 * windows, however without using filled mbus_dram_target_info 4544 * structure. 4545 */ 4546 if (pp->dram_target_info || pp->neta_armada3700) 4547 mvneta_conf_mbus_windows(pp, pp->dram_target_info); 4548 4549 pp->tx_ring_size = MVNETA_MAX_TXD; 4550 pp->rx_ring_size = MVNETA_MAX_RXD; 4551 4552 pp->dev = dev; 4553 SET_NETDEV_DEV(dev, &pdev->dev); 4554 4555 pp->id = global_port_id++; 4556 pp->rx_offset_correction = 0; /* not relevant for SW BM */ 4557 4558 /* Obtain access to BM resources if enabled and already initialized */ 4559 bm_node = of_parse_phandle(dn, "buffer-manager", 0); 4560 if (bm_node) { 4561 pp->bm_priv = mvneta_bm_get(bm_node); 4562 if (pp->bm_priv) { 4563 err = mvneta_bm_port_init(pdev, pp); 4564 if (err < 0) { 4565 dev_info(&pdev->dev, 4566 "use SW buffer management\n"); 4567 mvneta_bm_put(pp->bm_priv); 4568 pp->bm_priv = NULL; 4569 } 4570 } 4571 /* Set RX packet offset correction for platforms, whose 4572 * NET_SKB_PAD, exceeds 64B. It should be 64B for 64-bit 4573 * platforms and 0B for 32-bit ones. 4574 */ 4575 pp->rx_offset_correction = max(0, 4576 NET_SKB_PAD - 4577 MVNETA_RX_PKT_OFFSET_CORRECTION); 4578 } 4579 of_node_put(bm_node); 4580 4581 err = mvneta_init(&pdev->dev, pp); 4582 if (err < 0) 4583 goto err_netdev; 4584 4585 err = mvneta_port_power_up(pp, phy_mode); 4586 if (err < 0) { 4587 dev_err(&pdev->dev, "can't power up port\n"); 4588 goto err_netdev; 4589 } 4590 4591 /* Armada3700 network controller does not support per-cpu 4592 * operation, so only single NAPI should be initialized. 4593 */ 4594 if (pp->neta_armada3700) { 4595 netif_napi_add(dev, &pp->napi, mvneta_poll, NAPI_POLL_WEIGHT); 4596 } else { 4597 for_each_present_cpu(cpu) { 4598 struct mvneta_pcpu_port *port = 4599 per_cpu_ptr(pp->ports, cpu); 4600 4601 netif_napi_add(dev, &port->napi, mvneta_poll, 4602 NAPI_POLL_WEIGHT); 4603 port->pp = pp; 4604 } 4605 } 4606 4607 dev->features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | 4608 NETIF_F_TSO | NETIF_F_RXCSUM; 4609 dev->hw_features |= dev->features; 4610 dev->vlan_features |= dev->features; 4611 dev->priv_flags |= IFF_LIVE_ADDR_CHANGE; 4612 dev->gso_max_segs = MVNETA_MAX_TSO_SEGS; 4613 4614 /* MTU range: 68 - 9676 */ 4615 dev->min_mtu = ETH_MIN_MTU; 4616 /* 9676 == 9700 - 20 and rounding to 8 */ 4617 dev->max_mtu = 9676; 4618 4619 err = register_netdev(dev); 4620 if (err < 0) { 4621 dev_err(&pdev->dev, "failed to register\n"); 4622 goto err_free_stats; 4623 } 4624 4625 netdev_info(dev, "Using %s mac address %pM\n", mac_from, 4626 dev->dev_addr); 4627 4628 platform_set_drvdata(pdev, pp->dev); 4629 4630 return 0; 4631 4632 err_netdev: 4633 unregister_netdev(dev); 4634 if (pp->bm_priv) { 4635 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 4636 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 4637 1 << pp->id); 4638 mvneta_bm_put(pp->bm_priv); 4639 } 4640 err_free_stats: 4641 free_percpu(pp->stats); 4642 err_free_ports: 4643 free_percpu(pp->ports); 4644 err_clk: 4645 clk_disable_unprepare(pp->clk_bus); 4646 clk_disable_unprepare(pp->clk); 4647 err_free_phylink: 4648 if (pp->phylink) 4649 phylink_destroy(pp->phylink); 4650 err_free_irq: 4651 irq_dispose_mapping(dev->irq); 4652 err_free_netdev: 4653 free_netdev(dev); 4654 return err; 4655 } 4656 4657 /* Device removal routine */ 4658 static int mvneta_remove(struct platform_device *pdev) 4659 { 4660 struct net_device *dev = platform_get_drvdata(pdev); 4661 struct mvneta_port *pp = netdev_priv(dev); 4662 4663 unregister_netdev(dev); 4664 clk_disable_unprepare(pp->clk_bus); 4665 clk_disable_unprepare(pp->clk); 4666 free_percpu(pp->ports); 4667 free_percpu(pp->stats); 4668 irq_dispose_mapping(dev->irq); 4669 phylink_destroy(pp->phylink); 4670 free_netdev(dev); 4671 4672 if (pp->bm_priv) { 4673 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_long, 1 << pp->id); 4674 mvneta_bm_pool_destroy(pp->bm_priv, pp->pool_short, 4675 1 << pp->id); 4676 mvneta_bm_put(pp->bm_priv); 4677 } 4678 4679 return 0; 4680 } 4681 4682 #ifdef CONFIG_PM_SLEEP 4683 static int mvneta_suspend(struct device *device) 4684 { 4685 int queue; 4686 struct net_device *dev = dev_get_drvdata(device); 4687 struct mvneta_port *pp = netdev_priv(dev); 4688 4689 if (!netif_running(dev)) 4690 goto clean_exit; 4691 4692 if (!pp->neta_armada3700) { 4693 spin_lock(&pp->lock); 4694 pp->is_stopped = true; 4695 spin_unlock(&pp->lock); 4696 4697 cpuhp_state_remove_instance_nocalls(online_hpstate, 4698 &pp->node_online); 4699 cpuhp_state_remove_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 4700 &pp->node_dead); 4701 } 4702 4703 rtnl_lock(); 4704 mvneta_stop_dev(pp); 4705 rtnl_unlock(); 4706 4707 for (queue = 0; queue < rxq_number; queue++) { 4708 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 4709 4710 mvneta_rxq_drop_pkts(pp, rxq); 4711 } 4712 4713 for (queue = 0; queue < txq_number; queue++) { 4714 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 4715 4716 mvneta_txq_hw_deinit(pp, txq); 4717 } 4718 4719 clean_exit: 4720 netif_device_detach(dev); 4721 clk_disable_unprepare(pp->clk_bus); 4722 clk_disable_unprepare(pp->clk); 4723 4724 return 0; 4725 } 4726 4727 static int mvneta_resume(struct device *device) 4728 { 4729 struct platform_device *pdev = to_platform_device(device); 4730 struct net_device *dev = dev_get_drvdata(device); 4731 struct mvneta_port *pp = netdev_priv(dev); 4732 int err, queue; 4733 4734 clk_prepare_enable(pp->clk); 4735 if (!IS_ERR(pp->clk_bus)) 4736 clk_prepare_enable(pp->clk_bus); 4737 if (pp->dram_target_info || pp->neta_armada3700) 4738 mvneta_conf_mbus_windows(pp, pp->dram_target_info); 4739 if (pp->bm_priv) { 4740 err = mvneta_bm_port_init(pdev, pp); 4741 if (err < 0) { 4742 dev_info(&pdev->dev, "use SW buffer management\n"); 4743 pp->bm_priv = NULL; 4744 } 4745 } 4746 mvneta_defaults_set(pp); 4747 err = mvneta_port_power_up(pp, pp->phy_interface); 4748 if (err < 0) { 4749 dev_err(device, "can't power up port\n"); 4750 return err; 4751 } 4752 4753 netif_device_attach(dev); 4754 4755 if (!netif_running(dev)) 4756 return 0; 4757 4758 for (queue = 0; queue < rxq_number; queue++) { 4759 struct mvneta_rx_queue *rxq = &pp->rxqs[queue]; 4760 4761 rxq->next_desc_to_proc = 0; 4762 mvneta_rxq_hw_init(pp, rxq); 4763 } 4764 4765 for (queue = 0; queue < txq_number; queue++) { 4766 struct mvneta_tx_queue *txq = &pp->txqs[queue]; 4767 4768 txq->next_desc_to_proc = 0; 4769 mvneta_txq_hw_init(pp, txq); 4770 } 4771 4772 if (!pp->neta_armada3700) { 4773 spin_lock(&pp->lock); 4774 pp->is_stopped = false; 4775 spin_unlock(&pp->lock); 4776 cpuhp_state_add_instance_nocalls(online_hpstate, 4777 &pp->node_online); 4778 cpuhp_state_add_instance_nocalls(CPUHP_NET_MVNETA_DEAD, 4779 &pp->node_dead); 4780 } 4781 4782 rtnl_lock(); 4783 mvneta_start_dev(pp); 4784 rtnl_unlock(); 4785 mvneta_set_rx_mode(dev); 4786 4787 return 0; 4788 } 4789 #endif 4790 4791 static SIMPLE_DEV_PM_OPS(mvneta_pm_ops, mvneta_suspend, mvneta_resume); 4792 4793 static const struct of_device_id mvneta_match[] = { 4794 { .compatible = "marvell,armada-370-neta" }, 4795 { .compatible = "marvell,armada-xp-neta" }, 4796 { .compatible = "marvell,armada-3700-neta" }, 4797 { } 4798 }; 4799 MODULE_DEVICE_TABLE(of, mvneta_match); 4800 4801 static struct platform_driver mvneta_driver = { 4802 .probe = mvneta_probe, 4803 .remove = mvneta_remove, 4804 .driver = { 4805 .name = MVNETA_DRIVER_NAME, 4806 .of_match_table = mvneta_match, 4807 .pm = &mvneta_pm_ops, 4808 }, 4809 }; 4810 4811 static int __init mvneta_driver_init(void) 4812 { 4813 int ret; 4814 4815 ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "net/mvmeta:online", 4816 mvneta_cpu_online, 4817 mvneta_cpu_down_prepare); 4818 if (ret < 0) 4819 goto out; 4820 online_hpstate = ret; 4821 ret = cpuhp_setup_state_multi(CPUHP_NET_MVNETA_DEAD, "net/mvneta:dead", 4822 NULL, mvneta_cpu_dead); 4823 if (ret) 4824 goto err_dead; 4825 4826 ret = platform_driver_register(&mvneta_driver); 4827 if (ret) 4828 goto err; 4829 return 0; 4830 4831 err: 4832 cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD); 4833 err_dead: 4834 cpuhp_remove_multi_state(online_hpstate); 4835 out: 4836 return ret; 4837 } 4838 module_init(mvneta_driver_init); 4839 4840 static void __exit mvneta_driver_exit(void) 4841 { 4842 platform_driver_unregister(&mvneta_driver); 4843 cpuhp_remove_multi_state(CPUHP_NET_MVNETA_DEAD); 4844 cpuhp_remove_multi_state(online_hpstate); 4845 } 4846 module_exit(mvneta_driver_exit); 4847 4848 MODULE_DESCRIPTION("Marvell NETA Ethernet Driver - www.marvell.com"); 4849 MODULE_AUTHOR("Rami Rosen <rosenr@marvell.com>, Thomas Petazzoni <thomas.petazzoni@free-electrons.com>"); 4850 MODULE_LICENSE("GPL"); 4851 4852 module_param(rxq_number, int, 0444); 4853 module_param(txq_number, int, 0444); 4854 4855 module_param(rxq_def, int, 0444); 4856 module_param(rx_copybreak, int, 0644); 4857