1 // SPDX-License-Identifier: GPL-2.0+ 2 3 #include <linux/types.h> 4 #include <linux/clk.h> 5 #include <linux/platform_device.h> 6 #include <linux/pm_runtime.h> 7 #include <linux/acpi.h> 8 #include <linux/of_mdio.h> 9 #include <linux/etherdevice.h> 10 #include <linux/interrupt.h> 11 #include <linux/io.h> 12 #include <linux/netlink.h> 13 #include <linux/bpf.h> 14 #include <linux/bpf_trace.h> 15 16 #include <net/tcp.h> 17 #include <net/page_pool.h> 18 #include <net/ip6_checksum.h> 19 20 #define NETSEC_REG_SOFT_RST 0x104 21 #define NETSEC_REG_COM_INIT 0x120 22 23 #define NETSEC_REG_TOP_STATUS 0x200 24 #define NETSEC_IRQ_RX BIT(1) 25 #define NETSEC_IRQ_TX BIT(0) 26 27 #define NETSEC_REG_TOP_INTEN 0x204 28 #define NETSEC_REG_INTEN_SET 0x234 29 #define NETSEC_REG_INTEN_CLR 0x238 30 31 #define NETSEC_REG_NRM_TX_STATUS 0x400 32 #define NETSEC_REG_NRM_TX_INTEN 0x404 33 #define NETSEC_REG_NRM_TX_INTEN_SET 0x428 34 #define NETSEC_REG_NRM_TX_INTEN_CLR 0x42c 35 #define NRM_TX_ST_NTOWNR BIT(17) 36 #define NRM_TX_ST_TR_ERR BIT(16) 37 #define NRM_TX_ST_TXDONE BIT(15) 38 #define NRM_TX_ST_TMREXP BIT(14) 39 40 #define NETSEC_REG_NRM_RX_STATUS 0x440 41 #define NETSEC_REG_NRM_RX_INTEN 0x444 42 #define NETSEC_REG_NRM_RX_INTEN_SET 0x468 43 #define NETSEC_REG_NRM_RX_INTEN_CLR 0x46c 44 #define NRM_RX_ST_RC_ERR BIT(16) 45 #define NRM_RX_ST_PKTCNT BIT(15) 46 #define NRM_RX_ST_TMREXP BIT(14) 47 48 #define NETSEC_REG_PKT_CMD_BUF 0xd0 49 50 #define NETSEC_REG_CLK_EN 0x100 51 52 #define NETSEC_REG_PKT_CTRL 0x140 53 54 #define NETSEC_REG_DMA_TMR_CTRL 0x20c 55 #define NETSEC_REG_F_TAIKI_MC_VER 0x22c 56 #define NETSEC_REG_F_TAIKI_VER 0x230 57 #define NETSEC_REG_DMA_HM_CTRL 0x214 58 #define NETSEC_REG_DMA_MH_CTRL 0x220 59 #define NETSEC_REG_ADDR_DIS_CORE 0x218 60 #define NETSEC_REG_DMAC_HM_CMD_BUF 0x210 61 #define NETSEC_REG_DMAC_MH_CMD_BUF 0x21c 62 63 #define NETSEC_REG_NRM_TX_PKTCNT 0x410 64 65 #define NETSEC_REG_NRM_TX_DONE_PKTCNT 0x414 66 #define NETSEC_REG_NRM_TX_DONE_TXINT_PKTCNT 0x418 67 68 #define NETSEC_REG_NRM_TX_TMR 0x41c 69 70 #define NETSEC_REG_NRM_RX_PKTCNT 0x454 71 #define NETSEC_REG_NRM_RX_RXINT_PKTCNT 0x458 72 #define NETSEC_REG_NRM_TX_TXINT_TMR 0x420 73 #define NETSEC_REG_NRM_RX_RXINT_TMR 0x460 74 75 #define NETSEC_REG_NRM_RX_TMR 0x45c 76 77 #define NETSEC_REG_NRM_TX_DESC_START_UP 0x434 78 #define NETSEC_REG_NRM_TX_DESC_START_LW 0x408 79 #define NETSEC_REG_NRM_RX_DESC_START_UP 0x474 80 #define NETSEC_REG_NRM_RX_DESC_START_LW 0x448 81 82 #define NETSEC_REG_NRM_TX_CONFIG 0x430 83 #define NETSEC_REG_NRM_RX_CONFIG 0x470 84 85 #define MAC_REG_STATUS 0x1024 86 #define MAC_REG_DATA 0x11c0 87 #define MAC_REG_CMD 0x11c4 88 #define MAC_REG_FLOW_TH 0x11cc 89 #define MAC_REG_INTF_SEL 0x11d4 90 #define MAC_REG_DESC_INIT 0x11fc 91 #define MAC_REG_DESC_SOFT_RST 0x1204 92 #define NETSEC_REG_MODE_TRANS_COMP_STATUS 0x500 93 94 #define GMAC_REG_MCR 0x0000 95 #define GMAC_REG_MFFR 0x0004 96 #define GMAC_REG_GAR 0x0010 97 #define GMAC_REG_GDR 0x0014 98 #define GMAC_REG_FCR 0x0018 99 #define GMAC_REG_BMR 0x1000 100 #define GMAC_REG_RDLAR 0x100c 101 #define GMAC_REG_TDLAR 0x1010 102 #define GMAC_REG_OMR 0x1018 103 104 #define MHZ(n) ((n) * 1000 * 1000) 105 106 #define NETSEC_TX_SHIFT_OWN_FIELD 31 107 #define NETSEC_TX_SHIFT_LD_FIELD 30 108 #define NETSEC_TX_SHIFT_DRID_FIELD 24 109 #define NETSEC_TX_SHIFT_PT_FIELD 21 110 #define NETSEC_TX_SHIFT_TDRID_FIELD 16 111 #define NETSEC_TX_SHIFT_CC_FIELD 15 112 #define NETSEC_TX_SHIFT_FS_FIELD 9 113 #define NETSEC_TX_LAST 8 114 #define NETSEC_TX_SHIFT_CO 7 115 #define NETSEC_TX_SHIFT_SO 6 116 #define NETSEC_TX_SHIFT_TRS_FIELD 4 117 118 #define NETSEC_RX_PKT_OWN_FIELD 31 119 #define NETSEC_RX_PKT_LD_FIELD 30 120 #define NETSEC_RX_PKT_SDRID_FIELD 24 121 #define NETSEC_RX_PKT_FR_FIELD 23 122 #define NETSEC_RX_PKT_ER_FIELD 21 123 #define NETSEC_RX_PKT_ERR_FIELD 16 124 #define NETSEC_RX_PKT_TDRID_FIELD 12 125 #define NETSEC_RX_PKT_FS_FIELD 9 126 #define NETSEC_RX_PKT_LS_FIELD 8 127 #define NETSEC_RX_PKT_CO_FIELD 6 128 129 #define NETSEC_RX_PKT_ERR_MASK 3 130 131 #define NETSEC_MAX_TX_PKT_LEN 1518 132 #define NETSEC_MAX_TX_JUMBO_PKT_LEN 9018 133 134 #define NETSEC_RING_GMAC 15 135 #define NETSEC_RING_MAX 2 136 137 #define NETSEC_TCP_SEG_LEN_MAX 1460 138 #define NETSEC_TCP_JUMBO_SEG_LEN_MAX 8960 139 140 #define NETSEC_RX_CKSUM_NOTAVAIL 0 141 #define NETSEC_RX_CKSUM_OK 1 142 #define NETSEC_RX_CKSUM_NG 2 143 144 #define NETSEC_TOP_IRQ_REG_CODE_LOAD_END BIT(20) 145 #define NETSEC_IRQ_TRANSITION_COMPLETE BIT(4) 146 147 #define NETSEC_MODE_TRANS_COMP_IRQ_N2T BIT(20) 148 #define NETSEC_MODE_TRANS_COMP_IRQ_T2N BIT(19) 149 150 #define NETSEC_INT_PKTCNT_MAX 2047 151 152 #define NETSEC_FLOW_START_TH_MAX 95 153 #define NETSEC_FLOW_STOP_TH_MAX 95 154 #define NETSEC_FLOW_PAUSE_TIME_MIN 5 155 156 #define NETSEC_CLK_EN_REG_DOM_ALL 0x3f 157 158 #define NETSEC_PKT_CTRL_REG_MODE_NRM BIT(28) 159 #define NETSEC_PKT_CTRL_REG_EN_JUMBO BIT(27) 160 #define NETSEC_PKT_CTRL_REG_LOG_CHKSUM_ER BIT(3) 161 #define NETSEC_PKT_CTRL_REG_LOG_HD_INCOMPLETE BIT(2) 162 #define NETSEC_PKT_CTRL_REG_LOG_HD_ER BIT(1) 163 #define NETSEC_PKT_CTRL_REG_DRP_NO_MATCH BIT(0) 164 165 #define NETSEC_CLK_EN_REG_DOM_G BIT(5) 166 #define NETSEC_CLK_EN_REG_DOM_C BIT(1) 167 #define NETSEC_CLK_EN_REG_DOM_D BIT(0) 168 169 #define NETSEC_COM_INIT_REG_DB BIT(2) 170 #define NETSEC_COM_INIT_REG_CLS BIT(1) 171 #define NETSEC_COM_INIT_REG_ALL (NETSEC_COM_INIT_REG_CLS | \ 172 NETSEC_COM_INIT_REG_DB) 173 174 #define NETSEC_SOFT_RST_REG_RESET 0 175 #define NETSEC_SOFT_RST_REG_RUN BIT(31) 176 177 #define NETSEC_DMA_CTRL_REG_STOP 1 178 #define MH_CTRL__MODE_TRANS BIT(20) 179 180 #define NETSEC_GMAC_CMD_ST_READ 0 181 #define NETSEC_GMAC_CMD_ST_WRITE BIT(28) 182 #define NETSEC_GMAC_CMD_ST_BUSY BIT(31) 183 184 #define NETSEC_GMAC_BMR_REG_COMMON 0x00412080 185 #define NETSEC_GMAC_BMR_REG_RESET 0x00020181 186 #define NETSEC_GMAC_BMR_REG_SWR 0x00000001 187 188 #define NETSEC_GMAC_OMR_REG_ST BIT(13) 189 #define NETSEC_GMAC_OMR_REG_SR BIT(1) 190 191 #define NETSEC_GMAC_MCR_REG_IBN BIT(30) 192 #define NETSEC_GMAC_MCR_REG_CST BIT(25) 193 #define NETSEC_GMAC_MCR_REG_JE BIT(20) 194 #define NETSEC_MCR_PS BIT(15) 195 #define NETSEC_GMAC_MCR_REG_FES BIT(14) 196 #define NETSEC_GMAC_MCR_REG_FULL_DUPLEX_COMMON 0x0000280c 197 #define NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON 0x0001a00c 198 199 #define NETSEC_FCR_RFE BIT(2) 200 #define NETSEC_FCR_TFE BIT(1) 201 202 #define NETSEC_GMAC_GAR_REG_GW BIT(1) 203 #define NETSEC_GMAC_GAR_REG_GB BIT(0) 204 205 #define NETSEC_GMAC_GAR_REG_SHIFT_PA 11 206 #define NETSEC_GMAC_GAR_REG_SHIFT_GR 6 207 #define GMAC_REG_SHIFT_CR_GAR 2 208 209 #define NETSEC_GMAC_GAR_REG_CR_25_35_MHZ 2 210 #define NETSEC_GMAC_GAR_REG_CR_35_60_MHZ 3 211 #define NETSEC_GMAC_GAR_REG_CR_60_100_MHZ 0 212 #define NETSEC_GMAC_GAR_REG_CR_100_150_MHZ 1 213 #define NETSEC_GMAC_GAR_REG_CR_150_250_MHZ 4 214 #define NETSEC_GMAC_GAR_REG_CR_250_300_MHZ 5 215 216 #define NETSEC_GMAC_RDLAR_REG_COMMON 0x18000 217 #define NETSEC_GMAC_TDLAR_REG_COMMON 0x1c000 218 219 #define NETSEC_REG_NETSEC_VER_F_TAIKI 0x50000 220 221 #define NETSEC_REG_DESC_RING_CONFIG_CFG_UP BIT(31) 222 #define NETSEC_REG_DESC_RING_CONFIG_CH_RST BIT(30) 223 #define NETSEC_REG_DESC_TMR_MODE 4 224 #define NETSEC_REG_DESC_ENDIAN 0 225 226 #define NETSEC_MAC_DESC_SOFT_RST_SOFT_RST 1 227 #define NETSEC_MAC_DESC_INIT_REG_INIT 1 228 229 #define NETSEC_EEPROM_MAC_ADDRESS 0x00 230 #define NETSEC_EEPROM_HM_ME_ADDRESS_H 0x08 231 #define NETSEC_EEPROM_HM_ME_ADDRESS_L 0x0C 232 #define NETSEC_EEPROM_HM_ME_SIZE 0x10 233 #define NETSEC_EEPROM_MH_ME_ADDRESS_H 0x14 234 #define NETSEC_EEPROM_MH_ME_ADDRESS_L 0x18 235 #define NETSEC_EEPROM_MH_ME_SIZE 0x1C 236 #define NETSEC_EEPROM_PKT_ME_ADDRESS 0x20 237 #define NETSEC_EEPROM_PKT_ME_SIZE 0x24 238 239 #define DESC_NUM 256 240 241 #define NETSEC_SKB_PAD (NET_SKB_PAD + NET_IP_ALIGN) 242 #define NETSEC_RXBUF_HEADROOM (max(XDP_PACKET_HEADROOM, NET_SKB_PAD) + \ 243 NET_IP_ALIGN) 244 #define NETSEC_RX_BUF_NON_DATA (NETSEC_RXBUF_HEADROOM + \ 245 SKB_DATA_ALIGN(sizeof(struct skb_shared_info))) 246 247 #define DESC_SZ sizeof(struct netsec_de) 248 249 #define NETSEC_F_NETSEC_VER_MAJOR_NUM(x) ((x) & 0xffff0000) 250 251 #define NETSEC_XDP_PASS 0 252 #define NETSEC_XDP_CONSUMED BIT(0) 253 #define NETSEC_XDP_TX BIT(1) 254 #define NETSEC_XDP_REDIR BIT(2) 255 #define NETSEC_XDP_RX_OK (NETSEC_XDP_PASS | NETSEC_XDP_TX | NETSEC_XDP_REDIR) 256 257 enum ring_id { 258 NETSEC_RING_TX = 0, 259 NETSEC_RING_RX 260 }; 261 262 enum buf_type { 263 TYPE_NETSEC_SKB = 0, 264 TYPE_NETSEC_XDP_TX, 265 TYPE_NETSEC_XDP_NDO, 266 }; 267 268 struct netsec_desc { 269 union { 270 struct sk_buff *skb; 271 struct xdp_frame *xdpf; 272 }; 273 dma_addr_t dma_addr; 274 void *addr; 275 u16 len; 276 u8 buf_type; 277 }; 278 279 struct netsec_desc_ring { 280 dma_addr_t desc_dma; 281 struct netsec_desc *desc; 282 void *vaddr; 283 u16 head, tail; 284 u16 xdp_xmit; /* netsec_xdp_xmit packets */ 285 bool is_xdp; 286 struct page_pool *page_pool; 287 struct xdp_rxq_info xdp_rxq; 288 spinlock_t lock; /* XDP tx queue locking */ 289 }; 290 291 struct netsec_priv { 292 struct netsec_desc_ring desc_ring[NETSEC_RING_MAX]; 293 struct ethtool_coalesce et_coalesce; 294 struct bpf_prog *xdp_prog; 295 spinlock_t reglock; /* protect reg access */ 296 struct napi_struct napi; 297 phy_interface_t phy_interface; 298 struct net_device *ndev; 299 struct device_node *phy_np; 300 struct phy_device *phydev; 301 struct mii_bus *mii_bus; 302 void __iomem *ioaddr; 303 void __iomem *eeprom_base; 304 struct device *dev; 305 struct clk *clk; 306 u32 msg_enable; 307 u32 freq; 308 u32 phy_addr; 309 bool rx_cksum_offload_flag; 310 }; 311 312 struct netsec_de { /* Netsec Descriptor layout */ 313 u32 attr; 314 u32 data_buf_addr_up; 315 u32 data_buf_addr_lw; 316 u32 buf_len_info; 317 }; 318 319 struct netsec_tx_pkt_ctrl { 320 u16 tcp_seg_len; 321 bool tcp_seg_offload_flag; 322 bool cksum_offload_flag; 323 }; 324 325 struct netsec_rx_pkt_info { 326 int rx_cksum_result; 327 int err_code; 328 bool err_flag; 329 }; 330 331 static void netsec_set_tx_de(struct netsec_priv *priv, 332 struct netsec_desc_ring *dring, 333 const struct netsec_tx_pkt_ctrl *tx_ctrl, 334 const struct netsec_desc *desc, void *buf); 335 336 static void netsec_write(struct netsec_priv *priv, u32 reg_addr, u32 val) 337 { 338 writel(val, priv->ioaddr + reg_addr); 339 } 340 341 static u32 netsec_read(struct netsec_priv *priv, u32 reg_addr) 342 { 343 return readl(priv->ioaddr + reg_addr); 344 } 345 346 /************* MDIO BUS OPS FOLLOW *************/ 347 348 #define TIMEOUT_SPINS_MAC 1000 349 #define TIMEOUT_SECONDARY_MS_MAC 100 350 351 static u32 netsec_clk_type(u32 freq) 352 { 353 if (freq < MHZ(35)) 354 return NETSEC_GMAC_GAR_REG_CR_25_35_MHZ; 355 if (freq < MHZ(60)) 356 return NETSEC_GMAC_GAR_REG_CR_35_60_MHZ; 357 if (freq < MHZ(100)) 358 return NETSEC_GMAC_GAR_REG_CR_60_100_MHZ; 359 if (freq < MHZ(150)) 360 return NETSEC_GMAC_GAR_REG_CR_100_150_MHZ; 361 if (freq < MHZ(250)) 362 return NETSEC_GMAC_GAR_REG_CR_150_250_MHZ; 363 364 return NETSEC_GMAC_GAR_REG_CR_250_300_MHZ; 365 } 366 367 static int netsec_wait_while_busy(struct netsec_priv *priv, u32 addr, u32 mask) 368 { 369 u32 timeout = TIMEOUT_SPINS_MAC; 370 371 while (--timeout && netsec_read(priv, addr) & mask) 372 cpu_relax(); 373 if (timeout) 374 return 0; 375 376 timeout = TIMEOUT_SECONDARY_MS_MAC; 377 while (--timeout && netsec_read(priv, addr) & mask) 378 usleep_range(1000, 2000); 379 380 if (timeout) 381 return 0; 382 383 netdev_WARN(priv->ndev, "%s: timeout\n", __func__); 384 385 return -ETIMEDOUT; 386 } 387 388 static int netsec_mac_write(struct netsec_priv *priv, u32 addr, u32 value) 389 { 390 netsec_write(priv, MAC_REG_DATA, value); 391 netsec_write(priv, MAC_REG_CMD, addr | NETSEC_GMAC_CMD_ST_WRITE); 392 return netsec_wait_while_busy(priv, 393 MAC_REG_CMD, NETSEC_GMAC_CMD_ST_BUSY); 394 } 395 396 static int netsec_mac_read(struct netsec_priv *priv, u32 addr, u32 *read) 397 { 398 int ret; 399 400 netsec_write(priv, MAC_REG_CMD, addr | NETSEC_GMAC_CMD_ST_READ); 401 ret = netsec_wait_while_busy(priv, 402 MAC_REG_CMD, NETSEC_GMAC_CMD_ST_BUSY); 403 if (ret) 404 return ret; 405 406 *read = netsec_read(priv, MAC_REG_DATA); 407 408 return 0; 409 } 410 411 static int netsec_mac_wait_while_busy(struct netsec_priv *priv, 412 u32 addr, u32 mask) 413 { 414 u32 timeout = TIMEOUT_SPINS_MAC; 415 int ret, data; 416 417 do { 418 ret = netsec_mac_read(priv, addr, &data); 419 if (ret) 420 break; 421 cpu_relax(); 422 } while (--timeout && (data & mask)); 423 424 if (timeout) 425 return 0; 426 427 timeout = TIMEOUT_SECONDARY_MS_MAC; 428 do { 429 usleep_range(1000, 2000); 430 431 ret = netsec_mac_read(priv, addr, &data); 432 if (ret) 433 break; 434 cpu_relax(); 435 } while (--timeout && (data & mask)); 436 437 if (timeout && !ret) 438 return 0; 439 440 netdev_WARN(priv->ndev, "%s: timeout\n", __func__); 441 442 return -ETIMEDOUT; 443 } 444 445 static int netsec_mac_update_to_phy_state(struct netsec_priv *priv) 446 { 447 struct phy_device *phydev = priv->ndev->phydev; 448 u32 value = 0; 449 450 value = phydev->duplex ? NETSEC_GMAC_MCR_REG_FULL_DUPLEX_COMMON : 451 NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON; 452 453 if (phydev->speed != SPEED_1000) 454 value |= NETSEC_MCR_PS; 455 456 if (priv->phy_interface != PHY_INTERFACE_MODE_GMII && 457 phydev->speed == SPEED_100) 458 value |= NETSEC_GMAC_MCR_REG_FES; 459 460 value |= NETSEC_GMAC_MCR_REG_CST | NETSEC_GMAC_MCR_REG_JE; 461 462 if (phy_interface_mode_is_rgmii(priv->phy_interface)) 463 value |= NETSEC_GMAC_MCR_REG_IBN; 464 465 if (netsec_mac_write(priv, GMAC_REG_MCR, value)) 466 return -ETIMEDOUT; 467 468 return 0; 469 } 470 471 static int netsec_phy_read(struct mii_bus *bus, int phy_addr, int reg_addr); 472 473 static int netsec_phy_write(struct mii_bus *bus, 474 int phy_addr, int reg, u16 val) 475 { 476 int status; 477 struct netsec_priv *priv = bus->priv; 478 479 if (netsec_mac_write(priv, GMAC_REG_GDR, val)) 480 return -ETIMEDOUT; 481 if (netsec_mac_write(priv, GMAC_REG_GAR, 482 phy_addr << NETSEC_GMAC_GAR_REG_SHIFT_PA | 483 reg << NETSEC_GMAC_GAR_REG_SHIFT_GR | 484 NETSEC_GMAC_GAR_REG_GW | NETSEC_GMAC_GAR_REG_GB | 485 (netsec_clk_type(priv->freq) << 486 GMAC_REG_SHIFT_CR_GAR))) 487 return -ETIMEDOUT; 488 489 status = netsec_mac_wait_while_busy(priv, GMAC_REG_GAR, 490 NETSEC_GMAC_GAR_REG_GB); 491 492 /* Developerbox implements RTL8211E PHY and there is 493 * a compatibility problem with F_GMAC4. 494 * RTL8211E expects MDC clock must be kept toggling for several 495 * clock cycle with MDIO high before entering the IDLE state. 496 * To meet this requirement, netsec driver needs to issue dummy 497 * read(e.g. read PHYID1(offset 0x2) register) right after write. 498 */ 499 netsec_phy_read(bus, phy_addr, MII_PHYSID1); 500 501 return status; 502 } 503 504 static int netsec_phy_read(struct mii_bus *bus, int phy_addr, int reg_addr) 505 { 506 struct netsec_priv *priv = bus->priv; 507 u32 data; 508 int ret; 509 510 if (netsec_mac_write(priv, GMAC_REG_GAR, NETSEC_GMAC_GAR_REG_GB | 511 phy_addr << NETSEC_GMAC_GAR_REG_SHIFT_PA | 512 reg_addr << NETSEC_GMAC_GAR_REG_SHIFT_GR | 513 (netsec_clk_type(priv->freq) << 514 GMAC_REG_SHIFT_CR_GAR))) 515 return -ETIMEDOUT; 516 517 ret = netsec_mac_wait_while_busy(priv, GMAC_REG_GAR, 518 NETSEC_GMAC_GAR_REG_GB); 519 if (ret) 520 return ret; 521 522 ret = netsec_mac_read(priv, GMAC_REG_GDR, &data); 523 if (ret) 524 return ret; 525 526 return data; 527 } 528 529 /************* ETHTOOL_OPS FOLLOW *************/ 530 531 static void netsec_et_get_drvinfo(struct net_device *net_device, 532 struct ethtool_drvinfo *info) 533 { 534 strlcpy(info->driver, "netsec", sizeof(info->driver)); 535 strlcpy(info->bus_info, dev_name(net_device->dev.parent), 536 sizeof(info->bus_info)); 537 } 538 539 static int netsec_et_get_coalesce(struct net_device *net_device, 540 struct ethtool_coalesce *et_coalesce) 541 { 542 struct netsec_priv *priv = netdev_priv(net_device); 543 544 *et_coalesce = priv->et_coalesce; 545 546 return 0; 547 } 548 549 static int netsec_et_set_coalesce(struct net_device *net_device, 550 struct ethtool_coalesce *et_coalesce) 551 { 552 struct netsec_priv *priv = netdev_priv(net_device); 553 554 priv->et_coalesce = *et_coalesce; 555 556 if (priv->et_coalesce.tx_coalesce_usecs < 50) 557 priv->et_coalesce.tx_coalesce_usecs = 50; 558 if (priv->et_coalesce.tx_max_coalesced_frames < 1) 559 priv->et_coalesce.tx_max_coalesced_frames = 1; 560 561 netsec_write(priv, NETSEC_REG_NRM_TX_DONE_TXINT_PKTCNT, 562 priv->et_coalesce.tx_max_coalesced_frames); 563 netsec_write(priv, NETSEC_REG_NRM_TX_TXINT_TMR, 564 priv->et_coalesce.tx_coalesce_usecs); 565 netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_SET, NRM_TX_ST_TXDONE); 566 netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_SET, NRM_TX_ST_TMREXP); 567 568 if (priv->et_coalesce.rx_coalesce_usecs < 50) 569 priv->et_coalesce.rx_coalesce_usecs = 50; 570 if (priv->et_coalesce.rx_max_coalesced_frames < 1) 571 priv->et_coalesce.rx_max_coalesced_frames = 1; 572 573 netsec_write(priv, NETSEC_REG_NRM_RX_RXINT_PKTCNT, 574 priv->et_coalesce.rx_max_coalesced_frames); 575 netsec_write(priv, NETSEC_REG_NRM_RX_RXINT_TMR, 576 priv->et_coalesce.rx_coalesce_usecs); 577 netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_SET, NRM_RX_ST_PKTCNT); 578 netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_SET, NRM_RX_ST_TMREXP); 579 580 return 0; 581 } 582 583 static u32 netsec_et_get_msglevel(struct net_device *dev) 584 { 585 struct netsec_priv *priv = netdev_priv(dev); 586 587 return priv->msg_enable; 588 } 589 590 static void netsec_et_set_msglevel(struct net_device *dev, u32 datum) 591 { 592 struct netsec_priv *priv = netdev_priv(dev); 593 594 priv->msg_enable = datum; 595 } 596 597 static const struct ethtool_ops netsec_ethtool_ops = { 598 .get_drvinfo = netsec_et_get_drvinfo, 599 .get_link_ksettings = phy_ethtool_get_link_ksettings, 600 .set_link_ksettings = phy_ethtool_set_link_ksettings, 601 .get_link = ethtool_op_get_link, 602 .get_coalesce = netsec_et_get_coalesce, 603 .set_coalesce = netsec_et_set_coalesce, 604 .get_msglevel = netsec_et_get_msglevel, 605 .set_msglevel = netsec_et_set_msglevel, 606 }; 607 608 /************* NETDEV_OPS FOLLOW *************/ 609 610 611 static void netsec_set_rx_de(struct netsec_priv *priv, 612 struct netsec_desc_ring *dring, u16 idx, 613 const struct netsec_desc *desc) 614 { 615 struct netsec_de *de = dring->vaddr + DESC_SZ * idx; 616 u32 attr = (1 << NETSEC_RX_PKT_OWN_FIELD) | 617 (1 << NETSEC_RX_PKT_FS_FIELD) | 618 (1 << NETSEC_RX_PKT_LS_FIELD); 619 620 if (idx == DESC_NUM - 1) 621 attr |= (1 << NETSEC_RX_PKT_LD_FIELD); 622 623 de->data_buf_addr_up = upper_32_bits(desc->dma_addr); 624 de->data_buf_addr_lw = lower_32_bits(desc->dma_addr); 625 de->buf_len_info = desc->len; 626 de->attr = attr; 627 dma_wmb(); 628 629 dring->desc[idx].dma_addr = desc->dma_addr; 630 dring->desc[idx].addr = desc->addr; 631 dring->desc[idx].len = desc->len; 632 } 633 634 static bool netsec_clean_tx_dring(struct netsec_priv *priv) 635 { 636 struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX]; 637 struct netsec_de *entry; 638 int tail = dring->tail; 639 unsigned int bytes; 640 int cnt = 0; 641 642 if (dring->is_xdp) 643 spin_lock(&dring->lock); 644 645 bytes = 0; 646 entry = dring->vaddr + DESC_SZ * tail; 647 648 while (!(entry->attr & (1U << NETSEC_TX_SHIFT_OWN_FIELD)) && 649 cnt < DESC_NUM) { 650 struct netsec_desc *desc; 651 int eop; 652 653 desc = &dring->desc[tail]; 654 eop = (entry->attr >> NETSEC_TX_LAST) & 1; 655 dma_rmb(); 656 657 if (desc->buf_type == TYPE_NETSEC_SKB) 658 dma_unmap_single(priv->dev, desc->dma_addr, desc->len, 659 DMA_TO_DEVICE); 660 else if (desc->buf_type == TYPE_NETSEC_XDP_NDO) 661 dma_unmap_single(priv->dev, desc->dma_addr, 662 desc->len, DMA_TO_DEVICE); 663 664 if (!eop) 665 goto next; 666 667 if (desc->buf_type == TYPE_NETSEC_SKB) { 668 bytes += desc->skb->len; 669 dev_kfree_skb(desc->skb); 670 } else { 671 xdp_return_frame(desc->xdpf); 672 } 673 next: 674 /* clean up so netsec_uninit_pkt_dring() won't free the skb 675 * again 676 */ 677 *desc = (struct netsec_desc){}; 678 679 /* entry->attr is not going to be accessed by the NIC until 680 * netsec_set_tx_de() is called. No need for a dma_wmb() here 681 */ 682 entry->attr = 1U << NETSEC_TX_SHIFT_OWN_FIELD; 683 /* move tail ahead */ 684 dring->tail = (tail + 1) % DESC_NUM; 685 686 tail = dring->tail; 687 entry = dring->vaddr + DESC_SZ * tail; 688 cnt++; 689 } 690 if (dring->is_xdp) 691 spin_unlock(&dring->lock); 692 693 if (!cnt) 694 return false; 695 696 /* reading the register clears the irq */ 697 netsec_read(priv, NETSEC_REG_NRM_TX_DONE_PKTCNT); 698 699 priv->ndev->stats.tx_packets += cnt; 700 priv->ndev->stats.tx_bytes += bytes; 701 702 netdev_completed_queue(priv->ndev, cnt, bytes); 703 704 return true; 705 } 706 707 static void netsec_process_tx(struct netsec_priv *priv) 708 { 709 struct net_device *ndev = priv->ndev; 710 bool cleaned; 711 712 cleaned = netsec_clean_tx_dring(priv); 713 714 if (cleaned && netif_queue_stopped(ndev)) { 715 /* Make sure we update the value, anyone stopping the queue 716 * after this will read the proper consumer idx 717 */ 718 smp_wmb(); 719 netif_wake_queue(ndev); 720 } 721 } 722 723 static void *netsec_alloc_rx_data(struct netsec_priv *priv, 724 dma_addr_t *dma_handle, u16 *desc_len) 725 726 { 727 728 struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX]; 729 enum dma_data_direction dma_dir; 730 struct page *page; 731 732 page = page_pool_dev_alloc_pages(dring->page_pool); 733 if (!page) 734 return NULL; 735 736 /* We allocate the same buffer length for XDP and non-XDP cases. 737 * page_pool API will map the whole page, skip what's needed for 738 * network payloads and/or XDP 739 */ 740 *dma_handle = page_pool_get_dma_addr(page) + NETSEC_RXBUF_HEADROOM; 741 /* Make sure the incoming payload fits in the page for XDP and non-XDP 742 * cases and reserve enough space for headroom + skb_shared_info 743 */ 744 *desc_len = PAGE_SIZE - NETSEC_RX_BUF_NON_DATA; 745 dma_dir = page_pool_get_dma_dir(dring->page_pool); 746 dma_sync_single_for_device(priv->dev, *dma_handle, *desc_len, dma_dir); 747 748 return page_address(page); 749 } 750 751 static void netsec_rx_fill(struct netsec_priv *priv, u16 from, u16 num) 752 { 753 struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX]; 754 u16 idx = from; 755 756 while (num) { 757 netsec_set_rx_de(priv, dring, idx, &dring->desc[idx]); 758 idx++; 759 if (idx >= DESC_NUM) 760 idx = 0; 761 num--; 762 } 763 } 764 765 static void netsec_xdp_ring_tx_db(struct netsec_priv *priv, u16 pkts) 766 { 767 if (likely(pkts)) 768 netsec_write(priv, NETSEC_REG_NRM_TX_PKTCNT, pkts); 769 } 770 771 static void netsec_finalize_xdp_rx(struct netsec_priv *priv, u32 xdp_res, 772 u16 pkts) 773 { 774 if (xdp_res & NETSEC_XDP_REDIR) 775 xdp_do_flush_map(); 776 777 if (xdp_res & NETSEC_XDP_TX) 778 netsec_xdp_ring_tx_db(priv, pkts); 779 } 780 781 /* The current driver only supports 1 Txq, this should run under spin_lock() */ 782 static u32 netsec_xdp_queue_one(struct netsec_priv *priv, 783 struct xdp_frame *xdpf, bool is_ndo) 784 785 { 786 struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX]; 787 struct page *page = virt_to_page(xdpf->data); 788 struct netsec_tx_pkt_ctrl tx_ctrl = {}; 789 struct netsec_desc tx_desc; 790 dma_addr_t dma_handle; 791 u16 filled; 792 793 if (tx_ring->head >= tx_ring->tail) 794 filled = tx_ring->head - tx_ring->tail; 795 else 796 filled = tx_ring->head + DESC_NUM - tx_ring->tail; 797 798 if (DESC_NUM - filled <= 1) 799 return NETSEC_XDP_CONSUMED; 800 801 if (is_ndo) { 802 /* this is for ndo_xdp_xmit, the buffer needs mapping before 803 * sending 804 */ 805 dma_handle = dma_map_single(priv->dev, xdpf->data, xdpf->len, 806 DMA_TO_DEVICE); 807 if (dma_mapping_error(priv->dev, dma_handle)) 808 return NETSEC_XDP_CONSUMED; 809 tx_desc.buf_type = TYPE_NETSEC_XDP_NDO; 810 } else { 811 /* This is the device Rx buffer from page_pool. No need to remap 812 * just sync and send it 813 */ 814 struct netsec_desc_ring *rx_ring = 815 &priv->desc_ring[NETSEC_RING_RX]; 816 enum dma_data_direction dma_dir = 817 page_pool_get_dma_dir(rx_ring->page_pool); 818 819 dma_handle = page_pool_get_dma_addr(page) + 820 NETSEC_RXBUF_HEADROOM; 821 dma_sync_single_for_device(priv->dev, dma_handle, xdpf->len, 822 dma_dir); 823 tx_desc.buf_type = TYPE_NETSEC_XDP_TX; 824 } 825 826 tx_desc.dma_addr = dma_handle; 827 tx_desc.addr = xdpf->data; 828 tx_desc.len = xdpf->len; 829 830 netsec_set_tx_de(priv, tx_ring, &tx_ctrl, &tx_desc, xdpf); 831 832 return NETSEC_XDP_TX; 833 } 834 835 static u32 netsec_xdp_xmit_back(struct netsec_priv *priv, struct xdp_buff *xdp) 836 { 837 struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX]; 838 struct xdp_frame *xdpf = convert_to_xdp_frame(xdp); 839 u32 ret; 840 841 if (unlikely(!xdpf)) 842 return NETSEC_XDP_CONSUMED; 843 844 spin_lock(&tx_ring->lock); 845 ret = netsec_xdp_queue_one(priv, xdpf, false); 846 spin_unlock(&tx_ring->lock); 847 848 return ret; 849 } 850 851 static u32 netsec_run_xdp(struct netsec_priv *priv, struct bpf_prog *prog, 852 struct xdp_buff *xdp) 853 { 854 u32 ret = NETSEC_XDP_PASS; 855 int err; 856 u32 act; 857 858 act = bpf_prog_run_xdp(prog, xdp); 859 860 switch (act) { 861 case XDP_PASS: 862 ret = NETSEC_XDP_PASS; 863 break; 864 case XDP_TX: 865 ret = netsec_xdp_xmit_back(priv, xdp); 866 if (ret != NETSEC_XDP_TX) 867 xdp_return_buff(xdp); 868 break; 869 case XDP_REDIRECT: 870 err = xdp_do_redirect(priv->ndev, xdp, prog); 871 if (!err) { 872 ret = NETSEC_XDP_REDIR; 873 } else { 874 ret = NETSEC_XDP_CONSUMED; 875 xdp_return_buff(xdp); 876 } 877 break; 878 default: 879 bpf_warn_invalid_xdp_action(act); 880 /* fall through */ 881 case XDP_ABORTED: 882 trace_xdp_exception(priv->ndev, prog, act); 883 /* fall through -- handle aborts by dropping packet */ 884 case XDP_DROP: 885 ret = NETSEC_XDP_CONSUMED; 886 xdp_return_buff(xdp); 887 break; 888 } 889 890 return ret; 891 } 892 893 static int netsec_process_rx(struct netsec_priv *priv, int budget) 894 { 895 struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX]; 896 struct net_device *ndev = priv->ndev; 897 struct netsec_rx_pkt_info rx_info; 898 enum dma_data_direction dma_dir; 899 struct bpf_prog *xdp_prog; 900 struct sk_buff *skb = NULL; 901 u16 xdp_xmit = 0; 902 u32 xdp_act = 0; 903 int done = 0; 904 905 rcu_read_lock(); 906 xdp_prog = READ_ONCE(priv->xdp_prog); 907 dma_dir = page_pool_get_dma_dir(dring->page_pool); 908 909 while (done < budget) { 910 u16 idx = dring->tail; 911 struct netsec_de *de = dring->vaddr + (DESC_SZ * idx); 912 struct netsec_desc *desc = &dring->desc[idx]; 913 struct page *page = virt_to_page(desc->addr); 914 u32 xdp_result = XDP_PASS; 915 u16 pkt_len, desc_len; 916 dma_addr_t dma_handle; 917 struct xdp_buff xdp; 918 void *buf_addr; 919 920 if (de->attr & (1U << NETSEC_RX_PKT_OWN_FIELD)) { 921 /* reading the register clears the irq */ 922 netsec_read(priv, NETSEC_REG_NRM_RX_PKTCNT); 923 break; 924 } 925 926 /* This barrier is needed to keep us from reading 927 * any other fields out of the netsec_de until we have 928 * verified the descriptor has been written back 929 */ 930 dma_rmb(); 931 done++; 932 933 pkt_len = de->buf_len_info >> 16; 934 rx_info.err_code = (de->attr >> NETSEC_RX_PKT_ERR_FIELD) & 935 NETSEC_RX_PKT_ERR_MASK; 936 rx_info.err_flag = (de->attr >> NETSEC_RX_PKT_ER_FIELD) & 1; 937 if (rx_info.err_flag) { 938 netif_err(priv, drv, priv->ndev, 939 "%s: rx fail err(%d)\n", __func__, 940 rx_info.err_code); 941 ndev->stats.rx_dropped++; 942 dring->tail = (dring->tail + 1) % DESC_NUM; 943 /* reuse buffer page frag */ 944 netsec_rx_fill(priv, idx, 1); 945 continue; 946 } 947 rx_info.rx_cksum_result = 948 (de->attr >> NETSEC_RX_PKT_CO_FIELD) & 3; 949 950 /* allocate a fresh buffer and map it to the hardware. 951 * This will eventually replace the old buffer in the hardware 952 */ 953 buf_addr = netsec_alloc_rx_data(priv, &dma_handle, &desc_len); 954 955 if (unlikely(!buf_addr)) 956 break; 957 958 dma_sync_single_for_cpu(priv->dev, desc->dma_addr, pkt_len, 959 dma_dir); 960 prefetch(desc->addr); 961 962 xdp.data_hard_start = desc->addr; 963 xdp.data = desc->addr + NETSEC_RXBUF_HEADROOM; 964 xdp_set_data_meta_invalid(&xdp); 965 xdp.data_end = xdp.data + pkt_len; 966 xdp.rxq = &dring->xdp_rxq; 967 968 if (xdp_prog) { 969 xdp_result = netsec_run_xdp(priv, xdp_prog, &xdp); 970 if (xdp_result != NETSEC_XDP_PASS) { 971 xdp_act |= xdp_result; 972 if (xdp_result == NETSEC_XDP_TX) 973 xdp_xmit++; 974 goto next; 975 } 976 } 977 skb = build_skb(desc->addr, desc->len + NETSEC_RX_BUF_NON_DATA); 978 979 if (unlikely(!skb)) { 980 /* If skb fails recycle_direct will either unmap and 981 * free the page or refill the cache depending on the 982 * cache state. Since we paid the allocation cost if 983 * building an skb fails try to put the page into cache 984 */ 985 page_pool_recycle_direct(dring->page_pool, page); 986 netif_err(priv, drv, priv->ndev, 987 "rx failed to build skb\n"); 988 break; 989 } 990 page_pool_release_page(dring->page_pool, page); 991 992 skb_reserve(skb, xdp.data - xdp.data_hard_start); 993 skb_put(skb, xdp.data_end - xdp.data); 994 skb->protocol = eth_type_trans(skb, priv->ndev); 995 996 if (priv->rx_cksum_offload_flag && 997 rx_info.rx_cksum_result == NETSEC_RX_CKSUM_OK) 998 skb->ip_summed = CHECKSUM_UNNECESSARY; 999 1000 next: 1001 if ((skb && napi_gro_receive(&priv->napi, skb) != GRO_DROP) || 1002 xdp_result & NETSEC_XDP_RX_OK) { 1003 ndev->stats.rx_packets++; 1004 ndev->stats.rx_bytes += xdp.data_end - xdp.data; 1005 } 1006 1007 /* Update the descriptor with fresh buffers */ 1008 desc->len = desc_len; 1009 desc->dma_addr = dma_handle; 1010 desc->addr = buf_addr; 1011 1012 netsec_rx_fill(priv, idx, 1); 1013 dring->tail = (dring->tail + 1) % DESC_NUM; 1014 } 1015 netsec_finalize_xdp_rx(priv, xdp_act, xdp_xmit); 1016 1017 rcu_read_unlock(); 1018 1019 return done; 1020 } 1021 1022 static int netsec_napi_poll(struct napi_struct *napi, int budget) 1023 { 1024 struct netsec_priv *priv; 1025 int done; 1026 1027 priv = container_of(napi, struct netsec_priv, napi); 1028 1029 netsec_process_tx(priv); 1030 done = netsec_process_rx(priv, budget); 1031 1032 if (done < budget && napi_complete_done(napi, done)) { 1033 unsigned long flags; 1034 1035 spin_lock_irqsave(&priv->reglock, flags); 1036 netsec_write(priv, NETSEC_REG_INTEN_SET, 1037 NETSEC_IRQ_RX | NETSEC_IRQ_TX); 1038 spin_unlock_irqrestore(&priv->reglock, flags); 1039 } 1040 1041 return done; 1042 } 1043 1044 static void netsec_set_tx_de(struct netsec_priv *priv, 1045 struct netsec_desc_ring *dring, 1046 const struct netsec_tx_pkt_ctrl *tx_ctrl, 1047 const struct netsec_desc *desc, void *buf) 1048 { 1049 int idx = dring->head; 1050 struct netsec_de *de; 1051 u32 attr; 1052 1053 de = dring->vaddr + (DESC_SZ * idx); 1054 1055 attr = (1 << NETSEC_TX_SHIFT_OWN_FIELD) | 1056 (1 << NETSEC_TX_SHIFT_PT_FIELD) | 1057 (NETSEC_RING_GMAC << NETSEC_TX_SHIFT_TDRID_FIELD) | 1058 (1 << NETSEC_TX_SHIFT_FS_FIELD) | 1059 (1 << NETSEC_TX_LAST) | 1060 (tx_ctrl->cksum_offload_flag << NETSEC_TX_SHIFT_CO) | 1061 (tx_ctrl->tcp_seg_offload_flag << NETSEC_TX_SHIFT_SO) | 1062 (1 << NETSEC_TX_SHIFT_TRS_FIELD); 1063 if (idx == DESC_NUM - 1) 1064 attr |= (1 << NETSEC_TX_SHIFT_LD_FIELD); 1065 1066 de->data_buf_addr_up = upper_32_bits(desc->dma_addr); 1067 de->data_buf_addr_lw = lower_32_bits(desc->dma_addr); 1068 de->buf_len_info = (tx_ctrl->tcp_seg_len << 16) | desc->len; 1069 de->attr = attr; 1070 /* under spin_lock if using XDP */ 1071 if (!dring->is_xdp) 1072 dma_wmb(); 1073 1074 dring->desc[idx] = *desc; 1075 if (desc->buf_type == TYPE_NETSEC_SKB) 1076 dring->desc[idx].skb = buf; 1077 else if (desc->buf_type == TYPE_NETSEC_XDP_TX || 1078 desc->buf_type == TYPE_NETSEC_XDP_NDO) 1079 dring->desc[idx].xdpf = buf; 1080 1081 /* move head ahead */ 1082 dring->head = (dring->head + 1) % DESC_NUM; 1083 } 1084 1085 static int netsec_desc_used(struct netsec_desc_ring *dring) 1086 { 1087 int used; 1088 1089 if (dring->head >= dring->tail) 1090 used = dring->head - dring->tail; 1091 else 1092 used = dring->head + DESC_NUM - dring->tail; 1093 1094 return used; 1095 } 1096 1097 static int netsec_check_stop_tx(struct netsec_priv *priv, int used) 1098 { 1099 struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX]; 1100 1101 /* keep tail from touching the queue */ 1102 if (DESC_NUM - used < 2) { 1103 netif_stop_queue(priv->ndev); 1104 1105 /* Make sure we read the updated value in case 1106 * descriptors got freed 1107 */ 1108 smp_rmb(); 1109 1110 used = netsec_desc_used(dring); 1111 if (DESC_NUM - used < 2) 1112 return NETDEV_TX_BUSY; 1113 1114 netif_wake_queue(priv->ndev); 1115 } 1116 1117 return 0; 1118 } 1119 1120 static netdev_tx_t netsec_netdev_start_xmit(struct sk_buff *skb, 1121 struct net_device *ndev) 1122 { 1123 struct netsec_priv *priv = netdev_priv(ndev); 1124 struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX]; 1125 struct netsec_tx_pkt_ctrl tx_ctrl = {}; 1126 struct netsec_desc tx_desc; 1127 u16 tso_seg_len = 0; 1128 int filled; 1129 1130 if (dring->is_xdp) 1131 spin_lock_bh(&dring->lock); 1132 filled = netsec_desc_used(dring); 1133 if (netsec_check_stop_tx(priv, filled)) { 1134 if (dring->is_xdp) 1135 spin_unlock_bh(&dring->lock); 1136 net_warn_ratelimited("%s %s Tx queue full\n", 1137 dev_name(priv->dev), ndev->name); 1138 return NETDEV_TX_BUSY; 1139 } 1140 1141 if (skb->ip_summed == CHECKSUM_PARTIAL) 1142 tx_ctrl.cksum_offload_flag = true; 1143 1144 if (skb_is_gso(skb)) 1145 tso_seg_len = skb_shinfo(skb)->gso_size; 1146 1147 if (tso_seg_len > 0) { 1148 if (skb->protocol == htons(ETH_P_IP)) { 1149 ip_hdr(skb)->tot_len = 0; 1150 tcp_hdr(skb)->check = 1151 ~tcp_v4_check(0, ip_hdr(skb)->saddr, 1152 ip_hdr(skb)->daddr, 0); 1153 } else { 1154 ipv6_hdr(skb)->payload_len = 0; 1155 tcp_hdr(skb)->check = 1156 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 1157 &ipv6_hdr(skb)->daddr, 1158 0, IPPROTO_TCP, 0); 1159 } 1160 1161 tx_ctrl.tcp_seg_offload_flag = true; 1162 tx_ctrl.tcp_seg_len = tso_seg_len; 1163 } 1164 1165 tx_desc.dma_addr = dma_map_single(priv->dev, skb->data, 1166 skb_headlen(skb), DMA_TO_DEVICE); 1167 if (dma_mapping_error(priv->dev, tx_desc.dma_addr)) { 1168 if (dring->is_xdp) 1169 spin_unlock_bh(&dring->lock); 1170 netif_err(priv, drv, priv->ndev, 1171 "%s: DMA mapping failed\n", __func__); 1172 ndev->stats.tx_dropped++; 1173 dev_kfree_skb_any(skb); 1174 return NETDEV_TX_OK; 1175 } 1176 tx_desc.addr = skb->data; 1177 tx_desc.len = skb_headlen(skb); 1178 tx_desc.buf_type = TYPE_NETSEC_SKB; 1179 1180 skb_tx_timestamp(skb); 1181 netdev_sent_queue(priv->ndev, skb->len); 1182 1183 netsec_set_tx_de(priv, dring, &tx_ctrl, &tx_desc, skb); 1184 if (dring->is_xdp) 1185 spin_unlock_bh(&dring->lock); 1186 netsec_write(priv, NETSEC_REG_NRM_TX_PKTCNT, 1); /* submit another tx */ 1187 1188 return NETDEV_TX_OK; 1189 } 1190 1191 static void netsec_uninit_pkt_dring(struct netsec_priv *priv, int id) 1192 { 1193 struct netsec_desc_ring *dring = &priv->desc_ring[id]; 1194 struct netsec_desc *desc; 1195 u16 idx; 1196 1197 if (!dring->vaddr || !dring->desc) 1198 return; 1199 for (idx = 0; idx < DESC_NUM; idx++) { 1200 desc = &dring->desc[idx]; 1201 if (!desc->addr) 1202 continue; 1203 1204 if (id == NETSEC_RING_RX) { 1205 struct page *page = virt_to_page(desc->addr); 1206 1207 page_pool_put_page(dring->page_pool, page, false); 1208 } else if (id == NETSEC_RING_TX) { 1209 dma_unmap_single(priv->dev, desc->dma_addr, desc->len, 1210 DMA_TO_DEVICE); 1211 dev_kfree_skb(desc->skb); 1212 } 1213 } 1214 1215 /* Rx is currently using page_pool */ 1216 if (id == NETSEC_RING_RX) { 1217 if (xdp_rxq_info_is_reg(&dring->xdp_rxq)) 1218 xdp_rxq_info_unreg(&dring->xdp_rxq); 1219 page_pool_destroy(dring->page_pool); 1220 } 1221 1222 memset(dring->desc, 0, sizeof(struct netsec_desc) * DESC_NUM); 1223 memset(dring->vaddr, 0, DESC_SZ * DESC_NUM); 1224 1225 dring->head = 0; 1226 dring->tail = 0; 1227 1228 if (id == NETSEC_RING_TX) 1229 netdev_reset_queue(priv->ndev); 1230 } 1231 1232 static void netsec_free_dring(struct netsec_priv *priv, int id) 1233 { 1234 struct netsec_desc_ring *dring = &priv->desc_ring[id]; 1235 1236 if (dring->vaddr) { 1237 dma_free_coherent(priv->dev, DESC_SZ * DESC_NUM, 1238 dring->vaddr, dring->desc_dma); 1239 dring->vaddr = NULL; 1240 } 1241 1242 kfree(dring->desc); 1243 dring->desc = NULL; 1244 } 1245 1246 static int netsec_alloc_dring(struct netsec_priv *priv, enum ring_id id) 1247 { 1248 struct netsec_desc_ring *dring = &priv->desc_ring[id]; 1249 1250 dring->vaddr = dma_alloc_coherent(priv->dev, DESC_SZ * DESC_NUM, 1251 &dring->desc_dma, GFP_KERNEL); 1252 if (!dring->vaddr) 1253 goto err; 1254 1255 dring->desc = kcalloc(DESC_NUM, sizeof(*dring->desc), GFP_KERNEL); 1256 if (!dring->desc) 1257 goto err; 1258 1259 return 0; 1260 err: 1261 netsec_free_dring(priv, id); 1262 1263 return -ENOMEM; 1264 } 1265 1266 static void netsec_setup_tx_dring(struct netsec_priv *priv) 1267 { 1268 struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_TX]; 1269 struct bpf_prog *xdp_prog = READ_ONCE(priv->xdp_prog); 1270 int i; 1271 1272 for (i = 0; i < DESC_NUM; i++) { 1273 struct netsec_de *de; 1274 1275 de = dring->vaddr + (DESC_SZ * i); 1276 /* de->attr is not going to be accessed by the NIC 1277 * until netsec_set_tx_de() is called. 1278 * No need for a dma_wmb() here 1279 */ 1280 de->attr = 1U << NETSEC_TX_SHIFT_OWN_FIELD; 1281 } 1282 1283 if (xdp_prog) 1284 dring->is_xdp = true; 1285 else 1286 dring->is_xdp = false; 1287 1288 } 1289 1290 static int netsec_setup_rx_dring(struct netsec_priv *priv) 1291 { 1292 struct netsec_desc_ring *dring = &priv->desc_ring[NETSEC_RING_RX]; 1293 struct bpf_prog *xdp_prog = READ_ONCE(priv->xdp_prog); 1294 struct page_pool_params pp_params = { 0 }; 1295 int i, err; 1296 1297 pp_params.order = 0; 1298 /* internal DMA mapping in page_pool */ 1299 pp_params.flags = PP_FLAG_DMA_MAP; 1300 pp_params.pool_size = DESC_NUM; 1301 pp_params.nid = cpu_to_node(0); 1302 pp_params.dev = priv->dev; 1303 pp_params.dma_dir = xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 1304 1305 dring->page_pool = page_pool_create(&pp_params); 1306 if (IS_ERR(dring->page_pool)) { 1307 err = PTR_ERR(dring->page_pool); 1308 dring->page_pool = NULL; 1309 goto err_out; 1310 } 1311 1312 err = xdp_rxq_info_reg(&dring->xdp_rxq, priv->ndev, 0); 1313 if (err) 1314 goto err_out; 1315 1316 err = xdp_rxq_info_reg_mem_model(&dring->xdp_rxq, MEM_TYPE_PAGE_POOL, 1317 dring->page_pool); 1318 if (err) 1319 goto err_out; 1320 1321 for (i = 0; i < DESC_NUM; i++) { 1322 struct netsec_desc *desc = &dring->desc[i]; 1323 dma_addr_t dma_handle; 1324 void *buf; 1325 u16 len; 1326 1327 buf = netsec_alloc_rx_data(priv, &dma_handle, &len); 1328 1329 if (!buf) { 1330 err = -ENOMEM; 1331 goto err_out; 1332 } 1333 desc->dma_addr = dma_handle; 1334 desc->addr = buf; 1335 desc->len = len; 1336 } 1337 1338 netsec_rx_fill(priv, 0, DESC_NUM); 1339 1340 return 0; 1341 1342 err_out: 1343 netsec_uninit_pkt_dring(priv, NETSEC_RING_RX); 1344 return err; 1345 } 1346 1347 static int netsec_netdev_load_ucode_region(struct netsec_priv *priv, u32 reg, 1348 u32 addr_h, u32 addr_l, u32 size) 1349 { 1350 u64 base = (u64)addr_h << 32 | addr_l; 1351 void __iomem *ucode; 1352 u32 i; 1353 1354 ucode = ioremap(base, size * sizeof(u32)); 1355 if (!ucode) 1356 return -ENOMEM; 1357 1358 for (i = 0; i < size; i++) 1359 netsec_write(priv, reg, readl(ucode + i * 4)); 1360 1361 iounmap(ucode); 1362 return 0; 1363 } 1364 1365 static int netsec_netdev_load_microcode(struct netsec_priv *priv) 1366 { 1367 u32 addr_h, addr_l, size; 1368 int err; 1369 1370 addr_h = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_ADDRESS_H); 1371 addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_ADDRESS_L); 1372 size = readl(priv->eeprom_base + NETSEC_EEPROM_HM_ME_SIZE); 1373 err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_DMAC_HM_CMD_BUF, 1374 addr_h, addr_l, size); 1375 if (err) 1376 return err; 1377 1378 addr_h = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_ADDRESS_H); 1379 addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_ADDRESS_L); 1380 size = readl(priv->eeprom_base + NETSEC_EEPROM_MH_ME_SIZE); 1381 err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_DMAC_MH_CMD_BUF, 1382 addr_h, addr_l, size); 1383 if (err) 1384 return err; 1385 1386 addr_h = 0; 1387 addr_l = readl(priv->eeprom_base + NETSEC_EEPROM_PKT_ME_ADDRESS); 1388 size = readl(priv->eeprom_base + NETSEC_EEPROM_PKT_ME_SIZE); 1389 err = netsec_netdev_load_ucode_region(priv, NETSEC_REG_PKT_CMD_BUF, 1390 addr_h, addr_l, size); 1391 if (err) 1392 return err; 1393 1394 return 0; 1395 } 1396 1397 static int netsec_reset_hardware(struct netsec_priv *priv, 1398 bool load_ucode) 1399 { 1400 u32 value; 1401 int err; 1402 1403 /* stop DMA engines */ 1404 if (!netsec_read(priv, NETSEC_REG_ADDR_DIS_CORE)) { 1405 netsec_write(priv, NETSEC_REG_DMA_HM_CTRL, 1406 NETSEC_DMA_CTRL_REG_STOP); 1407 netsec_write(priv, NETSEC_REG_DMA_MH_CTRL, 1408 NETSEC_DMA_CTRL_REG_STOP); 1409 1410 while (netsec_read(priv, NETSEC_REG_DMA_HM_CTRL) & 1411 NETSEC_DMA_CTRL_REG_STOP) 1412 cpu_relax(); 1413 1414 while (netsec_read(priv, NETSEC_REG_DMA_MH_CTRL) & 1415 NETSEC_DMA_CTRL_REG_STOP) 1416 cpu_relax(); 1417 } 1418 1419 netsec_write(priv, NETSEC_REG_SOFT_RST, NETSEC_SOFT_RST_REG_RESET); 1420 netsec_write(priv, NETSEC_REG_SOFT_RST, NETSEC_SOFT_RST_REG_RUN); 1421 netsec_write(priv, NETSEC_REG_COM_INIT, NETSEC_COM_INIT_REG_ALL); 1422 1423 while (netsec_read(priv, NETSEC_REG_COM_INIT) != 0) 1424 cpu_relax(); 1425 1426 /* set desc_start addr */ 1427 netsec_write(priv, NETSEC_REG_NRM_RX_DESC_START_UP, 1428 upper_32_bits(priv->desc_ring[NETSEC_RING_RX].desc_dma)); 1429 netsec_write(priv, NETSEC_REG_NRM_RX_DESC_START_LW, 1430 lower_32_bits(priv->desc_ring[NETSEC_RING_RX].desc_dma)); 1431 1432 netsec_write(priv, NETSEC_REG_NRM_TX_DESC_START_UP, 1433 upper_32_bits(priv->desc_ring[NETSEC_RING_TX].desc_dma)); 1434 netsec_write(priv, NETSEC_REG_NRM_TX_DESC_START_LW, 1435 lower_32_bits(priv->desc_ring[NETSEC_RING_TX].desc_dma)); 1436 1437 /* set normal tx dring ring config */ 1438 netsec_write(priv, NETSEC_REG_NRM_TX_CONFIG, 1439 1 << NETSEC_REG_DESC_ENDIAN); 1440 netsec_write(priv, NETSEC_REG_NRM_RX_CONFIG, 1441 1 << NETSEC_REG_DESC_ENDIAN); 1442 1443 if (load_ucode) { 1444 err = netsec_netdev_load_microcode(priv); 1445 if (err) { 1446 netif_err(priv, probe, priv->ndev, 1447 "%s: failed to load microcode (%d)\n", 1448 __func__, err); 1449 return err; 1450 } 1451 } 1452 1453 /* start DMA engines */ 1454 netsec_write(priv, NETSEC_REG_DMA_TMR_CTRL, priv->freq / 1000000 - 1); 1455 netsec_write(priv, NETSEC_REG_ADDR_DIS_CORE, 0); 1456 1457 usleep_range(1000, 2000); 1458 1459 if (!(netsec_read(priv, NETSEC_REG_TOP_STATUS) & 1460 NETSEC_TOP_IRQ_REG_CODE_LOAD_END)) { 1461 netif_err(priv, probe, priv->ndev, 1462 "microengine start failed\n"); 1463 return -ENXIO; 1464 } 1465 netsec_write(priv, NETSEC_REG_TOP_STATUS, 1466 NETSEC_TOP_IRQ_REG_CODE_LOAD_END); 1467 1468 value = NETSEC_PKT_CTRL_REG_MODE_NRM; 1469 if (priv->ndev->mtu > ETH_DATA_LEN) 1470 value |= NETSEC_PKT_CTRL_REG_EN_JUMBO; 1471 1472 /* change to normal mode */ 1473 netsec_write(priv, NETSEC_REG_DMA_MH_CTRL, MH_CTRL__MODE_TRANS); 1474 netsec_write(priv, NETSEC_REG_PKT_CTRL, value); 1475 1476 while ((netsec_read(priv, NETSEC_REG_MODE_TRANS_COMP_STATUS) & 1477 NETSEC_MODE_TRANS_COMP_IRQ_T2N) == 0) 1478 cpu_relax(); 1479 1480 /* clear any pending EMPTY/ERR irq status */ 1481 netsec_write(priv, NETSEC_REG_NRM_TX_STATUS, ~0); 1482 1483 /* Disable TX & RX intr */ 1484 netsec_write(priv, NETSEC_REG_INTEN_CLR, ~0); 1485 1486 return 0; 1487 } 1488 1489 static int netsec_start_gmac(struct netsec_priv *priv) 1490 { 1491 struct phy_device *phydev = priv->ndev->phydev; 1492 u32 value = 0; 1493 int ret; 1494 1495 if (phydev->speed != SPEED_1000) 1496 value = (NETSEC_GMAC_MCR_REG_CST | 1497 NETSEC_GMAC_MCR_REG_HALF_DUPLEX_COMMON); 1498 1499 if (netsec_mac_write(priv, GMAC_REG_MCR, value)) 1500 return -ETIMEDOUT; 1501 if (netsec_mac_write(priv, GMAC_REG_BMR, 1502 NETSEC_GMAC_BMR_REG_RESET)) 1503 return -ETIMEDOUT; 1504 1505 /* Wait soft reset */ 1506 usleep_range(1000, 5000); 1507 1508 ret = netsec_mac_read(priv, GMAC_REG_BMR, &value); 1509 if (ret) 1510 return ret; 1511 if (value & NETSEC_GMAC_BMR_REG_SWR) 1512 return -EAGAIN; 1513 1514 netsec_write(priv, MAC_REG_DESC_SOFT_RST, 1); 1515 if (netsec_wait_while_busy(priv, MAC_REG_DESC_SOFT_RST, 1)) 1516 return -ETIMEDOUT; 1517 1518 netsec_write(priv, MAC_REG_DESC_INIT, 1); 1519 if (netsec_wait_while_busy(priv, MAC_REG_DESC_INIT, 1)) 1520 return -ETIMEDOUT; 1521 1522 if (netsec_mac_write(priv, GMAC_REG_BMR, 1523 NETSEC_GMAC_BMR_REG_COMMON)) 1524 return -ETIMEDOUT; 1525 if (netsec_mac_write(priv, GMAC_REG_RDLAR, 1526 NETSEC_GMAC_RDLAR_REG_COMMON)) 1527 return -ETIMEDOUT; 1528 if (netsec_mac_write(priv, GMAC_REG_TDLAR, 1529 NETSEC_GMAC_TDLAR_REG_COMMON)) 1530 return -ETIMEDOUT; 1531 if (netsec_mac_write(priv, GMAC_REG_MFFR, 0x80000001)) 1532 return -ETIMEDOUT; 1533 1534 ret = netsec_mac_update_to_phy_state(priv); 1535 if (ret) 1536 return ret; 1537 1538 ret = netsec_mac_read(priv, GMAC_REG_OMR, &value); 1539 if (ret) 1540 return ret; 1541 1542 value |= NETSEC_GMAC_OMR_REG_SR; 1543 value |= NETSEC_GMAC_OMR_REG_ST; 1544 1545 netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_CLR, ~0); 1546 netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_CLR, ~0); 1547 1548 netsec_et_set_coalesce(priv->ndev, &priv->et_coalesce); 1549 1550 if (netsec_mac_write(priv, GMAC_REG_OMR, value)) 1551 return -ETIMEDOUT; 1552 1553 return 0; 1554 } 1555 1556 static int netsec_stop_gmac(struct netsec_priv *priv) 1557 { 1558 u32 value; 1559 int ret; 1560 1561 ret = netsec_mac_read(priv, GMAC_REG_OMR, &value); 1562 if (ret) 1563 return ret; 1564 value &= ~NETSEC_GMAC_OMR_REG_SR; 1565 value &= ~NETSEC_GMAC_OMR_REG_ST; 1566 1567 /* disable all interrupts */ 1568 netsec_write(priv, NETSEC_REG_NRM_RX_INTEN_CLR, ~0); 1569 netsec_write(priv, NETSEC_REG_NRM_TX_INTEN_CLR, ~0); 1570 1571 return netsec_mac_write(priv, GMAC_REG_OMR, value); 1572 } 1573 1574 static void netsec_phy_adjust_link(struct net_device *ndev) 1575 { 1576 struct netsec_priv *priv = netdev_priv(ndev); 1577 1578 if (ndev->phydev->link) 1579 netsec_start_gmac(priv); 1580 else 1581 netsec_stop_gmac(priv); 1582 1583 phy_print_status(ndev->phydev); 1584 } 1585 1586 static irqreturn_t netsec_irq_handler(int irq, void *dev_id) 1587 { 1588 struct netsec_priv *priv = dev_id; 1589 u32 val, status = netsec_read(priv, NETSEC_REG_TOP_STATUS); 1590 unsigned long flags; 1591 1592 /* Disable interrupts */ 1593 if (status & NETSEC_IRQ_TX) { 1594 val = netsec_read(priv, NETSEC_REG_NRM_TX_STATUS); 1595 netsec_write(priv, NETSEC_REG_NRM_TX_STATUS, val); 1596 } 1597 if (status & NETSEC_IRQ_RX) { 1598 val = netsec_read(priv, NETSEC_REG_NRM_RX_STATUS); 1599 netsec_write(priv, NETSEC_REG_NRM_RX_STATUS, val); 1600 } 1601 1602 spin_lock_irqsave(&priv->reglock, flags); 1603 netsec_write(priv, NETSEC_REG_INTEN_CLR, NETSEC_IRQ_RX | NETSEC_IRQ_TX); 1604 spin_unlock_irqrestore(&priv->reglock, flags); 1605 1606 napi_schedule(&priv->napi); 1607 1608 return IRQ_HANDLED; 1609 } 1610 1611 static int netsec_netdev_open(struct net_device *ndev) 1612 { 1613 struct netsec_priv *priv = netdev_priv(ndev); 1614 int ret; 1615 1616 pm_runtime_get_sync(priv->dev); 1617 1618 netsec_setup_tx_dring(priv); 1619 ret = netsec_setup_rx_dring(priv); 1620 if (ret) { 1621 netif_err(priv, probe, priv->ndev, 1622 "%s: fail setup ring\n", __func__); 1623 goto err1; 1624 } 1625 1626 ret = request_irq(priv->ndev->irq, netsec_irq_handler, 1627 IRQF_SHARED, "netsec", priv); 1628 if (ret) { 1629 netif_err(priv, drv, priv->ndev, "request_irq failed\n"); 1630 goto err2; 1631 } 1632 1633 if (dev_of_node(priv->dev)) { 1634 if (!of_phy_connect(priv->ndev, priv->phy_np, 1635 netsec_phy_adjust_link, 0, 1636 priv->phy_interface)) { 1637 netif_err(priv, link, priv->ndev, "missing PHY\n"); 1638 ret = -ENODEV; 1639 goto err3; 1640 } 1641 } else { 1642 ret = phy_connect_direct(priv->ndev, priv->phydev, 1643 netsec_phy_adjust_link, 1644 priv->phy_interface); 1645 if (ret) { 1646 netif_err(priv, link, priv->ndev, 1647 "phy_connect_direct() failed (%d)\n", ret); 1648 goto err3; 1649 } 1650 } 1651 1652 phy_start(ndev->phydev); 1653 1654 netsec_start_gmac(priv); 1655 napi_enable(&priv->napi); 1656 netif_start_queue(ndev); 1657 1658 /* Enable TX+RX intr. */ 1659 netsec_write(priv, NETSEC_REG_INTEN_SET, NETSEC_IRQ_RX | NETSEC_IRQ_TX); 1660 1661 return 0; 1662 err3: 1663 free_irq(priv->ndev->irq, priv); 1664 err2: 1665 netsec_uninit_pkt_dring(priv, NETSEC_RING_RX); 1666 err1: 1667 pm_runtime_put_sync(priv->dev); 1668 return ret; 1669 } 1670 1671 static int netsec_netdev_stop(struct net_device *ndev) 1672 { 1673 int ret; 1674 struct netsec_priv *priv = netdev_priv(ndev); 1675 1676 netif_stop_queue(priv->ndev); 1677 dma_wmb(); 1678 1679 napi_disable(&priv->napi); 1680 1681 netsec_write(priv, NETSEC_REG_INTEN_CLR, ~0); 1682 netsec_stop_gmac(priv); 1683 1684 free_irq(priv->ndev->irq, priv); 1685 1686 netsec_uninit_pkt_dring(priv, NETSEC_RING_TX); 1687 netsec_uninit_pkt_dring(priv, NETSEC_RING_RX); 1688 1689 phy_stop(ndev->phydev); 1690 phy_disconnect(ndev->phydev); 1691 1692 ret = netsec_reset_hardware(priv, false); 1693 1694 pm_runtime_put_sync(priv->dev); 1695 1696 return ret; 1697 } 1698 1699 static int netsec_netdev_init(struct net_device *ndev) 1700 { 1701 struct netsec_priv *priv = netdev_priv(ndev); 1702 int ret; 1703 u16 data; 1704 1705 BUILD_BUG_ON_NOT_POWER_OF_2(DESC_NUM); 1706 1707 ret = netsec_alloc_dring(priv, NETSEC_RING_TX); 1708 if (ret) 1709 return ret; 1710 1711 ret = netsec_alloc_dring(priv, NETSEC_RING_RX); 1712 if (ret) 1713 goto err1; 1714 1715 /* set phy power down */ 1716 data = netsec_phy_read(priv->mii_bus, priv->phy_addr, MII_BMCR) | 1717 BMCR_PDOWN; 1718 netsec_phy_write(priv->mii_bus, priv->phy_addr, MII_BMCR, data); 1719 1720 ret = netsec_reset_hardware(priv, true); 1721 if (ret) 1722 goto err2; 1723 1724 spin_lock_init(&priv->desc_ring[NETSEC_RING_TX].lock); 1725 spin_lock_init(&priv->desc_ring[NETSEC_RING_RX].lock); 1726 1727 return 0; 1728 err2: 1729 netsec_free_dring(priv, NETSEC_RING_RX); 1730 err1: 1731 netsec_free_dring(priv, NETSEC_RING_TX); 1732 return ret; 1733 } 1734 1735 static void netsec_netdev_uninit(struct net_device *ndev) 1736 { 1737 struct netsec_priv *priv = netdev_priv(ndev); 1738 1739 netsec_free_dring(priv, NETSEC_RING_RX); 1740 netsec_free_dring(priv, NETSEC_RING_TX); 1741 } 1742 1743 static int netsec_netdev_set_features(struct net_device *ndev, 1744 netdev_features_t features) 1745 { 1746 struct netsec_priv *priv = netdev_priv(ndev); 1747 1748 priv->rx_cksum_offload_flag = !!(features & NETIF_F_RXCSUM); 1749 1750 return 0; 1751 } 1752 1753 static int netsec_netdev_ioctl(struct net_device *ndev, struct ifreq *ifr, 1754 int cmd) 1755 { 1756 return phy_mii_ioctl(ndev->phydev, ifr, cmd); 1757 } 1758 1759 static int netsec_xdp_xmit(struct net_device *ndev, int n, 1760 struct xdp_frame **frames, u32 flags) 1761 { 1762 struct netsec_priv *priv = netdev_priv(ndev); 1763 struct netsec_desc_ring *tx_ring = &priv->desc_ring[NETSEC_RING_TX]; 1764 int drops = 0; 1765 int i; 1766 1767 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK)) 1768 return -EINVAL; 1769 1770 spin_lock(&tx_ring->lock); 1771 for (i = 0; i < n; i++) { 1772 struct xdp_frame *xdpf = frames[i]; 1773 int err; 1774 1775 err = netsec_xdp_queue_one(priv, xdpf, true); 1776 if (err != NETSEC_XDP_TX) { 1777 xdp_return_frame_rx_napi(xdpf); 1778 drops++; 1779 } else { 1780 tx_ring->xdp_xmit++; 1781 } 1782 } 1783 spin_unlock(&tx_ring->lock); 1784 1785 if (unlikely(flags & XDP_XMIT_FLUSH)) { 1786 netsec_xdp_ring_tx_db(priv, tx_ring->xdp_xmit); 1787 tx_ring->xdp_xmit = 0; 1788 } 1789 1790 return n - drops; 1791 } 1792 1793 static int netsec_xdp_setup(struct netsec_priv *priv, struct bpf_prog *prog, 1794 struct netlink_ext_ack *extack) 1795 { 1796 struct net_device *dev = priv->ndev; 1797 struct bpf_prog *old_prog; 1798 1799 /* For now just support only the usual MTU sized frames */ 1800 if (prog && dev->mtu > 1500) { 1801 NL_SET_ERR_MSG_MOD(extack, "Jumbo frames not supported on XDP"); 1802 return -EOPNOTSUPP; 1803 } 1804 1805 if (netif_running(dev)) 1806 netsec_netdev_stop(dev); 1807 1808 /* Detach old prog, if any */ 1809 old_prog = xchg(&priv->xdp_prog, prog); 1810 if (old_prog) 1811 bpf_prog_put(old_prog); 1812 1813 if (netif_running(dev)) 1814 netsec_netdev_open(dev); 1815 1816 return 0; 1817 } 1818 1819 static int netsec_xdp(struct net_device *ndev, struct netdev_bpf *xdp) 1820 { 1821 struct netsec_priv *priv = netdev_priv(ndev); 1822 1823 switch (xdp->command) { 1824 case XDP_SETUP_PROG: 1825 return netsec_xdp_setup(priv, xdp->prog, xdp->extack); 1826 case XDP_QUERY_PROG: 1827 xdp->prog_id = priv->xdp_prog ? priv->xdp_prog->aux->id : 0; 1828 return 0; 1829 default: 1830 return -EINVAL; 1831 } 1832 } 1833 1834 static const struct net_device_ops netsec_netdev_ops = { 1835 .ndo_init = netsec_netdev_init, 1836 .ndo_uninit = netsec_netdev_uninit, 1837 .ndo_open = netsec_netdev_open, 1838 .ndo_stop = netsec_netdev_stop, 1839 .ndo_start_xmit = netsec_netdev_start_xmit, 1840 .ndo_set_features = netsec_netdev_set_features, 1841 .ndo_set_mac_address = eth_mac_addr, 1842 .ndo_validate_addr = eth_validate_addr, 1843 .ndo_do_ioctl = netsec_netdev_ioctl, 1844 .ndo_xdp_xmit = netsec_xdp_xmit, 1845 .ndo_bpf = netsec_xdp, 1846 }; 1847 1848 static int netsec_of_probe(struct platform_device *pdev, 1849 struct netsec_priv *priv, u32 *phy_addr) 1850 { 1851 priv->phy_np = of_parse_phandle(pdev->dev.of_node, "phy-handle", 0); 1852 if (!priv->phy_np) { 1853 dev_err(&pdev->dev, "missing required property 'phy-handle'\n"); 1854 return -EINVAL; 1855 } 1856 1857 *phy_addr = of_mdio_parse_addr(&pdev->dev, priv->phy_np); 1858 1859 priv->clk = devm_clk_get(&pdev->dev, NULL); /* get by 'phy_ref_clk' */ 1860 if (IS_ERR(priv->clk)) { 1861 dev_err(&pdev->dev, "phy_ref_clk not found\n"); 1862 return PTR_ERR(priv->clk); 1863 } 1864 priv->freq = clk_get_rate(priv->clk); 1865 1866 return 0; 1867 } 1868 1869 static int netsec_acpi_probe(struct platform_device *pdev, 1870 struct netsec_priv *priv, u32 *phy_addr) 1871 { 1872 int ret; 1873 1874 if (!IS_ENABLED(CONFIG_ACPI)) 1875 return -ENODEV; 1876 1877 ret = device_property_read_u32(&pdev->dev, "phy-channel", phy_addr); 1878 if (ret) { 1879 dev_err(&pdev->dev, 1880 "missing required property 'phy-channel'\n"); 1881 return ret; 1882 } 1883 1884 ret = device_property_read_u32(&pdev->dev, 1885 "socionext,phy-clock-frequency", 1886 &priv->freq); 1887 if (ret) 1888 dev_err(&pdev->dev, 1889 "missing required property 'socionext,phy-clock-frequency'\n"); 1890 return ret; 1891 } 1892 1893 static void netsec_unregister_mdio(struct netsec_priv *priv) 1894 { 1895 struct phy_device *phydev = priv->phydev; 1896 1897 if (!dev_of_node(priv->dev) && phydev) { 1898 phy_device_remove(phydev); 1899 phy_device_free(phydev); 1900 } 1901 1902 mdiobus_unregister(priv->mii_bus); 1903 } 1904 1905 static int netsec_register_mdio(struct netsec_priv *priv, u32 phy_addr) 1906 { 1907 struct mii_bus *bus; 1908 int ret; 1909 1910 bus = devm_mdiobus_alloc(priv->dev); 1911 if (!bus) 1912 return -ENOMEM; 1913 1914 snprintf(bus->id, MII_BUS_ID_SIZE, "%s", dev_name(priv->dev)); 1915 bus->priv = priv; 1916 bus->name = "SNI NETSEC MDIO"; 1917 bus->read = netsec_phy_read; 1918 bus->write = netsec_phy_write; 1919 bus->parent = priv->dev; 1920 priv->mii_bus = bus; 1921 1922 if (dev_of_node(priv->dev)) { 1923 struct device_node *mdio_node, *parent = dev_of_node(priv->dev); 1924 1925 mdio_node = of_get_child_by_name(parent, "mdio"); 1926 if (mdio_node) { 1927 parent = mdio_node; 1928 } else { 1929 /* older f/w doesn't populate the mdio subnode, 1930 * allow relaxed upgrade of f/w in due time. 1931 */ 1932 dev_info(priv->dev, "Upgrade f/w for mdio subnode!\n"); 1933 } 1934 1935 ret = of_mdiobus_register(bus, parent); 1936 of_node_put(mdio_node); 1937 1938 if (ret) { 1939 dev_err(priv->dev, "mdiobus register err(%d)\n", ret); 1940 return ret; 1941 } 1942 } else { 1943 /* Mask out all PHYs from auto probing. */ 1944 bus->phy_mask = ~0; 1945 ret = mdiobus_register(bus); 1946 if (ret) { 1947 dev_err(priv->dev, "mdiobus register err(%d)\n", ret); 1948 return ret; 1949 } 1950 1951 priv->phydev = get_phy_device(bus, phy_addr, false); 1952 if (IS_ERR(priv->phydev)) { 1953 ret = PTR_ERR(priv->phydev); 1954 dev_err(priv->dev, "get_phy_device err(%d)\n", ret); 1955 priv->phydev = NULL; 1956 return -ENODEV; 1957 } 1958 1959 ret = phy_device_register(priv->phydev); 1960 if (ret) { 1961 mdiobus_unregister(bus); 1962 dev_err(priv->dev, 1963 "phy_device_register err(%d)\n", ret); 1964 } 1965 } 1966 1967 return ret; 1968 } 1969 1970 static int netsec_probe(struct platform_device *pdev) 1971 { 1972 struct resource *mmio_res, *eeprom_res, *irq_res; 1973 u8 *mac, macbuf[ETH_ALEN]; 1974 struct netsec_priv *priv; 1975 u32 hw_ver, phy_addr = 0; 1976 struct net_device *ndev; 1977 int ret; 1978 1979 mmio_res = platform_get_resource(pdev, IORESOURCE_MEM, 0); 1980 if (!mmio_res) { 1981 dev_err(&pdev->dev, "No MMIO resource found.\n"); 1982 return -ENODEV; 1983 } 1984 1985 eeprom_res = platform_get_resource(pdev, IORESOURCE_MEM, 1); 1986 if (!eeprom_res) { 1987 dev_info(&pdev->dev, "No EEPROM resource found.\n"); 1988 return -ENODEV; 1989 } 1990 1991 irq_res = platform_get_resource(pdev, IORESOURCE_IRQ, 0); 1992 if (!irq_res) { 1993 dev_err(&pdev->dev, "No IRQ resource found.\n"); 1994 return -ENODEV; 1995 } 1996 1997 ndev = alloc_etherdev(sizeof(*priv)); 1998 if (!ndev) 1999 return -ENOMEM; 2000 2001 priv = netdev_priv(ndev); 2002 2003 spin_lock_init(&priv->reglock); 2004 SET_NETDEV_DEV(ndev, &pdev->dev); 2005 platform_set_drvdata(pdev, priv); 2006 ndev->irq = irq_res->start; 2007 priv->dev = &pdev->dev; 2008 priv->ndev = ndev; 2009 2010 priv->msg_enable = NETIF_MSG_TX_ERR | NETIF_MSG_HW | NETIF_MSG_DRV | 2011 NETIF_MSG_LINK | NETIF_MSG_PROBE; 2012 2013 priv->phy_interface = device_get_phy_mode(&pdev->dev); 2014 if (priv->phy_interface < 0) { 2015 dev_err(&pdev->dev, "missing required property 'phy-mode'\n"); 2016 ret = -ENODEV; 2017 goto free_ndev; 2018 } 2019 2020 priv->ioaddr = devm_ioremap(&pdev->dev, mmio_res->start, 2021 resource_size(mmio_res)); 2022 if (!priv->ioaddr) { 2023 dev_err(&pdev->dev, "devm_ioremap() failed\n"); 2024 ret = -ENXIO; 2025 goto free_ndev; 2026 } 2027 2028 priv->eeprom_base = devm_ioremap(&pdev->dev, eeprom_res->start, 2029 resource_size(eeprom_res)); 2030 if (!priv->eeprom_base) { 2031 dev_err(&pdev->dev, "devm_ioremap() failed for EEPROM\n"); 2032 ret = -ENXIO; 2033 goto free_ndev; 2034 } 2035 2036 mac = device_get_mac_address(&pdev->dev, macbuf, sizeof(macbuf)); 2037 if (mac) 2038 ether_addr_copy(ndev->dev_addr, mac); 2039 2040 if (priv->eeprom_base && 2041 (!mac || !is_valid_ether_addr(ndev->dev_addr))) { 2042 void __iomem *macp = priv->eeprom_base + 2043 NETSEC_EEPROM_MAC_ADDRESS; 2044 2045 ndev->dev_addr[0] = readb(macp + 3); 2046 ndev->dev_addr[1] = readb(macp + 2); 2047 ndev->dev_addr[2] = readb(macp + 1); 2048 ndev->dev_addr[3] = readb(macp + 0); 2049 ndev->dev_addr[4] = readb(macp + 7); 2050 ndev->dev_addr[5] = readb(macp + 6); 2051 } 2052 2053 if (!is_valid_ether_addr(ndev->dev_addr)) { 2054 dev_warn(&pdev->dev, "No MAC address found, using random\n"); 2055 eth_hw_addr_random(ndev); 2056 } 2057 2058 if (dev_of_node(&pdev->dev)) 2059 ret = netsec_of_probe(pdev, priv, &phy_addr); 2060 else 2061 ret = netsec_acpi_probe(pdev, priv, &phy_addr); 2062 if (ret) 2063 goto free_ndev; 2064 2065 priv->phy_addr = phy_addr; 2066 2067 if (!priv->freq) { 2068 dev_err(&pdev->dev, "missing PHY reference clock frequency\n"); 2069 ret = -ENODEV; 2070 goto free_ndev; 2071 } 2072 2073 /* default for throughput */ 2074 priv->et_coalesce.rx_coalesce_usecs = 500; 2075 priv->et_coalesce.rx_max_coalesced_frames = 8; 2076 priv->et_coalesce.tx_coalesce_usecs = 500; 2077 priv->et_coalesce.tx_max_coalesced_frames = 8; 2078 2079 ret = device_property_read_u32(&pdev->dev, "max-frame-size", 2080 &ndev->max_mtu); 2081 if (ret < 0) 2082 ndev->max_mtu = ETH_DATA_LEN; 2083 2084 /* runtime_pm coverage just for probe, open/close also cover it */ 2085 pm_runtime_enable(&pdev->dev); 2086 pm_runtime_get_sync(&pdev->dev); 2087 2088 hw_ver = netsec_read(priv, NETSEC_REG_F_TAIKI_VER); 2089 /* this driver only supports F_TAIKI style NETSEC */ 2090 if (NETSEC_F_NETSEC_VER_MAJOR_NUM(hw_ver) != 2091 NETSEC_F_NETSEC_VER_MAJOR_NUM(NETSEC_REG_NETSEC_VER_F_TAIKI)) { 2092 ret = -ENODEV; 2093 goto pm_disable; 2094 } 2095 2096 dev_info(&pdev->dev, "hardware revision %d.%d\n", 2097 hw_ver >> 16, hw_ver & 0xffff); 2098 2099 netif_napi_add(ndev, &priv->napi, netsec_napi_poll, NAPI_POLL_WEIGHT); 2100 2101 ndev->netdev_ops = &netsec_netdev_ops; 2102 ndev->ethtool_ops = &netsec_ethtool_ops; 2103 2104 ndev->features |= NETIF_F_HIGHDMA | NETIF_F_RXCSUM | NETIF_F_GSO | 2105 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM; 2106 ndev->hw_features = ndev->features; 2107 2108 priv->rx_cksum_offload_flag = true; 2109 2110 ret = netsec_register_mdio(priv, phy_addr); 2111 if (ret) 2112 goto unreg_napi; 2113 2114 if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(40))) 2115 dev_warn(&pdev->dev, "Failed to set DMA mask\n"); 2116 2117 ret = register_netdev(ndev); 2118 if (ret) { 2119 netif_err(priv, probe, ndev, "register_netdev() failed\n"); 2120 goto unreg_mii; 2121 } 2122 2123 pm_runtime_put_sync(&pdev->dev); 2124 return 0; 2125 2126 unreg_mii: 2127 netsec_unregister_mdio(priv); 2128 unreg_napi: 2129 netif_napi_del(&priv->napi); 2130 pm_disable: 2131 pm_runtime_put_sync(&pdev->dev); 2132 pm_runtime_disable(&pdev->dev); 2133 free_ndev: 2134 free_netdev(ndev); 2135 dev_err(&pdev->dev, "init failed\n"); 2136 2137 return ret; 2138 } 2139 2140 static int netsec_remove(struct platform_device *pdev) 2141 { 2142 struct netsec_priv *priv = platform_get_drvdata(pdev); 2143 2144 unregister_netdev(priv->ndev); 2145 2146 netsec_unregister_mdio(priv); 2147 2148 netif_napi_del(&priv->napi); 2149 2150 pm_runtime_disable(&pdev->dev); 2151 free_netdev(priv->ndev); 2152 2153 return 0; 2154 } 2155 2156 #ifdef CONFIG_PM 2157 static int netsec_runtime_suspend(struct device *dev) 2158 { 2159 struct netsec_priv *priv = dev_get_drvdata(dev); 2160 2161 netsec_write(priv, NETSEC_REG_CLK_EN, 0); 2162 2163 clk_disable_unprepare(priv->clk); 2164 2165 return 0; 2166 } 2167 2168 static int netsec_runtime_resume(struct device *dev) 2169 { 2170 struct netsec_priv *priv = dev_get_drvdata(dev); 2171 2172 clk_prepare_enable(priv->clk); 2173 2174 netsec_write(priv, NETSEC_REG_CLK_EN, NETSEC_CLK_EN_REG_DOM_D | 2175 NETSEC_CLK_EN_REG_DOM_C | 2176 NETSEC_CLK_EN_REG_DOM_G); 2177 return 0; 2178 } 2179 #endif 2180 2181 static const struct dev_pm_ops netsec_pm_ops = { 2182 SET_RUNTIME_PM_OPS(netsec_runtime_suspend, netsec_runtime_resume, NULL) 2183 }; 2184 2185 static const struct of_device_id netsec_dt_ids[] = { 2186 { .compatible = "socionext,synquacer-netsec" }, 2187 { } 2188 }; 2189 MODULE_DEVICE_TABLE(of, netsec_dt_ids); 2190 2191 #ifdef CONFIG_ACPI 2192 static const struct acpi_device_id netsec_acpi_ids[] = { 2193 { "SCX0001" }, 2194 { } 2195 }; 2196 MODULE_DEVICE_TABLE(acpi, netsec_acpi_ids); 2197 #endif 2198 2199 static struct platform_driver netsec_driver = { 2200 .probe = netsec_probe, 2201 .remove = netsec_remove, 2202 .driver = { 2203 .name = "netsec", 2204 .pm = &netsec_pm_ops, 2205 .of_match_table = netsec_dt_ids, 2206 .acpi_match_table = ACPI_PTR(netsec_acpi_ids), 2207 }, 2208 }; 2209 module_platform_driver(netsec_driver); 2210 2211 MODULE_AUTHOR("Jassi Brar <jaswinder.singh@linaro.org>"); 2212 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>"); 2213 MODULE_DESCRIPTION("NETSEC Ethernet driver"); 2214 MODULE_LICENSE("GPL"); 2215