1 /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. 2 * 3 * This program is free software; you can redistribute it and/or modify 4 * it under the terms of the GNU General Public License version 2 and 5 * only version 2 as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 */ 12 13 /* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support 14 */ 15 16 #include <linux/tcp.h> 17 #include <linux/ip.h> 18 #include <linux/ipv6.h> 19 #include <linux/crc32.h> 20 #include <linux/if_vlan.h> 21 #include <linux/jiffies.h> 22 #include <linux/phy.h> 23 #include <linux/of.h> 24 #include <net/ip6_checksum.h> 25 #include "emac.h" 26 #include "emac-sgmii.h" 27 28 /* EMAC_MAC_CTRL */ 29 #define SINGLE_PAUSE_MODE 0x10000000 30 #define DEBUG_MODE 0x08000000 31 #define BROAD_EN 0x04000000 32 #define MULTI_ALL 0x02000000 33 #define RX_CHKSUM_EN 0x01000000 34 #define HUGE 0x00800000 35 #define SPEED(x) (((x) & 0x3) << 20) 36 #define SPEED_MASK SPEED(0x3) 37 #define SIMR 0x00080000 38 #define TPAUSE 0x00010000 39 #define PROM_MODE 0x00008000 40 #define VLAN_STRIP 0x00004000 41 #define PRLEN_BMSK 0x00003c00 42 #define PRLEN_SHFT 10 43 #define HUGEN 0x00000200 44 #define FLCHK 0x00000100 45 #define PCRCE 0x00000080 46 #define CRCE 0x00000040 47 #define FULLD 0x00000020 48 #define MAC_LP_EN 0x00000010 49 #define RXFC 0x00000008 50 #define TXFC 0x00000004 51 #define RXEN 0x00000002 52 #define TXEN 0x00000001 53 54 /* EMAC_DESC_CTRL_3 */ 55 #define RFD_RING_SIZE_BMSK 0xfff 56 57 /* EMAC_DESC_CTRL_4 */ 58 #define RX_BUFFER_SIZE_BMSK 0xffff 59 60 /* EMAC_DESC_CTRL_6 */ 61 #define RRD_RING_SIZE_BMSK 0xfff 62 63 /* EMAC_DESC_CTRL_9 */ 64 #define TPD_RING_SIZE_BMSK 0xffff 65 66 /* EMAC_TXQ_CTRL_0 */ 67 #define NUM_TXF_BURST_PREF_BMSK 0xffff0000 68 #define NUM_TXF_BURST_PREF_SHFT 16 69 #define LS_8023_SP 0x80 70 #define TXQ_MODE 0x40 71 #define TXQ_EN 0x20 72 #define IP_OP_SP 0x10 73 #define NUM_TPD_BURST_PREF_BMSK 0xf 74 #define NUM_TPD_BURST_PREF_SHFT 0 75 76 /* EMAC_TXQ_CTRL_1 */ 77 #define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK 0x7ff 78 79 /* EMAC_TXQ_CTRL_2 */ 80 #define TXF_HWM_BMSK 0xfff0000 81 #define TXF_LWM_BMSK 0xfff 82 83 /* EMAC_RXQ_CTRL_0 */ 84 #define RXQ_EN BIT(31) 85 #define CUT_THRU_EN BIT(30) 86 #define RSS_HASH_EN BIT(29) 87 #define NUM_RFD_BURST_PREF_BMSK 0x3f00000 88 #define NUM_RFD_BURST_PREF_SHFT 20 89 #define IDT_TABLE_SIZE_BMSK 0x1ff00 90 #define IDT_TABLE_SIZE_SHFT 8 91 #define SP_IPV6 0x80 92 93 /* EMAC_RXQ_CTRL_1 */ 94 #define JUMBO_1KAH_BMSK 0xf000 95 #define JUMBO_1KAH_SHFT 12 96 #define RFD_PREF_LOW_TH 0x10 97 #define RFD_PREF_LOW_THRESHOLD_BMSK 0xfc0 98 #define RFD_PREF_LOW_THRESHOLD_SHFT 6 99 #define RFD_PREF_UP_TH 0x10 100 #define RFD_PREF_UP_THRESHOLD_BMSK 0x3f 101 #define RFD_PREF_UP_THRESHOLD_SHFT 0 102 103 /* EMAC_RXQ_CTRL_2 */ 104 #define RXF_DOF_THRESFHOLD 0x1a0 105 #define RXF_DOF_THRESHOLD_BMSK 0xfff0000 106 #define RXF_DOF_THRESHOLD_SHFT 16 107 #define RXF_UOF_THRESFHOLD 0xbe 108 #define RXF_UOF_THRESHOLD_BMSK 0xfff 109 #define RXF_UOF_THRESHOLD_SHFT 0 110 111 /* EMAC_RXQ_CTRL_3 */ 112 #define RXD_TIMER_BMSK 0xffff0000 113 #define RXD_THRESHOLD_BMSK 0xfff 114 #define RXD_THRESHOLD_SHFT 0 115 116 /* EMAC_DMA_CTRL */ 117 #define DMAW_DLY_CNT_BMSK 0xf0000 118 #define DMAW_DLY_CNT_SHFT 16 119 #define DMAR_DLY_CNT_BMSK 0xf800 120 #define DMAR_DLY_CNT_SHFT 11 121 #define DMAR_REQ_PRI 0x400 122 #define REGWRBLEN_BMSK 0x380 123 #define REGWRBLEN_SHFT 7 124 #define REGRDBLEN_BMSK 0x70 125 #define REGRDBLEN_SHFT 4 126 #define OUT_ORDER_MODE 0x4 127 #define ENH_ORDER_MODE 0x2 128 #define IN_ORDER_MODE 0x1 129 130 /* EMAC_MAILBOX_13 */ 131 #define RFD3_PROC_IDX_BMSK 0xfff0000 132 #define RFD3_PROC_IDX_SHFT 16 133 #define RFD3_PROD_IDX_BMSK 0xfff 134 #define RFD3_PROD_IDX_SHFT 0 135 136 /* EMAC_MAILBOX_2 */ 137 #define NTPD_CONS_IDX_BMSK 0xffff0000 138 #define NTPD_CONS_IDX_SHFT 16 139 140 /* EMAC_MAILBOX_3 */ 141 #define RFD0_CONS_IDX_BMSK 0xfff 142 #define RFD0_CONS_IDX_SHFT 0 143 144 /* EMAC_MAILBOX_11 */ 145 #define H3TPD_PROD_IDX_BMSK 0xffff0000 146 #define H3TPD_PROD_IDX_SHFT 16 147 148 /* EMAC_AXI_MAST_CTRL */ 149 #define DATA_BYTE_SWAP 0x8 150 #define MAX_BOUND 0x2 151 #define MAX_BTYPE 0x1 152 153 /* EMAC_MAILBOX_12 */ 154 #define H3TPD_CONS_IDX_BMSK 0xffff0000 155 #define H3TPD_CONS_IDX_SHFT 16 156 157 /* EMAC_MAILBOX_9 */ 158 #define H2TPD_PROD_IDX_BMSK 0xffff 159 #define H2TPD_PROD_IDX_SHFT 0 160 161 /* EMAC_MAILBOX_10 */ 162 #define H1TPD_CONS_IDX_BMSK 0xffff0000 163 #define H1TPD_CONS_IDX_SHFT 16 164 #define H2TPD_CONS_IDX_BMSK 0xffff 165 #define H2TPD_CONS_IDX_SHFT 0 166 167 /* EMAC_ATHR_HEADER_CTRL */ 168 #define HEADER_CNT_EN 0x2 169 #define HEADER_ENABLE 0x1 170 171 /* EMAC_MAILBOX_0 */ 172 #define RFD0_PROC_IDX_BMSK 0xfff0000 173 #define RFD0_PROC_IDX_SHFT 16 174 #define RFD0_PROD_IDX_BMSK 0xfff 175 #define RFD0_PROD_IDX_SHFT 0 176 177 /* EMAC_MAILBOX_5 */ 178 #define RFD1_PROC_IDX_BMSK 0xfff0000 179 #define RFD1_PROC_IDX_SHFT 16 180 #define RFD1_PROD_IDX_BMSK 0xfff 181 #define RFD1_PROD_IDX_SHFT 0 182 183 /* EMAC_MISC_CTRL */ 184 #define RX_UNCPL_INT_EN 0x1 185 186 /* EMAC_MAILBOX_7 */ 187 #define RFD2_CONS_IDX_BMSK 0xfff0000 188 #define RFD2_CONS_IDX_SHFT 16 189 #define RFD1_CONS_IDX_BMSK 0xfff 190 #define RFD1_CONS_IDX_SHFT 0 191 192 /* EMAC_MAILBOX_8 */ 193 #define RFD3_CONS_IDX_BMSK 0xfff 194 #define RFD3_CONS_IDX_SHFT 0 195 196 /* EMAC_MAILBOX_15 */ 197 #define NTPD_PROD_IDX_BMSK 0xffff 198 #define NTPD_PROD_IDX_SHFT 0 199 200 /* EMAC_MAILBOX_16 */ 201 #define H1TPD_PROD_IDX_BMSK 0xffff 202 #define H1TPD_PROD_IDX_SHFT 0 203 204 #define RXQ0_RSS_HSTYP_IPV6_TCP_EN 0x20 205 #define RXQ0_RSS_HSTYP_IPV6_EN 0x10 206 #define RXQ0_RSS_HSTYP_IPV4_TCP_EN 0x8 207 #define RXQ0_RSS_HSTYP_IPV4_EN 0x4 208 209 /* EMAC_EMAC_WRAPPER_TX_TS_INX */ 210 #define EMAC_WRAPPER_TX_TS_EMPTY BIT(31) 211 #define EMAC_WRAPPER_TX_TS_INX_BMSK 0xffff 212 213 struct emac_skb_cb { 214 u32 tpd_idx; 215 unsigned long jiffies; 216 }; 217 218 #define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb) 219 #define EMAC_RSS_IDT_SIZE 256 220 #define JUMBO_1KAH 0x4 221 #define RXD_TH 0x100 222 #define EMAC_TPD_LAST_FRAGMENT 0x80000000 223 #define EMAC_TPD_TSTAMP_SAVE 0x80000000 224 225 /* EMAC Errors in emac_rrd.word[3] */ 226 #define EMAC_RRD_L4F BIT(14) 227 #define EMAC_RRD_IPF BIT(15) 228 #define EMAC_RRD_CRC BIT(21) 229 #define EMAC_RRD_FAE BIT(22) 230 #define EMAC_RRD_TRN BIT(23) 231 #define EMAC_RRD_RNT BIT(24) 232 #define EMAC_RRD_INC BIT(25) 233 #define EMAC_RRD_FOV BIT(29) 234 #define EMAC_RRD_LEN BIT(30) 235 236 /* Error bits that will result in a received frame being discarded */ 237 #define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \ 238 EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \ 239 EMAC_RRD_FOV | EMAC_RRD_LEN) 240 #define EMAC_RRD_STATS_DW_IDX 3 241 242 #define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX))) 243 #define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX))) 244 #define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX))) 245 246 #define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)])) 247 #define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)])) 248 249 #define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD 8 250 251 #define ISR_RX_PKT (\ 252 RX_PKT_INT0 |\ 253 RX_PKT_INT1 |\ 254 RX_PKT_INT2 |\ 255 RX_PKT_INT3) 256 257 void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr) 258 { 259 u32 crc32, bit, reg, mta; 260 261 /* Calculate the CRC of the MAC address */ 262 crc32 = ether_crc(ETH_ALEN, addr); 263 264 /* The HASH Table is an array of 2 32-bit registers. It is 265 * treated like an array of 64 bits (BitArray[hash_value]). 266 * Use the upper 6 bits of the above CRC as the hash value. 267 */ 268 reg = (crc32 >> 31) & 0x1; 269 bit = (crc32 >> 26) & 0x1F; 270 271 mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2)); 272 mta |= BIT(bit); 273 writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2)); 274 } 275 276 void emac_mac_multicast_addr_clear(struct emac_adapter *adpt) 277 { 278 writel(0, adpt->base + EMAC_HASH_TAB_REG0); 279 writel(0, adpt->base + EMAC_HASH_TAB_REG1); 280 } 281 282 /* definitions for RSS */ 283 #define EMAC_RSS_KEY(_i, _type) \ 284 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type))) 285 #define EMAC_RSS_TBL(_i, _type) \ 286 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type))) 287 288 /* Config MAC modes */ 289 void emac_mac_mode_config(struct emac_adapter *adpt) 290 { 291 struct net_device *netdev = adpt->netdev; 292 u32 mac; 293 294 mac = readl(adpt->base + EMAC_MAC_CTRL); 295 mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN); 296 297 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 298 mac |= VLAN_STRIP; 299 300 if (netdev->flags & IFF_PROMISC) 301 mac |= PROM_MODE; 302 303 if (netdev->flags & IFF_ALLMULTI) 304 mac |= MULTI_ALL; 305 306 writel(mac, adpt->base + EMAC_MAC_CTRL); 307 } 308 309 /* Config descriptor rings */ 310 static void emac_mac_dma_rings_config(struct emac_adapter *adpt) 311 { 312 static const unsigned short tpd_q_offset[] = { 313 EMAC_DESC_CTRL_8, EMAC_H1TPD_BASE_ADDR_LO, 314 EMAC_H2TPD_BASE_ADDR_LO, EMAC_H3TPD_BASE_ADDR_LO}; 315 static const unsigned short rfd_q_offset[] = { 316 EMAC_DESC_CTRL_2, EMAC_DESC_CTRL_10, 317 EMAC_DESC_CTRL_12, EMAC_DESC_CTRL_13}; 318 static const unsigned short rrd_q_offset[] = { 319 EMAC_DESC_CTRL_5, EMAC_DESC_CTRL_14, 320 EMAC_DESC_CTRL_15, EMAC_DESC_CTRL_16}; 321 322 /* TPD (Transmit Packet Descriptor) */ 323 writel(upper_32_bits(adpt->tx_q.tpd.dma_addr), 324 adpt->base + EMAC_DESC_CTRL_1); 325 326 writel(lower_32_bits(adpt->tx_q.tpd.dma_addr), 327 adpt->base + tpd_q_offset[0]); 328 329 writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK, 330 adpt->base + EMAC_DESC_CTRL_9); 331 332 /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */ 333 writel(upper_32_bits(adpt->rx_q.rfd.dma_addr), 334 adpt->base + EMAC_DESC_CTRL_0); 335 336 writel(lower_32_bits(adpt->rx_q.rfd.dma_addr), 337 adpt->base + rfd_q_offset[0]); 338 writel(lower_32_bits(adpt->rx_q.rrd.dma_addr), 339 adpt->base + rrd_q_offset[0]); 340 341 writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK, 342 adpt->base + EMAC_DESC_CTRL_3); 343 writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK, 344 adpt->base + EMAC_DESC_CTRL_6); 345 346 writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK, 347 adpt->base + EMAC_DESC_CTRL_4); 348 349 writel(0, adpt->base + EMAC_DESC_CTRL_11); 350 351 /* Load all of the base addresses above and ensure that triggering HW to 352 * read ring pointers is flushed 353 */ 354 writel(1, adpt->base + EMAC_INTER_SRAM_PART9); 355 } 356 357 /* Config transmit parameters */ 358 static void emac_mac_tx_config(struct emac_adapter *adpt) 359 { 360 u32 val; 361 362 writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) & 363 JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1); 364 365 val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) & 366 NUM_TPD_BURST_PREF_BMSK; 367 368 val |= TXQ_MODE | LS_8023_SP; 369 val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) & 370 NUM_TXF_BURST_PREF_BMSK; 371 372 writel(val, adpt->base + EMAC_TXQ_CTRL_0); 373 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2, 374 (TXF_HWM_BMSK | TXF_LWM_BMSK), 0); 375 } 376 377 /* Config receive parameters */ 378 static void emac_mac_rx_config(struct emac_adapter *adpt) 379 { 380 u32 val; 381 382 val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) & 383 NUM_RFD_BURST_PREF_BMSK; 384 val |= (SP_IPV6 | CUT_THRU_EN); 385 386 writel(val, adpt->base + EMAC_RXQ_CTRL_0); 387 388 val = readl(adpt->base + EMAC_RXQ_CTRL_1); 389 val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK | 390 RFD_PREF_UP_THRESHOLD_BMSK); 391 val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) | 392 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) | 393 (RFD_PREF_UP_TH << RFD_PREF_UP_THRESHOLD_SHFT); 394 writel(val, adpt->base + EMAC_RXQ_CTRL_1); 395 396 val = readl(adpt->base + EMAC_RXQ_CTRL_2); 397 val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK); 398 val |= (RXF_DOF_THRESFHOLD << RXF_DOF_THRESHOLD_SHFT) | 399 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT); 400 writel(val, adpt->base + EMAC_RXQ_CTRL_2); 401 402 val = readl(adpt->base + EMAC_RXQ_CTRL_3); 403 val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK); 404 val |= RXD_TH << RXD_THRESHOLD_SHFT; 405 writel(val, adpt->base + EMAC_RXQ_CTRL_3); 406 } 407 408 /* Config dma */ 409 static void emac_mac_dma_config(struct emac_adapter *adpt) 410 { 411 u32 dma_ctrl = DMAR_REQ_PRI; 412 413 switch (adpt->dma_order) { 414 case emac_dma_ord_in: 415 dma_ctrl |= IN_ORDER_MODE; 416 break; 417 case emac_dma_ord_enh: 418 dma_ctrl |= ENH_ORDER_MODE; 419 break; 420 case emac_dma_ord_out: 421 dma_ctrl |= OUT_ORDER_MODE; 422 break; 423 default: 424 break; 425 } 426 427 dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) & 428 REGRDBLEN_BMSK; 429 dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) & 430 REGWRBLEN_BMSK; 431 dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) & 432 DMAR_DLY_CNT_BMSK; 433 dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) & 434 DMAW_DLY_CNT_BMSK; 435 436 /* config DMA and ensure that configuration is flushed to HW */ 437 writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL); 438 } 439 440 /* set MAC address */ 441 static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr) 442 { 443 u32 sta; 444 445 /* for example: 00-A0-C6-11-22-33 446 * 0<-->C6112233, 1<-->00A0. 447 */ 448 449 /* low 32bit word */ 450 sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) | 451 (((u32)addr[4]) << 8) | (((u32)addr[5])); 452 writel(sta, adpt->base + EMAC_MAC_STA_ADDR0); 453 454 /* hight 32bit word */ 455 sta = (((u32)addr[0]) << 8) | (u32)addr[1]; 456 writel(sta, adpt->base + EMAC_MAC_STA_ADDR1); 457 } 458 459 static void emac_mac_config(struct emac_adapter *adpt) 460 { 461 struct net_device *netdev = adpt->netdev; 462 unsigned int max_frame; 463 u32 val; 464 465 emac_set_mac_address(adpt, netdev->dev_addr); 466 467 max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 468 adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ? 469 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE; 470 471 emac_mac_dma_rings_config(adpt); 472 473 writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN, 474 adpt->base + EMAC_MAX_FRAM_LEN_CTRL); 475 476 emac_mac_tx_config(adpt); 477 emac_mac_rx_config(adpt); 478 emac_mac_dma_config(adpt); 479 480 val = readl(adpt->base + EMAC_AXI_MAST_CTRL); 481 val &= ~(DATA_BYTE_SWAP | MAX_BOUND); 482 val |= MAX_BTYPE; 483 writel(val, adpt->base + EMAC_AXI_MAST_CTRL); 484 writel(0, adpt->base + EMAC_CLK_GATE_CTRL); 485 writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL); 486 } 487 488 void emac_mac_reset(struct emac_adapter *adpt) 489 { 490 emac_mac_stop(adpt); 491 492 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST); 493 usleep_range(100, 150); /* reset may take up to 100usec */ 494 495 /* interrupt clear-on-read */ 496 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN); 497 } 498 499 static void emac_mac_start(struct emac_adapter *adpt) 500 { 501 struct phy_device *phydev = adpt->phydev; 502 u32 mac, csr1; 503 504 /* enable tx queue */ 505 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN); 506 507 /* enable rx queue */ 508 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN); 509 510 /* enable mac control */ 511 mac = readl(adpt->base + EMAC_MAC_CTRL); 512 csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1); 513 514 mac |= TXEN | RXEN; /* enable RX/TX */ 515 516 /* Configure MAC flow control. If set to automatic, then match 517 * whatever the PHY does. Otherwise, enable or disable it, depending 518 * on what the user configured via ethtool. 519 */ 520 mac &= ~(RXFC | TXFC); 521 522 if (adpt->automatic) { 523 /* If it's set to automatic, then update our local values */ 524 adpt->rx_flow_control = phydev->pause; 525 adpt->tx_flow_control = phydev->pause != phydev->asym_pause; 526 } 527 mac |= adpt->rx_flow_control ? RXFC : 0; 528 mac |= adpt->tx_flow_control ? TXFC : 0; 529 530 /* setup link speed */ 531 mac &= ~SPEED_MASK; 532 if (phydev->speed == SPEED_1000) { 533 mac |= SPEED(2); 534 csr1 |= FREQ_MODE; 535 } else { 536 mac |= SPEED(1); 537 csr1 &= ~FREQ_MODE; 538 } 539 540 if (phydev->duplex == DUPLEX_FULL) 541 mac |= FULLD; 542 else 543 mac &= ~FULLD; 544 545 /* other parameters */ 546 mac |= (CRCE | PCRCE); 547 mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK); 548 mac |= BROAD_EN; 549 mac |= FLCHK; 550 mac &= ~RX_CHKSUM_EN; 551 mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL | 552 DEBUG_MODE | SINGLE_PAUSE_MODE); 553 554 /* Enable single-pause-frame mode if requested. 555 * 556 * If enabled, the EMAC will send a single pause frame when the RX 557 * queue is full. This normally leads to packet loss because 558 * the pause frame disables the remote MAC only for 33ms (the quanta), 559 * and then the remote MAC continues sending packets even though 560 * the RX queue is still full. 561 * 562 * If disabled, the EMAC sends a pause frame every 31ms until the RX 563 * queue is no longer full. Normally, this is the preferred 564 * method of operation. However, when the system is hung (e.g. 565 * cores are halted), the EMAC interrupt handler is never called 566 * and so the RX queue fills up quickly and stays full. The resuling 567 * non-stop "flood" of pause frames sometimes has the effect of 568 * disabling nearby switches. In some cases, other nearby switches 569 * are also affected, shutting down the entire network. 570 * 571 * The user can enable or disable single-pause-frame mode 572 * via ethtool. 573 */ 574 mac |= adpt->single_pause_mode ? SINGLE_PAUSE_MODE : 0; 575 576 writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1); 577 578 writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL); 579 580 /* enable interrupt read clear, low power sleep mode and 581 * the irq moderators 582 */ 583 584 writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT); 585 writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN | 586 IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL); 587 588 emac_mac_mode_config(adpt); 589 590 emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL, 591 (HEADER_ENABLE | HEADER_CNT_EN), 0); 592 } 593 594 void emac_mac_stop(struct emac_adapter *adpt) 595 { 596 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0); 597 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0); 598 emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0); 599 usleep_range(1000, 1050); /* stopping mac may take upto 1msec */ 600 } 601 602 /* Free all descriptors of given transmit queue */ 603 static void emac_tx_q_descs_free(struct emac_adapter *adpt) 604 { 605 struct emac_tx_queue *tx_q = &adpt->tx_q; 606 unsigned int i; 607 size_t size; 608 609 /* ring already cleared, nothing to do */ 610 if (!tx_q->tpd.tpbuff) 611 return; 612 613 for (i = 0; i < tx_q->tpd.count; i++) { 614 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i); 615 616 if (tpbuf->dma_addr) { 617 dma_unmap_single(adpt->netdev->dev.parent, 618 tpbuf->dma_addr, tpbuf->length, 619 DMA_TO_DEVICE); 620 tpbuf->dma_addr = 0; 621 } 622 if (tpbuf->skb) { 623 dev_kfree_skb_any(tpbuf->skb); 624 tpbuf->skb = NULL; 625 } 626 } 627 628 size = sizeof(struct emac_buffer) * tx_q->tpd.count; 629 memset(tx_q->tpd.tpbuff, 0, size); 630 631 /* clear the descriptor ring */ 632 memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size); 633 634 tx_q->tpd.consume_idx = 0; 635 tx_q->tpd.produce_idx = 0; 636 } 637 638 /* Free all descriptors of given receive queue */ 639 static void emac_rx_q_free_descs(struct emac_adapter *adpt) 640 { 641 struct device *dev = adpt->netdev->dev.parent; 642 struct emac_rx_queue *rx_q = &adpt->rx_q; 643 unsigned int i; 644 size_t size; 645 646 /* ring already cleared, nothing to do */ 647 if (!rx_q->rfd.rfbuff) 648 return; 649 650 for (i = 0; i < rx_q->rfd.count; i++) { 651 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i); 652 653 if (rfbuf->dma_addr) { 654 dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length, 655 DMA_FROM_DEVICE); 656 rfbuf->dma_addr = 0; 657 } 658 if (rfbuf->skb) { 659 dev_kfree_skb(rfbuf->skb); 660 rfbuf->skb = NULL; 661 } 662 } 663 664 size = sizeof(struct emac_buffer) * rx_q->rfd.count; 665 memset(rx_q->rfd.rfbuff, 0, size); 666 667 /* clear the descriptor rings */ 668 memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size); 669 rx_q->rrd.produce_idx = 0; 670 rx_q->rrd.consume_idx = 0; 671 672 memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size); 673 rx_q->rfd.produce_idx = 0; 674 rx_q->rfd.consume_idx = 0; 675 } 676 677 /* Free all buffers associated with given transmit queue */ 678 static void emac_tx_q_bufs_free(struct emac_adapter *adpt) 679 { 680 struct emac_tx_queue *tx_q = &adpt->tx_q; 681 682 emac_tx_q_descs_free(adpt); 683 684 kfree(tx_q->tpd.tpbuff); 685 tx_q->tpd.tpbuff = NULL; 686 tx_q->tpd.v_addr = NULL; 687 tx_q->tpd.dma_addr = 0; 688 tx_q->tpd.size = 0; 689 } 690 691 /* Allocate TX descriptor ring for the given transmit queue */ 692 static int emac_tx_q_desc_alloc(struct emac_adapter *adpt, 693 struct emac_tx_queue *tx_q) 694 { 695 struct emac_ring_header *ring_header = &adpt->ring_header; 696 size_t size; 697 698 size = sizeof(struct emac_buffer) * tx_q->tpd.count; 699 tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL); 700 if (!tx_q->tpd.tpbuff) 701 return -ENOMEM; 702 703 tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4); 704 tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used; 705 tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used; 706 ring_header->used += ALIGN(tx_q->tpd.size, 8); 707 tx_q->tpd.produce_idx = 0; 708 tx_q->tpd.consume_idx = 0; 709 710 return 0; 711 } 712 713 /* Free all buffers associated with given transmit queue */ 714 static void emac_rx_q_bufs_free(struct emac_adapter *adpt) 715 { 716 struct emac_rx_queue *rx_q = &adpt->rx_q; 717 718 emac_rx_q_free_descs(adpt); 719 720 kfree(rx_q->rfd.rfbuff); 721 rx_q->rfd.rfbuff = NULL; 722 723 rx_q->rfd.v_addr = NULL; 724 rx_q->rfd.dma_addr = 0; 725 rx_q->rfd.size = 0; 726 727 rx_q->rrd.v_addr = NULL; 728 rx_q->rrd.dma_addr = 0; 729 rx_q->rrd.size = 0; 730 } 731 732 /* Allocate RX descriptor rings for the given receive queue */ 733 static int emac_rx_descs_alloc(struct emac_adapter *adpt) 734 { 735 struct emac_ring_header *ring_header = &adpt->ring_header; 736 struct emac_rx_queue *rx_q = &adpt->rx_q; 737 size_t size; 738 739 size = sizeof(struct emac_buffer) * rx_q->rfd.count; 740 rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL); 741 if (!rx_q->rfd.rfbuff) 742 return -ENOMEM; 743 744 rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4); 745 rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4); 746 747 rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used; 748 rx_q->rrd.v_addr = ring_header->v_addr + ring_header->used; 749 ring_header->used += ALIGN(rx_q->rrd.size, 8); 750 751 rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used; 752 rx_q->rfd.v_addr = ring_header->v_addr + ring_header->used; 753 ring_header->used += ALIGN(rx_q->rfd.size, 8); 754 755 rx_q->rrd.produce_idx = 0; 756 rx_q->rrd.consume_idx = 0; 757 758 rx_q->rfd.produce_idx = 0; 759 rx_q->rfd.consume_idx = 0; 760 761 return 0; 762 } 763 764 /* Allocate all TX and RX descriptor rings */ 765 int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt) 766 { 767 struct emac_ring_header *ring_header = &adpt->ring_header; 768 struct device *dev = adpt->netdev->dev.parent; 769 unsigned int num_tx_descs = adpt->tx_desc_cnt; 770 unsigned int num_rx_descs = adpt->rx_desc_cnt; 771 int ret; 772 773 adpt->tx_q.tpd.count = adpt->tx_desc_cnt; 774 775 adpt->rx_q.rrd.count = adpt->rx_desc_cnt; 776 adpt->rx_q.rfd.count = adpt->rx_desc_cnt; 777 778 /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment, 779 * hence the additional padding bytes are allocated. 780 */ 781 ring_header->size = num_tx_descs * (adpt->tpd_size * 4) + 782 num_rx_descs * (adpt->rfd_size * 4) + 783 num_rx_descs * (adpt->rrd_size * 4) + 784 8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */ 785 786 ring_header->used = 0; 787 ring_header->v_addr = dma_zalloc_coherent(dev, ring_header->size, 788 &ring_header->dma_addr, 789 GFP_KERNEL); 790 if (!ring_header->v_addr) 791 return -ENOMEM; 792 793 ring_header->used = ALIGN(ring_header->dma_addr, 8) - 794 ring_header->dma_addr; 795 796 ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q); 797 if (ret) { 798 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n"); 799 goto err_alloc_tx; 800 } 801 802 ret = emac_rx_descs_alloc(adpt); 803 if (ret) { 804 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n"); 805 goto err_alloc_rx; 806 } 807 808 return 0; 809 810 err_alloc_rx: 811 emac_tx_q_bufs_free(adpt); 812 err_alloc_tx: 813 dma_free_coherent(dev, ring_header->size, 814 ring_header->v_addr, ring_header->dma_addr); 815 816 ring_header->v_addr = NULL; 817 ring_header->dma_addr = 0; 818 ring_header->size = 0; 819 ring_header->used = 0; 820 821 return ret; 822 } 823 824 /* Free all TX and RX descriptor rings */ 825 void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt) 826 { 827 struct emac_ring_header *ring_header = &adpt->ring_header; 828 struct device *dev = adpt->netdev->dev.parent; 829 830 emac_tx_q_bufs_free(adpt); 831 emac_rx_q_bufs_free(adpt); 832 833 dma_free_coherent(dev, ring_header->size, 834 ring_header->v_addr, ring_header->dma_addr); 835 836 ring_header->v_addr = NULL; 837 ring_header->dma_addr = 0; 838 ring_header->size = 0; 839 ring_header->used = 0; 840 } 841 842 /* Initialize descriptor rings */ 843 static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt) 844 { 845 unsigned int i; 846 847 adpt->tx_q.tpd.produce_idx = 0; 848 adpt->tx_q.tpd.consume_idx = 0; 849 for (i = 0; i < adpt->tx_q.tpd.count; i++) 850 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0; 851 852 adpt->rx_q.rrd.produce_idx = 0; 853 adpt->rx_q.rrd.consume_idx = 0; 854 adpt->rx_q.rfd.produce_idx = 0; 855 adpt->rx_q.rfd.consume_idx = 0; 856 for (i = 0; i < adpt->rx_q.rfd.count; i++) 857 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0; 858 } 859 860 /* Produce new receive free descriptor */ 861 static void emac_mac_rx_rfd_create(struct emac_adapter *adpt, 862 struct emac_rx_queue *rx_q, 863 dma_addr_t addr) 864 { 865 u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx); 866 867 *(hw_rfd++) = lower_32_bits(addr); 868 *hw_rfd = upper_32_bits(addr); 869 870 if (++rx_q->rfd.produce_idx == rx_q->rfd.count) 871 rx_q->rfd.produce_idx = 0; 872 } 873 874 /* Fill up receive queue's RFD with preallocated receive buffers */ 875 static void emac_mac_rx_descs_refill(struct emac_adapter *adpt, 876 struct emac_rx_queue *rx_q) 877 { 878 struct emac_buffer *curr_rxbuf; 879 struct emac_buffer *next_rxbuf; 880 unsigned int count = 0; 881 u32 next_produce_idx; 882 883 next_produce_idx = rx_q->rfd.produce_idx + 1; 884 if (next_produce_idx == rx_q->rfd.count) 885 next_produce_idx = 0; 886 887 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx); 888 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx); 889 890 /* this always has a blank rx_buffer*/ 891 while (!next_rxbuf->dma_addr) { 892 struct sk_buff *skb; 893 int ret; 894 895 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size); 896 if (!skb) 897 break; 898 899 curr_rxbuf->dma_addr = 900 dma_map_single(adpt->netdev->dev.parent, skb->data, 901 adpt->rxbuf_size, DMA_FROM_DEVICE); 902 903 ret = dma_mapping_error(adpt->netdev->dev.parent, 904 curr_rxbuf->dma_addr); 905 if (ret) { 906 dev_kfree_skb(skb); 907 break; 908 } 909 curr_rxbuf->skb = skb; 910 curr_rxbuf->length = adpt->rxbuf_size; 911 912 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr); 913 next_produce_idx = rx_q->rfd.produce_idx + 1; 914 if (next_produce_idx == rx_q->rfd.count) 915 next_produce_idx = 0; 916 917 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx); 918 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx); 919 count++; 920 } 921 922 if (count) { 923 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) & 924 rx_q->produce_mask; 925 emac_reg_update32(adpt->base + rx_q->produce_reg, 926 rx_q->produce_mask, prod_idx); 927 } 928 } 929 930 static void emac_adjust_link(struct net_device *netdev) 931 { 932 struct emac_adapter *adpt = netdev_priv(netdev); 933 struct emac_sgmii *sgmii = &adpt->phy; 934 struct phy_device *phydev = netdev->phydev; 935 936 if (phydev->link) { 937 emac_mac_start(adpt); 938 sgmii->link_up(adpt); 939 } else { 940 sgmii->link_down(adpt); 941 emac_mac_stop(adpt); 942 } 943 944 phy_print_status(phydev); 945 } 946 947 /* Bringup the interface/HW */ 948 int emac_mac_up(struct emac_adapter *adpt) 949 { 950 struct net_device *netdev = adpt->netdev; 951 int ret; 952 953 emac_mac_rx_tx_ring_reset_all(adpt); 954 emac_mac_config(adpt); 955 emac_mac_rx_descs_refill(adpt, &adpt->rx_q); 956 957 adpt->phydev->irq = PHY_POLL; 958 ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link, 959 PHY_INTERFACE_MODE_SGMII); 960 if (ret) { 961 netdev_err(adpt->netdev, "could not connect phy\n"); 962 return ret; 963 } 964 965 phy_attached_print(adpt->phydev, NULL); 966 967 /* enable mac irq */ 968 writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS); 969 writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK); 970 971 phy_start(adpt->phydev); 972 973 napi_enable(&adpt->rx_q.napi); 974 netif_start_queue(netdev); 975 976 return 0; 977 } 978 979 /* Bring down the interface/HW */ 980 void emac_mac_down(struct emac_adapter *adpt) 981 { 982 struct net_device *netdev = adpt->netdev; 983 984 netif_stop_queue(netdev); 985 napi_disable(&adpt->rx_q.napi); 986 987 phy_stop(adpt->phydev); 988 989 /* Interrupts must be disabled before the PHY is disconnected, to 990 * avoid a race condition where adjust_link is null when we get 991 * an interrupt. 992 */ 993 writel(DIS_INT, adpt->base + EMAC_INT_STATUS); 994 writel(0, adpt->base + EMAC_INT_MASK); 995 synchronize_irq(adpt->irq.irq); 996 997 phy_disconnect(adpt->phydev); 998 999 emac_mac_reset(adpt); 1000 1001 emac_tx_q_descs_free(adpt); 1002 netdev_reset_queue(adpt->netdev); 1003 emac_rx_q_free_descs(adpt); 1004 } 1005 1006 /* Consume next received packet descriptor */ 1007 static bool emac_rx_process_rrd(struct emac_adapter *adpt, 1008 struct emac_rx_queue *rx_q, 1009 struct emac_rrd *rrd) 1010 { 1011 u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx); 1012 1013 rrd->word[3] = *(hw_rrd + 3); 1014 1015 if (!RRD_UPDT(rrd)) 1016 return false; 1017 1018 rrd->word[4] = 0; 1019 rrd->word[5] = 0; 1020 1021 rrd->word[0] = *(hw_rrd++); 1022 rrd->word[1] = *(hw_rrd++); 1023 rrd->word[2] = *(hw_rrd++); 1024 1025 if (unlikely(RRD_NOR(rrd) != 1)) { 1026 netdev_err(adpt->netdev, 1027 "error: multi-RFD not support yet! nor:%lu\n", 1028 RRD_NOR(rrd)); 1029 } 1030 1031 /* mark rrd as processed */ 1032 RRD_UPDT_SET(rrd, 0); 1033 *hw_rrd = rrd->word[3]; 1034 1035 if (++rx_q->rrd.consume_idx == rx_q->rrd.count) 1036 rx_q->rrd.consume_idx = 0; 1037 1038 return true; 1039 } 1040 1041 /* Produce new transmit descriptor */ 1042 static void emac_tx_tpd_create(struct emac_adapter *adpt, 1043 struct emac_tx_queue *tx_q, struct emac_tpd *tpd) 1044 { 1045 u32 *hw_tpd; 1046 1047 tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx; 1048 hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx); 1049 1050 if (++tx_q->tpd.produce_idx == tx_q->tpd.count) 1051 tx_q->tpd.produce_idx = 0; 1052 1053 *(hw_tpd++) = tpd->word[0]; 1054 *(hw_tpd++) = tpd->word[1]; 1055 *(hw_tpd++) = tpd->word[2]; 1056 *hw_tpd = tpd->word[3]; 1057 } 1058 1059 /* Mark the last transmit descriptor as such (for the transmit packet) */ 1060 static void emac_tx_tpd_mark_last(struct emac_adapter *adpt, 1061 struct emac_tx_queue *tx_q) 1062 { 1063 u32 *hw_tpd = 1064 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx); 1065 u32 tmp_tpd; 1066 1067 tmp_tpd = *(hw_tpd + 1); 1068 tmp_tpd |= EMAC_TPD_LAST_FRAGMENT; 1069 *(hw_tpd + 1) = tmp_tpd; 1070 } 1071 1072 static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd) 1073 { 1074 struct emac_buffer *rfbuf = rx_q->rfd.rfbuff; 1075 u32 consume_idx = RRD_SI(rrd); 1076 unsigned int i; 1077 1078 for (i = 0; i < RRD_NOR(rrd); i++) { 1079 rfbuf[consume_idx].skb = NULL; 1080 if (++consume_idx == rx_q->rfd.count) 1081 consume_idx = 0; 1082 } 1083 1084 rx_q->rfd.consume_idx = consume_idx; 1085 rx_q->rfd.process_idx = consume_idx; 1086 } 1087 1088 /* Push the received skb to upper layers */ 1089 static void emac_receive_skb(struct emac_rx_queue *rx_q, 1090 struct sk_buff *skb, 1091 u16 vlan_tag, bool vlan_flag) 1092 { 1093 if (vlan_flag) { 1094 u16 vlan; 1095 1096 EMAC_TAG_TO_VLAN(vlan_tag, vlan); 1097 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan); 1098 } 1099 1100 napi_gro_receive(&rx_q->napi, skb); 1101 } 1102 1103 /* Process receive event */ 1104 void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q, 1105 int *num_pkts, int max_pkts) 1106 { 1107 u32 proc_idx, hw_consume_idx, num_consume_pkts; 1108 struct net_device *netdev = adpt->netdev; 1109 struct emac_buffer *rfbuf; 1110 unsigned int count = 0; 1111 struct emac_rrd rrd; 1112 struct sk_buff *skb; 1113 u32 reg; 1114 1115 reg = readl_relaxed(adpt->base + rx_q->consume_reg); 1116 1117 hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift; 1118 num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ? 1119 (hw_consume_idx - rx_q->rrd.consume_idx) : 1120 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx); 1121 1122 do { 1123 if (!num_consume_pkts) 1124 break; 1125 1126 if (!emac_rx_process_rrd(adpt, rx_q, &rrd)) 1127 break; 1128 1129 if (likely(RRD_NOR(&rrd) == 1)) { 1130 /* good receive */ 1131 rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd)); 1132 dma_unmap_single(adpt->netdev->dev.parent, 1133 rfbuf->dma_addr, rfbuf->length, 1134 DMA_FROM_DEVICE); 1135 rfbuf->dma_addr = 0; 1136 skb = rfbuf->skb; 1137 } else { 1138 netdev_err(adpt->netdev, 1139 "error: multi-RFD not support yet!\n"); 1140 break; 1141 } 1142 emac_rx_rfd_clean(rx_q, &rrd); 1143 num_consume_pkts--; 1144 count++; 1145 1146 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags 1147 * with DF set are marked as error), drop packets based on the 1148 * error mask rather than the summary bit (ignoring L4F errors) 1149 */ 1150 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) { 1151 netif_dbg(adpt, rx_status, adpt->netdev, 1152 "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n", 1153 rrd.word[0], rrd.word[1], 1154 rrd.word[2], rrd.word[3]); 1155 1156 dev_kfree_skb(skb); 1157 continue; 1158 } 1159 1160 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN); 1161 skb->dev = netdev; 1162 skb->protocol = eth_type_trans(skb, skb->dev); 1163 if (netdev->features & NETIF_F_RXCSUM) 1164 skb->ip_summed = RRD_L4F(&rrd) ? 1165 CHECKSUM_NONE : CHECKSUM_UNNECESSARY; 1166 else 1167 skb_checksum_none_assert(skb); 1168 1169 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd), 1170 (bool)RRD_CVTAG(&rrd)); 1171 1172 (*num_pkts)++; 1173 } while (*num_pkts < max_pkts); 1174 1175 if (count) { 1176 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) & 1177 rx_q->process_mask; 1178 emac_reg_update32(adpt->base + rx_q->process_reg, 1179 rx_q->process_mask, proc_idx); 1180 emac_mac_rx_descs_refill(adpt, rx_q); 1181 } 1182 } 1183 1184 /* get the number of free transmit descriptors */ 1185 static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q) 1186 { 1187 u32 produce_idx = tx_q->tpd.produce_idx; 1188 u32 consume_idx = tx_q->tpd.consume_idx; 1189 1190 return (consume_idx > produce_idx) ? 1191 (consume_idx - produce_idx - 1) : 1192 (tx_q->tpd.count + consume_idx - produce_idx - 1); 1193 } 1194 1195 /* Process transmit event */ 1196 void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q) 1197 { 1198 u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg); 1199 u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0; 1200 struct emac_buffer *tpbuf; 1201 1202 hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift; 1203 1204 while (tx_q->tpd.consume_idx != hw_consume_idx) { 1205 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx); 1206 if (tpbuf->dma_addr) { 1207 dma_unmap_single(adpt->netdev->dev.parent, 1208 tpbuf->dma_addr, tpbuf->length, 1209 DMA_TO_DEVICE); 1210 tpbuf->dma_addr = 0; 1211 } 1212 1213 if (tpbuf->skb) { 1214 pkts_compl++; 1215 bytes_compl += tpbuf->skb->len; 1216 dev_kfree_skb_irq(tpbuf->skb); 1217 tpbuf->skb = NULL; 1218 } 1219 1220 if (++tx_q->tpd.consume_idx == tx_q->tpd.count) 1221 tx_q->tpd.consume_idx = 0; 1222 } 1223 1224 netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl); 1225 1226 if (netif_queue_stopped(adpt->netdev)) 1227 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1)) 1228 netif_wake_queue(adpt->netdev); 1229 } 1230 1231 /* Initialize all queue data structures */ 1232 void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev, 1233 struct emac_adapter *adpt) 1234 { 1235 adpt->rx_q.netdev = adpt->netdev; 1236 1237 adpt->rx_q.produce_reg = EMAC_MAILBOX_0; 1238 adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK; 1239 adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT; 1240 1241 adpt->rx_q.process_reg = EMAC_MAILBOX_0; 1242 adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK; 1243 adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT; 1244 1245 adpt->rx_q.consume_reg = EMAC_MAILBOX_3; 1246 adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK; 1247 adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT; 1248 1249 adpt->rx_q.irq = &adpt->irq; 1250 adpt->rx_q.intr = adpt->irq.mask & ISR_RX_PKT; 1251 1252 adpt->tx_q.produce_reg = EMAC_MAILBOX_15; 1253 adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK; 1254 adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT; 1255 1256 adpt->tx_q.consume_reg = EMAC_MAILBOX_2; 1257 adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK; 1258 adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT; 1259 } 1260 1261 /* Fill up transmit descriptors with TSO and Checksum offload information */ 1262 static int emac_tso_csum(struct emac_adapter *adpt, 1263 struct emac_tx_queue *tx_q, 1264 struct sk_buff *skb, 1265 struct emac_tpd *tpd) 1266 { 1267 unsigned int hdr_len; 1268 int ret; 1269 1270 if (skb_is_gso(skb)) { 1271 if (skb_header_cloned(skb)) { 1272 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 1273 if (unlikely(ret)) 1274 return ret; 1275 } 1276 1277 if (skb->protocol == htons(ETH_P_IP)) { 1278 u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data) 1279 + ntohs(ip_hdr(skb)->tot_len); 1280 if (skb->len > pkt_len) 1281 pskb_trim(skb, pkt_len); 1282 } 1283 1284 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 1285 if (unlikely(skb->len == hdr_len)) { 1286 /* we only need to do csum */ 1287 netif_warn(adpt, tx_err, adpt->netdev, 1288 "tso not needed for packet with 0 data\n"); 1289 goto do_csum; 1290 } 1291 1292 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 1293 ip_hdr(skb)->check = 0; 1294 tcp_hdr(skb)->check = 1295 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 1296 ip_hdr(skb)->daddr, 1297 0, IPPROTO_TCP, 0); 1298 TPD_IPV4_SET(tpd, 1); 1299 } 1300 1301 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) { 1302 /* ipv6 tso need an extra tpd */ 1303 struct emac_tpd extra_tpd; 1304 1305 memset(tpd, 0, sizeof(*tpd)); 1306 memset(&extra_tpd, 0, sizeof(extra_tpd)); 1307 1308 ipv6_hdr(skb)->payload_len = 0; 1309 tcp_hdr(skb)->check = 1310 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 1311 &ipv6_hdr(skb)->daddr, 1312 0, IPPROTO_TCP, 0); 1313 TPD_PKT_LEN_SET(&extra_tpd, skb->len); 1314 TPD_LSO_SET(&extra_tpd, 1); 1315 TPD_LSOV_SET(&extra_tpd, 1); 1316 emac_tx_tpd_create(adpt, tx_q, &extra_tpd); 1317 TPD_LSOV_SET(tpd, 1); 1318 } 1319 1320 TPD_LSO_SET(tpd, 1); 1321 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb)); 1322 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size); 1323 return 0; 1324 } 1325 1326 do_csum: 1327 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 1328 unsigned int css, cso; 1329 1330 cso = skb_transport_offset(skb); 1331 if (unlikely(cso & 0x1)) { 1332 netdev_err(adpt->netdev, 1333 "error: payload offset should be even\n"); 1334 return -EINVAL; 1335 } 1336 css = cso + skb->csum_offset; 1337 1338 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1); 1339 TPD_CXSUM_OFFSET_SET(tpd, css >> 1); 1340 TPD_CSX_SET(tpd, 1); 1341 } 1342 1343 return 0; 1344 } 1345 1346 /* Fill up transmit descriptors */ 1347 static void emac_tx_fill_tpd(struct emac_adapter *adpt, 1348 struct emac_tx_queue *tx_q, struct sk_buff *skb, 1349 struct emac_tpd *tpd) 1350 { 1351 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 1352 unsigned int first = tx_q->tpd.produce_idx; 1353 unsigned int len = skb_headlen(skb); 1354 struct emac_buffer *tpbuf = NULL; 1355 unsigned int mapped_len = 0; 1356 unsigned int i; 1357 int count = 0; 1358 int ret; 1359 1360 /* if Large Segment Offload is (in TCP Segmentation Offload struct) */ 1361 if (TPD_LSO(tpd)) { 1362 mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 1363 1364 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); 1365 tpbuf->length = mapped_len; 1366 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent, 1367 skb->data, tpbuf->length, 1368 DMA_TO_DEVICE); 1369 ret = dma_mapping_error(adpt->netdev->dev.parent, 1370 tpbuf->dma_addr); 1371 if (ret) 1372 goto error; 1373 1374 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); 1375 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); 1376 TPD_BUF_LEN_SET(tpd, tpbuf->length); 1377 emac_tx_tpd_create(adpt, tx_q, tpd); 1378 count++; 1379 } 1380 1381 if (mapped_len < len) { 1382 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); 1383 tpbuf->length = len - mapped_len; 1384 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent, 1385 skb->data + mapped_len, 1386 tpbuf->length, DMA_TO_DEVICE); 1387 ret = dma_mapping_error(adpt->netdev->dev.parent, 1388 tpbuf->dma_addr); 1389 if (ret) 1390 goto error; 1391 1392 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); 1393 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); 1394 TPD_BUF_LEN_SET(tpd, tpbuf->length); 1395 emac_tx_tpd_create(adpt, tx_q, tpd); 1396 count++; 1397 } 1398 1399 for (i = 0; i < nr_frags; i++) { 1400 struct skb_frag_struct *frag; 1401 1402 frag = &skb_shinfo(skb)->frags[i]; 1403 1404 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); 1405 tpbuf->length = frag->size; 1406 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent, 1407 frag->page.p, frag->page_offset, 1408 tpbuf->length, DMA_TO_DEVICE); 1409 ret = dma_mapping_error(adpt->netdev->dev.parent, 1410 tpbuf->dma_addr); 1411 if (ret) 1412 goto error; 1413 1414 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); 1415 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); 1416 TPD_BUF_LEN_SET(tpd, tpbuf->length); 1417 emac_tx_tpd_create(adpt, tx_q, tpd); 1418 count++; 1419 } 1420 1421 /* The last tpd */ 1422 wmb(); 1423 emac_tx_tpd_mark_last(adpt, tx_q); 1424 1425 /* The last buffer info contain the skb address, 1426 * so it will be freed after unmap 1427 */ 1428 tpbuf->skb = skb; 1429 1430 return; 1431 1432 error: 1433 /* One of the memory mappings failed, so undo everything */ 1434 tx_q->tpd.produce_idx = first; 1435 1436 while (count--) { 1437 tpbuf = GET_TPD_BUFFER(tx_q, first); 1438 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr, 1439 tpbuf->length, DMA_TO_DEVICE); 1440 tpbuf->dma_addr = 0; 1441 tpbuf->length = 0; 1442 1443 if (++first == tx_q->tpd.count) 1444 first = 0; 1445 } 1446 1447 dev_kfree_skb(skb); 1448 } 1449 1450 /* Transmit the packet using specified transmit queue */ 1451 int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q, 1452 struct sk_buff *skb) 1453 { 1454 struct emac_tpd tpd; 1455 u32 prod_idx; 1456 1457 memset(&tpd, 0, sizeof(tpd)); 1458 1459 if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) { 1460 dev_kfree_skb_any(skb); 1461 return NETDEV_TX_OK; 1462 } 1463 1464 if (skb_vlan_tag_present(skb)) { 1465 u16 tag; 1466 1467 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag); 1468 TPD_CVLAN_TAG_SET(&tpd, tag); 1469 TPD_INSTC_SET(&tpd, 1); 1470 } 1471 1472 if (skb_network_offset(skb) != ETH_HLEN) 1473 TPD_TYP_SET(&tpd, 1); 1474 1475 emac_tx_fill_tpd(adpt, tx_q, skb, &tpd); 1476 1477 netdev_sent_queue(adpt->netdev, skb->len); 1478 1479 /* Make sure the are enough free descriptors to hold one 1480 * maximum-sized SKB. We need one desc for each fragment, 1481 * one for the checksum (emac_tso_csum), one for TSO, and 1482 * and one for the SKB header. 1483 */ 1484 if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3)) 1485 netif_stop_queue(adpt->netdev); 1486 1487 /* update produce idx */ 1488 prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) & 1489 tx_q->produce_mask; 1490 emac_reg_update32(adpt->base + tx_q->produce_reg, 1491 tx_q->produce_mask, prod_idx); 1492 1493 return NETDEV_TX_OK; 1494 } 1495