1 /* Copyright (c) 2013-2016, The Linux Foundation. All rights reserved. 2 * 3 * This program is free software; you can redistribute it and/or modify 4 * it under the terms of the GNU General Public License version 2 and 5 * only version 2 as published by the Free Software Foundation. 6 * 7 * This program is distributed in the hope that it will be useful, 8 * but WITHOUT ANY WARRANTY; without even the implied warranty of 9 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 10 * GNU General Public License for more details. 11 */ 12 13 /* Qualcomm Technologies, Inc. EMAC Ethernet Controller MAC layer support 14 */ 15 16 #include <linux/tcp.h> 17 #include <linux/ip.h> 18 #include <linux/ipv6.h> 19 #include <linux/crc32.h> 20 #include <linux/if_vlan.h> 21 #include <linux/jiffies.h> 22 #include <linux/phy.h> 23 #include <linux/of.h> 24 #include <net/ip6_checksum.h> 25 #include "emac.h" 26 #include "emac-sgmii.h" 27 28 /* EMAC base register offsets */ 29 #define EMAC_MAC_CTRL 0x001480 30 #define EMAC_WOL_CTRL0 0x0014a0 31 #define EMAC_RSS_KEY0 0x0014b0 32 #define EMAC_H1TPD_BASE_ADDR_LO 0x0014e0 33 #define EMAC_H2TPD_BASE_ADDR_LO 0x0014e4 34 #define EMAC_H3TPD_BASE_ADDR_LO 0x0014e8 35 #define EMAC_INTER_SRAM_PART9 0x001534 36 #define EMAC_DESC_CTRL_0 0x001540 37 #define EMAC_DESC_CTRL_1 0x001544 38 #define EMAC_DESC_CTRL_2 0x001550 39 #define EMAC_DESC_CTRL_10 0x001554 40 #define EMAC_DESC_CTRL_12 0x001558 41 #define EMAC_DESC_CTRL_13 0x00155c 42 #define EMAC_DESC_CTRL_3 0x001560 43 #define EMAC_DESC_CTRL_4 0x001564 44 #define EMAC_DESC_CTRL_5 0x001568 45 #define EMAC_DESC_CTRL_14 0x00156c 46 #define EMAC_DESC_CTRL_15 0x001570 47 #define EMAC_DESC_CTRL_16 0x001574 48 #define EMAC_DESC_CTRL_6 0x001578 49 #define EMAC_DESC_CTRL_8 0x001580 50 #define EMAC_DESC_CTRL_9 0x001584 51 #define EMAC_DESC_CTRL_11 0x001588 52 #define EMAC_TXQ_CTRL_0 0x001590 53 #define EMAC_TXQ_CTRL_1 0x001594 54 #define EMAC_TXQ_CTRL_2 0x001598 55 #define EMAC_RXQ_CTRL_0 0x0015a0 56 #define EMAC_RXQ_CTRL_1 0x0015a4 57 #define EMAC_RXQ_CTRL_2 0x0015a8 58 #define EMAC_RXQ_CTRL_3 0x0015ac 59 #define EMAC_BASE_CPU_NUMBER 0x0015b8 60 #define EMAC_DMA_CTRL 0x0015c0 61 #define EMAC_MAILBOX_0 0x0015e0 62 #define EMAC_MAILBOX_5 0x0015e4 63 #define EMAC_MAILBOX_6 0x0015e8 64 #define EMAC_MAILBOX_13 0x0015ec 65 #define EMAC_MAILBOX_2 0x0015f4 66 #define EMAC_MAILBOX_3 0x0015f8 67 #define EMAC_MAILBOX_11 0x00160c 68 #define EMAC_AXI_MAST_CTRL 0x001610 69 #define EMAC_MAILBOX_12 0x001614 70 #define EMAC_MAILBOX_9 0x001618 71 #define EMAC_MAILBOX_10 0x00161c 72 #define EMAC_ATHR_HEADER_CTRL 0x001620 73 #define EMAC_CLK_GATE_CTRL 0x001814 74 #define EMAC_MISC_CTRL 0x001990 75 #define EMAC_MAILBOX_7 0x0019e0 76 #define EMAC_MAILBOX_8 0x0019e4 77 #define EMAC_MAILBOX_15 0x001bd4 78 #define EMAC_MAILBOX_16 0x001bd8 79 80 /* EMAC_MAC_CTRL */ 81 #define SINGLE_PAUSE_MODE 0x10000000 82 #define DEBUG_MODE 0x08000000 83 #define BROAD_EN 0x04000000 84 #define MULTI_ALL 0x02000000 85 #define RX_CHKSUM_EN 0x01000000 86 #define HUGE 0x00800000 87 #define SPEED(x) (((x) & 0x3) << 20) 88 #define SPEED_MASK SPEED(0x3) 89 #define SIMR 0x00080000 90 #define TPAUSE 0x00010000 91 #define PROM_MODE 0x00008000 92 #define VLAN_STRIP 0x00004000 93 #define PRLEN_BMSK 0x00003c00 94 #define PRLEN_SHFT 10 95 #define HUGEN 0x00000200 96 #define FLCHK 0x00000100 97 #define PCRCE 0x00000080 98 #define CRCE 0x00000040 99 #define FULLD 0x00000020 100 #define MAC_LP_EN 0x00000010 101 #define RXFC 0x00000008 102 #define TXFC 0x00000004 103 #define RXEN 0x00000002 104 #define TXEN 0x00000001 105 106 107 /* EMAC_WOL_CTRL0 */ 108 #define LK_CHG_PME 0x20 109 #define LK_CHG_EN 0x10 110 #define MG_FRAME_PME 0x8 111 #define MG_FRAME_EN 0x4 112 #define WK_FRAME_EN 0x1 113 114 /* EMAC_DESC_CTRL_3 */ 115 #define RFD_RING_SIZE_BMSK 0xfff 116 117 /* EMAC_DESC_CTRL_4 */ 118 #define RX_BUFFER_SIZE_BMSK 0xffff 119 120 /* EMAC_DESC_CTRL_6 */ 121 #define RRD_RING_SIZE_BMSK 0xfff 122 123 /* EMAC_DESC_CTRL_9 */ 124 #define TPD_RING_SIZE_BMSK 0xffff 125 126 /* EMAC_TXQ_CTRL_0 */ 127 #define NUM_TXF_BURST_PREF_BMSK 0xffff0000 128 #define NUM_TXF_BURST_PREF_SHFT 16 129 #define LS_8023_SP 0x80 130 #define TXQ_MODE 0x40 131 #define TXQ_EN 0x20 132 #define IP_OP_SP 0x10 133 #define NUM_TPD_BURST_PREF_BMSK 0xf 134 #define NUM_TPD_BURST_PREF_SHFT 0 135 136 /* EMAC_TXQ_CTRL_1 */ 137 #define JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK 0x7ff 138 139 /* EMAC_TXQ_CTRL_2 */ 140 #define TXF_HWM_BMSK 0xfff0000 141 #define TXF_LWM_BMSK 0xfff 142 143 /* EMAC_RXQ_CTRL_0 */ 144 #define RXQ_EN BIT(31) 145 #define CUT_THRU_EN BIT(30) 146 #define RSS_HASH_EN BIT(29) 147 #define NUM_RFD_BURST_PREF_BMSK 0x3f00000 148 #define NUM_RFD_BURST_PREF_SHFT 20 149 #define IDT_TABLE_SIZE_BMSK 0x1ff00 150 #define IDT_TABLE_SIZE_SHFT 8 151 #define SP_IPV6 0x80 152 153 /* EMAC_RXQ_CTRL_1 */ 154 #define JUMBO_1KAH_BMSK 0xf000 155 #define JUMBO_1KAH_SHFT 12 156 #define RFD_PREF_LOW_TH 0x10 157 #define RFD_PREF_LOW_THRESHOLD_BMSK 0xfc0 158 #define RFD_PREF_LOW_THRESHOLD_SHFT 6 159 #define RFD_PREF_UP_TH 0x10 160 #define RFD_PREF_UP_THRESHOLD_BMSK 0x3f 161 #define RFD_PREF_UP_THRESHOLD_SHFT 0 162 163 /* EMAC_RXQ_CTRL_2 */ 164 #define RXF_DOF_THRESFHOLD 0x1a0 165 #define RXF_DOF_THRESHOLD_BMSK 0xfff0000 166 #define RXF_DOF_THRESHOLD_SHFT 16 167 #define RXF_UOF_THRESFHOLD 0xbe 168 #define RXF_UOF_THRESHOLD_BMSK 0xfff 169 #define RXF_UOF_THRESHOLD_SHFT 0 170 171 /* EMAC_RXQ_CTRL_3 */ 172 #define RXD_TIMER_BMSK 0xffff0000 173 #define RXD_THRESHOLD_BMSK 0xfff 174 #define RXD_THRESHOLD_SHFT 0 175 176 /* EMAC_DMA_CTRL */ 177 #define DMAW_DLY_CNT_BMSK 0xf0000 178 #define DMAW_DLY_CNT_SHFT 16 179 #define DMAR_DLY_CNT_BMSK 0xf800 180 #define DMAR_DLY_CNT_SHFT 11 181 #define DMAR_REQ_PRI 0x400 182 #define REGWRBLEN_BMSK 0x380 183 #define REGWRBLEN_SHFT 7 184 #define REGRDBLEN_BMSK 0x70 185 #define REGRDBLEN_SHFT 4 186 #define OUT_ORDER_MODE 0x4 187 #define ENH_ORDER_MODE 0x2 188 #define IN_ORDER_MODE 0x1 189 190 /* EMAC_MAILBOX_13 */ 191 #define RFD3_PROC_IDX_BMSK 0xfff0000 192 #define RFD3_PROC_IDX_SHFT 16 193 #define RFD3_PROD_IDX_BMSK 0xfff 194 #define RFD3_PROD_IDX_SHFT 0 195 196 /* EMAC_MAILBOX_2 */ 197 #define NTPD_CONS_IDX_BMSK 0xffff0000 198 #define NTPD_CONS_IDX_SHFT 16 199 200 /* EMAC_MAILBOX_3 */ 201 #define RFD0_CONS_IDX_BMSK 0xfff 202 #define RFD0_CONS_IDX_SHFT 0 203 204 /* EMAC_MAILBOX_11 */ 205 #define H3TPD_PROD_IDX_BMSK 0xffff0000 206 #define H3TPD_PROD_IDX_SHFT 16 207 208 /* EMAC_AXI_MAST_CTRL */ 209 #define DATA_BYTE_SWAP 0x8 210 #define MAX_BOUND 0x2 211 #define MAX_BTYPE 0x1 212 213 /* EMAC_MAILBOX_12 */ 214 #define H3TPD_CONS_IDX_BMSK 0xffff0000 215 #define H3TPD_CONS_IDX_SHFT 16 216 217 /* EMAC_MAILBOX_9 */ 218 #define H2TPD_PROD_IDX_BMSK 0xffff 219 #define H2TPD_PROD_IDX_SHFT 0 220 221 /* EMAC_MAILBOX_10 */ 222 #define H1TPD_CONS_IDX_BMSK 0xffff0000 223 #define H1TPD_CONS_IDX_SHFT 16 224 #define H2TPD_CONS_IDX_BMSK 0xffff 225 #define H2TPD_CONS_IDX_SHFT 0 226 227 /* EMAC_ATHR_HEADER_CTRL */ 228 #define HEADER_CNT_EN 0x2 229 #define HEADER_ENABLE 0x1 230 231 /* EMAC_MAILBOX_0 */ 232 #define RFD0_PROC_IDX_BMSK 0xfff0000 233 #define RFD0_PROC_IDX_SHFT 16 234 #define RFD0_PROD_IDX_BMSK 0xfff 235 #define RFD0_PROD_IDX_SHFT 0 236 237 /* EMAC_MAILBOX_5 */ 238 #define RFD1_PROC_IDX_BMSK 0xfff0000 239 #define RFD1_PROC_IDX_SHFT 16 240 #define RFD1_PROD_IDX_BMSK 0xfff 241 #define RFD1_PROD_IDX_SHFT 0 242 243 /* EMAC_MISC_CTRL */ 244 #define RX_UNCPL_INT_EN 0x1 245 246 /* EMAC_MAILBOX_7 */ 247 #define RFD2_CONS_IDX_BMSK 0xfff0000 248 #define RFD2_CONS_IDX_SHFT 16 249 #define RFD1_CONS_IDX_BMSK 0xfff 250 #define RFD1_CONS_IDX_SHFT 0 251 252 /* EMAC_MAILBOX_8 */ 253 #define RFD3_CONS_IDX_BMSK 0xfff 254 #define RFD3_CONS_IDX_SHFT 0 255 256 /* EMAC_MAILBOX_15 */ 257 #define NTPD_PROD_IDX_BMSK 0xffff 258 #define NTPD_PROD_IDX_SHFT 0 259 260 /* EMAC_MAILBOX_16 */ 261 #define H1TPD_PROD_IDX_BMSK 0xffff 262 #define H1TPD_PROD_IDX_SHFT 0 263 264 #define RXQ0_RSS_HSTYP_IPV6_TCP_EN 0x20 265 #define RXQ0_RSS_HSTYP_IPV6_EN 0x10 266 #define RXQ0_RSS_HSTYP_IPV4_TCP_EN 0x8 267 #define RXQ0_RSS_HSTYP_IPV4_EN 0x4 268 269 /* EMAC_EMAC_WRAPPER_TX_TS_INX */ 270 #define EMAC_WRAPPER_TX_TS_EMPTY BIT(31) 271 #define EMAC_WRAPPER_TX_TS_INX_BMSK 0xffff 272 273 struct emac_skb_cb { 274 u32 tpd_idx; 275 unsigned long jiffies; 276 }; 277 278 #define EMAC_SKB_CB(skb) ((struct emac_skb_cb *)(skb)->cb) 279 #define EMAC_RSS_IDT_SIZE 256 280 #define JUMBO_1KAH 0x4 281 #define RXD_TH 0x100 282 #define EMAC_TPD_LAST_FRAGMENT 0x80000000 283 #define EMAC_TPD_TSTAMP_SAVE 0x80000000 284 285 /* EMAC Errors in emac_rrd.word[3] */ 286 #define EMAC_RRD_L4F BIT(14) 287 #define EMAC_RRD_IPF BIT(15) 288 #define EMAC_RRD_CRC BIT(21) 289 #define EMAC_RRD_FAE BIT(22) 290 #define EMAC_RRD_TRN BIT(23) 291 #define EMAC_RRD_RNT BIT(24) 292 #define EMAC_RRD_INC BIT(25) 293 #define EMAC_RRD_FOV BIT(29) 294 #define EMAC_RRD_LEN BIT(30) 295 296 /* Error bits that will result in a received frame being discarded */ 297 #define EMAC_RRD_ERROR (EMAC_RRD_IPF | EMAC_RRD_CRC | EMAC_RRD_FAE | \ 298 EMAC_RRD_TRN | EMAC_RRD_RNT | EMAC_RRD_INC | \ 299 EMAC_RRD_FOV | EMAC_RRD_LEN) 300 #define EMAC_RRD_STATS_DW_IDX 3 301 302 #define EMAC_RRD(RXQ, SIZE, IDX) ((RXQ)->rrd.v_addr + (SIZE * (IDX))) 303 #define EMAC_RFD(RXQ, SIZE, IDX) ((RXQ)->rfd.v_addr + (SIZE * (IDX))) 304 #define EMAC_TPD(TXQ, SIZE, IDX) ((TXQ)->tpd.v_addr + (SIZE * (IDX))) 305 306 #define GET_RFD_BUFFER(RXQ, IDX) (&((RXQ)->rfd.rfbuff[(IDX)])) 307 #define GET_TPD_BUFFER(RTQ, IDX) (&((RTQ)->tpd.tpbuff[(IDX)])) 308 309 #define EMAC_TX_POLL_HWTXTSTAMP_THRESHOLD 8 310 311 #define ISR_RX_PKT (\ 312 RX_PKT_INT0 |\ 313 RX_PKT_INT1 |\ 314 RX_PKT_INT2 |\ 315 RX_PKT_INT3) 316 317 #define EMAC_MAC_IRQ_RES "core0" 318 319 void emac_mac_multicast_addr_set(struct emac_adapter *adpt, u8 *addr) 320 { 321 u32 crc32, bit, reg, mta; 322 323 /* Calculate the CRC of the MAC address */ 324 crc32 = ether_crc(ETH_ALEN, addr); 325 326 /* The HASH Table is an array of 2 32-bit registers. It is 327 * treated like an array of 64 bits (BitArray[hash_value]). 328 * Use the upper 6 bits of the above CRC as the hash value. 329 */ 330 reg = (crc32 >> 31) & 0x1; 331 bit = (crc32 >> 26) & 0x1F; 332 333 mta = readl(adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2)); 334 mta |= BIT(bit); 335 writel(mta, adpt->base + EMAC_HASH_TAB_REG0 + (reg << 2)); 336 } 337 338 void emac_mac_multicast_addr_clear(struct emac_adapter *adpt) 339 { 340 writel(0, adpt->base + EMAC_HASH_TAB_REG0); 341 writel(0, adpt->base + EMAC_HASH_TAB_REG1); 342 } 343 344 /* definitions for RSS */ 345 #define EMAC_RSS_KEY(_i, _type) \ 346 (EMAC_RSS_KEY0 + ((_i) * sizeof(_type))) 347 #define EMAC_RSS_TBL(_i, _type) \ 348 (EMAC_IDT_TABLE0 + ((_i) * sizeof(_type))) 349 350 /* Config MAC modes */ 351 void emac_mac_mode_config(struct emac_adapter *adpt) 352 { 353 struct net_device *netdev = adpt->netdev; 354 u32 mac; 355 356 mac = readl(adpt->base + EMAC_MAC_CTRL); 357 mac &= ~(VLAN_STRIP | PROM_MODE | MULTI_ALL | MAC_LP_EN); 358 359 if (netdev->features & NETIF_F_HW_VLAN_CTAG_RX) 360 mac |= VLAN_STRIP; 361 362 if (netdev->flags & IFF_PROMISC) 363 mac |= PROM_MODE; 364 365 if (netdev->flags & IFF_ALLMULTI) 366 mac |= MULTI_ALL; 367 368 writel(mac, adpt->base + EMAC_MAC_CTRL); 369 } 370 371 /* Config descriptor rings */ 372 static void emac_mac_dma_rings_config(struct emac_adapter *adpt) 373 { 374 static const unsigned short tpd_q_offset[] = { 375 EMAC_DESC_CTRL_8, EMAC_H1TPD_BASE_ADDR_LO, 376 EMAC_H2TPD_BASE_ADDR_LO, EMAC_H3TPD_BASE_ADDR_LO}; 377 static const unsigned short rfd_q_offset[] = { 378 EMAC_DESC_CTRL_2, EMAC_DESC_CTRL_10, 379 EMAC_DESC_CTRL_12, EMAC_DESC_CTRL_13}; 380 static const unsigned short rrd_q_offset[] = { 381 EMAC_DESC_CTRL_5, EMAC_DESC_CTRL_14, 382 EMAC_DESC_CTRL_15, EMAC_DESC_CTRL_16}; 383 384 /* TPD (Transmit Packet Descriptor) */ 385 writel(upper_32_bits(adpt->tx_q.tpd.dma_addr), 386 adpt->base + EMAC_DESC_CTRL_1); 387 388 writel(lower_32_bits(adpt->tx_q.tpd.dma_addr), 389 adpt->base + tpd_q_offset[0]); 390 391 writel(adpt->tx_q.tpd.count & TPD_RING_SIZE_BMSK, 392 adpt->base + EMAC_DESC_CTRL_9); 393 394 /* RFD (Receive Free Descriptor) & RRD (Receive Return Descriptor) */ 395 writel(upper_32_bits(adpt->rx_q.rfd.dma_addr), 396 adpt->base + EMAC_DESC_CTRL_0); 397 398 writel(lower_32_bits(adpt->rx_q.rfd.dma_addr), 399 adpt->base + rfd_q_offset[0]); 400 writel(lower_32_bits(adpt->rx_q.rrd.dma_addr), 401 adpt->base + rrd_q_offset[0]); 402 403 writel(adpt->rx_q.rfd.count & RFD_RING_SIZE_BMSK, 404 adpt->base + EMAC_DESC_CTRL_3); 405 writel(adpt->rx_q.rrd.count & RRD_RING_SIZE_BMSK, 406 adpt->base + EMAC_DESC_CTRL_6); 407 408 writel(adpt->rxbuf_size & RX_BUFFER_SIZE_BMSK, 409 adpt->base + EMAC_DESC_CTRL_4); 410 411 writel(0, adpt->base + EMAC_DESC_CTRL_11); 412 413 /* Load all of the base addresses above and ensure that triggering HW to 414 * read ring pointers is flushed 415 */ 416 writel(1, adpt->base + EMAC_INTER_SRAM_PART9); 417 } 418 419 /* Config transmit parameters */ 420 static void emac_mac_tx_config(struct emac_adapter *adpt) 421 { 422 u32 val; 423 424 writel((EMAC_MAX_TX_OFFLOAD_THRESH >> 3) & 425 JUMBO_TASK_OFFLOAD_THRESHOLD_BMSK, adpt->base + EMAC_TXQ_CTRL_1); 426 427 val = (adpt->tpd_burst << NUM_TPD_BURST_PREF_SHFT) & 428 NUM_TPD_BURST_PREF_BMSK; 429 430 val |= TXQ_MODE | LS_8023_SP; 431 val |= (0x0100 << NUM_TXF_BURST_PREF_SHFT) & 432 NUM_TXF_BURST_PREF_BMSK; 433 434 writel(val, adpt->base + EMAC_TXQ_CTRL_0); 435 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_2, 436 (TXF_HWM_BMSK | TXF_LWM_BMSK), 0); 437 } 438 439 /* Config receive parameters */ 440 static void emac_mac_rx_config(struct emac_adapter *adpt) 441 { 442 u32 val; 443 444 val = (adpt->rfd_burst << NUM_RFD_BURST_PREF_SHFT) & 445 NUM_RFD_BURST_PREF_BMSK; 446 val |= (SP_IPV6 | CUT_THRU_EN); 447 448 writel(val, adpt->base + EMAC_RXQ_CTRL_0); 449 450 val = readl(adpt->base + EMAC_RXQ_CTRL_1); 451 val &= ~(JUMBO_1KAH_BMSK | RFD_PREF_LOW_THRESHOLD_BMSK | 452 RFD_PREF_UP_THRESHOLD_BMSK); 453 val |= (JUMBO_1KAH << JUMBO_1KAH_SHFT) | 454 (RFD_PREF_LOW_TH << RFD_PREF_LOW_THRESHOLD_SHFT) | 455 (RFD_PREF_UP_TH << RFD_PREF_UP_THRESHOLD_SHFT); 456 writel(val, adpt->base + EMAC_RXQ_CTRL_1); 457 458 val = readl(adpt->base + EMAC_RXQ_CTRL_2); 459 val &= ~(RXF_DOF_THRESHOLD_BMSK | RXF_UOF_THRESHOLD_BMSK); 460 val |= (RXF_DOF_THRESFHOLD << RXF_DOF_THRESHOLD_SHFT) | 461 (RXF_UOF_THRESFHOLD << RXF_UOF_THRESHOLD_SHFT); 462 writel(val, adpt->base + EMAC_RXQ_CTRL_2); 463 464 val = readl(adpt->base + EMAC_RXQ_CTRL_3); 465 val &= ~(RXD_TIMER_BMSK | RXD_THRESHOLD_BMSK); 466 val |= RXD_TH << RXD_THRESHOLD_SHFT; 467 writel(val, adpt->base + EMAC_RXQ_CTRL_3); 468 } 469 470 /* Config dma */ 471 static void emac_mac_dma_config(struct emac_adapter *adpt) 472 { 473 u32 dma_ctrl = DMAR_REQ_PRI; 474 475 switch (adpt->dma_order) { 476 case emac_dma_ord_in: 477 dma_ctrl |= IN_ORDER_MODE; 478 break; 479 case emac_dma_ord_enh: 480 dma_ctrl |= ENH_ORDER_MODE; 481 break; 482 case emac_dma_ord_out: 483 dma_ctrl |= OUT_ORDER_MODE; 484 break; 485 default: 486 break; 487 } 488 489 dma_ctrl |= (((u32)adpt->dmar_block) << REGRDBLEN_SHFT) & 490 REGRDBLEN_BMSK; 491 dma_ctrl |= (((u32)adpt->dmaw_block) << REGWRBLEN_SHFT) & 492 REGWRBLEN_BMSK; 493 dma_ctrl |= (((u32)adpt->dmar_dly_cnt) << DMAR_DLY_CNT_SHFT) & 494 DMAR_DLY_CNT_BMSK; 495 dma_ctrl |= (((u32)adpt->dmaw_dly_cnt) << DMAW_DLY_CNT_SHFT) & 496 DMAW_DLY_CNT_BMSK; 497 498 /* config DMA and ensure that configuration is flushed to HW */ 499 writel(dma_ctrl, adpt->base + EMAC_DMA_CTRL); 500 } 501 502 /* set MAC address */ 503 static void emac_set_mac_address(struct emac_adapter *adpt, u8 *addr) 504 { 505 u32 sta; 506 507 /* for example: 00-A0-C6-11-22-33 508 * 0<-->C6112233, 1<-->00A0. 509 */ 510 511 /* low 32bit word */ 512 sta = (((u32)addr[2]) << 24) | (((u32)addr[3]) << 16) | 513 (((u32)addr[4]) << 8) | (((u32)addr[5])); 514 writel(sta, adpt->base + EMAC_MAC_STA_ADDR0); 515 516 /* hight 32bit word */ 517 sta = (((u32)addr[0]) << 8) | (u32)addr[1]; 518 writel(sta, adpt->base + EMAC_MAC_STA_ADDR1); 519 } 520 521 static void emac_mac_config(struct emac_adapter *adpt) 522 { 523 struct net_device *netdev = adpt->netdev; 524 unsigned int max_frame; 525 u32 val; 526 527 emac_set_mac_address(adpt, netdev->dev_addr); 528 529 max_frame = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; 530 adpt->rxbuf_size = netdev->mtu > EMAC_DEF_RX_BUF_SIZE ? 531 ALIGN(max_frame, 8) : EMAC_DEF_RX_BUF_SIZE; 532 533 emac_mac_dma_rings_config(adpt); 534 535 writel(netdev->mtu + ETH_HLEN + VLAN_HLEN + ETH_FCS_LEN, 536 adpt->base + EMAC_MAX_FRAM_LEN_CTRL); 537 538 emac_mac_tx_config(adpt); 539 emac_mac_rx_config(adpt); 540 emac_mac_dma_config(adpt); 541 542 val = readl(adpt->base + EMAC_AXI_MAST_CTRL); 543 val &= ~(DATA_BYTE_SWAP | MAX_BOUND); 544 val |= MAX_BTYPE; 545 writel(val, adpt->base + EMAC_AXI_MAST_CTRL); 546 writel(0, adpt->base + EMAC_CLK_GATE_CTRL); 547 writel(RX_UNCPL_INT_EN, adpt->base + EMAC_MISC_CTRL); 548 } 549 550 void emac_mac_reset(struct emac_adapter *adpt) 551 { 552 emac_mac_stop(adpt); 553 554 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, SOFT_RST); 555 usleep_range(100, 150); /* reset may take up to 100usec */ 556 557 /* interrupt clear-on-read */ 558 emac_reg_update32(adpt->base + EMAC_DMA_MAS_CTRL, 0, INT_RD_CLR_EN); 559 } 560 561 void emac_mac_start(struct emac_adapter *adpt) 562 { 563 struct phy_device *phydev = adpt->phydev; 564 u32 mac, csr1; 565 566 /* enable tx queue */ 567 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, 0, TXQ_EN); 568 569 /* enable rx queue */ 570 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, 0, RXQ_EN); 571 572 /* enable mac control */ 573 mac = readl(adpt->base + EMAC_MAC_CTRL); 574 csr1 = readl(adpt->csr + EMAC_EMAC_WRAPPER_CSR1); 575 576 mac |= TXEN | RXEN; /* enable RX/TX */ 577 578 /* Configure MAC flow control to match the PHY's settings. */ 579 if (phydev->pause) 580 mac |= RXFC; 581 if (phydev->pause != phydev->asym_pause) 582 mac |= TXFC; 583 584 /* setup link speed */ 585 mac &= ~SPEED_MASK; 586 if (phydev->speed == SPEED_1000) { 587 mac |= SPEED(2); 588 csr1 |= FREQ_MODE; 589 } else { 590 mac |= SPEED(1); 591 csr1 &= ~FREQ_MODE; 592 } 593 594 if (phydev->duplex == DUPLEX_FULL) 595 mac |= FULLD; 596 else 597 mac &= ~FULLD; 598 599 /* other parameters */ 600 mac |= (CRCE | PCRCE); 601 mac |= ((adpt->preamble << PRLEN_SHFT) & PRLEN_BMSK); 602 mac |= BROAD_EN; 603 mac |= FLCHK; 604 mac &= ~RX_CHKSUM_EN; 605 mac &= ~(HUGEN | VLAN_STRIP | TPAUSE | SIMR | HUGE | MULTI_ALL | 606 DEBUG_MODE | SINGLE_PAUSE_MODE); 607 608 writel_relaxed(csr1, adpt->csr + EMAC_EMAC_WRAPPER_CSR1); 609 610 writel_relaxed(mac, adpt->base + EMAC_MAC_CTRL); 611 612 /* enable interrupt read clear, low power sleep mode and 613 * the irq moderators 614 */ 615 616 writel_relaxed(adpt->irq_mod, adpt->base + EMAC_IRQ_MOD_TIM_INIT); 617 writel_relaxed(INT_RD_CLR_EN | LPW_MODE | IRQ_MODERATOR_EN | 618 IRQ_MODERATOR2_EN, adpt->base + EMAC_DMA_MAS_CTRL); 619 620 emac_mac_mode_config(adpt); 621 622 emac_reg_update32(adpt->base + EMAC_ATHR_HEADER_CTRL, 623 (HEADER_ENABLE | HEADER_CNT_EN), 0); 624 625 emac_reg_update32(adpt->csr + EMAC_EMAC_WRAPPER_CSR2, 0, WOL_EN); 626 } 627 628 void emac_mac_stop(struct emac_adapter *adpt) 629 { 630 emac_reg_update32(adpt->base + EMAC_RXQ_CTRL_0, RXQ_EN, 0); 631 emac_reg_update32(adpt->base + EMAC_TXQ_CTRL_0, TXQ_EN, 0); 632 emac_reg_update32(adpt->base + EMAC_MAC_CTRL, TXEN | RXEN, 0); 633 usleep_range(1000, 1050); /* stopping mac may take upto 1msec */ 634 } 635 636 /* Free all descriptors of given transmit queue */ 637 static void emac_tx_q_descs_free(struct emac_adapter *adpt) 638 { 639 struct emac_tx_queue *tx_q = &adpt->tx_q; 640 unsigned int i; 641 size_t size; 642 643 /* ring already cleared, nothing to do */ 644 if (!tx_q->tpd.tpbuff) 645 return; 646 647 for (i = 0; i < tx_q->tpd.count; i++) { 648 struct emac_buffer *tpbuf = GET_TPD_BUFFER(tx_q, i); 649 650 if (tpbuf->dma_addr) { 651 dma_unmap_single(adpt->netdev->dev.parent, 652 tpbuf->dma_addr, tpbuf->length, 653 DMA_TO_DEVICE); 654 tpbuf->dma_addr = 0; 655 } 656 if (tpbuf->skb) { 657 dev_kfree_skb_any(tpbuf->skb); 658 tpbuf->skb = NULL; 659 } 660 } 661 662 size = sizeof(struct emac_buffer) * tx_q->tpd.count; 663 memset(tx_q->tpd.tpbuff, 0, size); 664 665 /* clear the descriptor ring */ 666 memset(tx_q->tpd.v_addr, 0, tx_q->tpd.size); 667 668 tx_q->tpd.consume_idx = 0; 669 tx_q->tpd.produce_idx = 0; 670 } 671 672 /* Free all descriptors of given receive queue */ 673 static void emac_rx_q_free_descs(struct emac_adapter *adpt) 674 { 675 struct device *dev = adpt->netdev->dev.parent; 676 struct emac_rx_queue *rx_q = &adpt->rx_q; 677 unsigned int i; 678 size_t size; 679 680 /* ring already cleared, nothing to do */ 681 if (!rx_q->rfd.rfbuff) 682 return; 683 684 for (i = 0; i < rx_q->rfd.count; i++) { 685 struct emac_buffer *rfbuf = GET_RFD_BUFFER(rx_q, i); 686 687 if (rfbuf->dma_addr) { 688 dma_unmap_single(dev, rfbuf->dma_addr, rfbuf->length, 689 DMA_FROM_DEVICE); 690 rfbuf->dma_addr = 0; 691 } 692 if (rfbuf->skb) { 693 dev_kfree_skb(rfbuf->skb); 694 rfbuf->skb = NULL; 695 } 696 } 697 698 size = sizeof(struct emac_buffer) * rx_q->rfd.count; 699 memset(rx_q->rfd.rfbuff, 0, size); 700 701 /* clear the descriptor rings */ 702 memset(rx_q->rrd.v_addr, 0, rx_q->rrd.size); 703 rx_q->rrd.produce_idx = 0; 704 rx_q->rrd.consume_idx = 0; 705 706 memset(rx_q->rfd.v_addr, 0, rx_q->rfd.size); 707 rx_q->rfd.produce_idx = 0; 708 rx_q->rfd.consume_idx = 0; 709 } 710 711 /* Free all buffers associated with given transmit queue */ 712 static void emac_tx_q_bufs_free(struct emac_adapter *adpt) 713 { 714 struct emac_tx_queue *tx_q = &adpt->tx_q; 715 716 emac_tx_q_descs_free(adpt); 717 718 kfree(tx_q->tpd.tpbuff); 719 tx_q->tpd.tpbuff = NULL; 720 tx_q->tpd.v_addr = NULL; 721 tx_q->tpd.dma_addr = 0; 722 tx_q->tpd.size = 0; 723 } 724 725 /* Allocate TX descriptor ring for the given transmit queue */ 726 static int emac_tx_q_desc_alloc(struct emac_adapter *adpt, 727 struct emac_tx_queue *tx_q) 728 { 729 struct emac_ring_header *ring_header = &adpt->ring_header; 730 size_t size; 731 732 size = sizeof(struct emac_buffer) * tx_q->tpd.count; 733 tx_q->tpd.tpbuff = kzalloc(size, GFP_KERNEL); 734 if (!tx_q->tpd.tpbuff) 735 return -ENOMEM; 736 737 tx_q->tpd.size = tx_q->tpd.count * (adpt->tpd_size * 4); 738 tx_q->tpd.dma_addr = ring_header->dma_addr + ring_header->used; 739 tx_q->tpd.v_addr = ring_header->v_addr + ring_header->used; 740 ring_header->used += ALIGN(tx_q->tpd.size, 8); 741 tx_q->tpd.produce_idx = 0; 742 tx_q->tpd.consume_idx = 0; 743 744 return 0; 745 } 746 747 /* Free all buffers associated with given transmit queue */ 748 static void emac_rx_q_bufs_free(struct emac_adapter *adpt) 749 { 750 struct emac_rx_queue *rx_q = &adpt->rx_q; 751 752 emac_rx_q_free_descs(adpt); 753 754 kfree(rx_q->rfd.rfbuff); 755 rx_q->rfd.rfbuff = NULL; 756 757 rx_q->rfd.v_addr = NULL; 758 rx_q->rfd.dma_addr = 0; 759 rx_q->rfd.size = 0; 760 761 rx_q->rrd.v_addr = NULL; 762 rx_q->rrd.dma_addr = 0; 763 rx_q->rrd.size = 0; 764 } 765 766 /* Allocate RX descriptor rings for the given receive queue */ 767 static int emac_rx_descs_alloc(struct emac_adapter *adpt) 768 { 769 struct emac_ring_header *ring_header = &adpt->ring_header; 770 struct emac_rx_queue *rx_q = &adpt->rx_q; 771 size_t size; 772 773 size = sizeof(struct emac_buffer) * rx_q->rfd.count; 774 rx_q->rfd.rfbuff = kzalloc(size, GFP_KERNEL); 775 if (!rx_q->rfd.rfbuff) 776 return -ENOMEM; 777 778 rx_q->rrd.size = rx_q->rrd.count * (adpt->rrd_size * 4); 779 rx_q->rfd.size = rx_q->rfd.count * (adpt->rfd_size * 4); 780 781 rx_q->rrd.dma_addr = ring_header->dma_addr + ring_header->used; 782 rx_q->rrd.v_addr = ring_header->v_addr + ring_header->used; 783 ring_header->used += ALIGN(rx_q->rrd.size, 8); 784 785 rx_q->rfd.dma_addr = ring_header->dma_addr + ring_header->used; 786 rx_q->rfd.v_addr = ring_header->v_addr + ring_header->used; 787 ring_header->used += ALIGN(rx_q->rfd.size, 8); 788 789 rx_q->rrd.produce_idx = 0; 790 rx_q->rrd.consume_idx = 0; 791 792 rx_q->rfd.produce_idx = 0; 793 rx_q->rfd.consume_idx = 0; 794 795 return 0; 796 } 797 798 /* Allocate all TX and RX descriptor rings */ 799 int emac_mac_rx_tx_rings_alloc_all(struct emac_adapter *adpt) 800 { 801 struct emac_ring_header *ring_header = &adpt->ring_header; 802 struct device *dev = adpt->netdev->dev.parent; 803 unsigned int num_tx_descs = adpt->tx_desc_cnt; 804 unsigned int num_rx_descs = adpt->rx_desc_cnt; 805 int ret; 806 807 adpt->tx_q.tpd.count = adpt->tx_desc_cnt; 808 809 adpt->rx_q.rrd.count = adpt->rx_desc_cnt; 810 adpt->rx_q.rfd.count = adpt->rx_desc_cnt; 811 812 /* Ring DMA buffer. Each ring may need up to 8 bytes for alignment, 813 * hence the additional padding bytes are allocated. 814 */ 815 ring_header->size = num_tx_descs * (adpt->tpd_size * 4) + 816 num_rx_descs * (adpt->rfd_size * 4) + 817 num_rx_descs * (adpt->rrd_size * 4) + 818 8 + 2 * 8; /* 8 byte per one Tx and two Rx rings */ 819 820 ring_header->used = 0; 821 ring_header->v_addr = dma_zalloc_coherent(dev, ring_header->size, 822 &ring_header->dma_addr, 823 GFP_KERNEL); 824 if (!ring_header->v_addr) 825 return -ENOMEM; 826 827 ring_header->used = ALIGN(ring_header->dma_addr, 8) - 828 ring_header->dma_addr; 829 830 ret = emac_tx_q_desc_alloc(adpt, &adpt->tx_q); 831 if (ret) { 832 netdev_err(adpt->netdev, "error: Tx Queue alloc failed\n"); 833 goto err_alloc_tx; 834 } 835 836 ret = emac_rx_descs_alloc(adpt); 837 if (ret) { 838 netdev_err(adpt->netdev, "error: Rx Queue alloc failed\n"); 839 goto err_alloc_rx; 840 } 841 842 return 0; 843 844 err_alloc_rx: 845 emac_tx_q_bufs_free(adpt); 846 err_alloc_tx: 847 dma_free_coherent(dev, ring_header->size, 848 ring_header->v_addr, ring_header->dma_addr); 849 850 ring_header->v_addr = NULL; 851 ring_header->dma_addr = 0; 852 ring_header->size = 0; 853 ring_header->used = 0; 854 855 return ret; 856 } 857 858 /* Free all TX and RX descriptor rings */ 859 void emac_mac_rx_tx_rings_free_all(struct emac_adapter *adpt) 860 { 861 struct emac_ring_header *ring_header = &adpt->ring_header; 862 struct device *dev = adpt->netdev->dev.parent; 863 864 emac_tx_q_bufs_free(adpt); 865 emac_rx_q_bufs_free(adpt); 866 867 dma_free_coherent(dev, ring_header->size, 868 ring_header->v_addr, ring_header->dma_addr); 869 870 ring_header->v_addr = NULL; 871 ring_header->dma_addr = 0; 872 ring_header->size = 0; 873 ring_header->used = 0; 874 } 875 876 /* Initialize descriptor rings */ 877 static void emac_mac_rx_tx_ring_reset_all(struct emac_adapter *adpt) 878 { 879 unsigned int i; 880 881 adpt->tx_q.tpd.produce_idx = 0; 882 adpt->tx_q.tpd.consume_idx = 0; 883 for (i = 0; i < adpt->tx_q.tpd.count; i++) 884 adpt->tx_q.tpd.tpbuff[i].dma_addr = 0; 885 886 adpt->rx_q.rrd.produce_idx = 0; 887 adpt->rx_q.rrd.consume_idx = 0; 888 adpt->rx_q.rfd.produce_idx = 0; 889 adpt->rx_q.rfd.consume_idx = 0; 890 for (i = 0; i < adpt->rx_q.rfd.count; i++) 891 adpt->rx_q.rfd.rfbuff[i].dma_addr = 0; 892 } 893 894 /* Produce new receive free descriptor */ 895 static void emac_mac_rx_rfd_create(struct emac_adapter *adpt, 896 struct emac_rx_queue *rx_q, 897 dma_addr_t addr) 898 { 899 u32 *hw_rfd = EMAC_RFD(rx_q, adpt->rfd_size, rx_q->rfd.produce_idx); 900 901 *(hw_rfd++) = lower_32_bits(addr); 902 *hw_rfd = upper_32_bits(addr); 903 904 if (++rx_q->rfd.produce_idx == rx_q->rfd.count) 905 rx_q->rfd.produce_idx = 0; 906 } 907 908 /* Fill up receive queue's RFD with preallocated receive buffers */ 909 static void emac_mac_rx_descs_refill(struct emac_adapter *adpt, 910 struct emac_rx_queue *rx_q) 911 { 912 struct emac_buffer *curr_rxbuf; 913 struct emac_buffer *next_rxbuf; 914 unsigned int count = 0; 915 u32 next_produce_idx; 916 917 next_produce_idx = rx_q->rfd.produce_idx + 1; 918 if (next_produce_idx == rx_q->rfd.count) 919 next_produce_idx = 0; 920 921 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx); 922 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx); 923 924 /* this always has a blank rx_buffer*/ 925 while (!next_rxbuf->dma_addr) { 926 struct sk_buff *skb; 927 int ret; 928 929 skb = netdev_alloc_skb_ip_align(adpt->netdev, adpt->rxbuf_size); 930 if (!skb) 931 break; 932 933 curr_rxbuf->dma_addr = 934 dma_map_single(adpt->netdev->dev.parent, skb->data, 935 curr_rxbuf->length, DMA_FROM_DEVICE); 936 ret = dma_mapping_error(adpt->netdev->dev.parent, 937 curr_rxbuf->dma_addr); 938 if (ret) { 939 dev_kfree_skb(skb); 940 break; 941 } 942 curr_rxbuf->skb = skb; 943 curr_rxbuf->length = adpt->rxbuf_size; 944 945 emac_mac_rx_rfd_create(adpt, rx_q, curr_rxbuf->dma_addr); 946 next_produce_idx = rx_q->rfd.produce_idx + 1; 947 if (next_produce_idx == rx_q->rfd.count) 948 next_produce_idx = 0; 949 950 curr_rxbuf = GET_RFD_BUFFER(rx_q, rx_q->rfd.produce_idx); 951 next_rxbuf = GET_RFD_BUFFER(rx_q, next_produce_idx); 952 count++; 953 } 954 955 if (count) { 956 u32 prod_idx = (rx_q->rfd.produce_idx << rx_q->produce_shift) & 957 rx_q->produce_mask; 958 emac_reg_update32(adpt->base + rx_q->produce_reg, 959 rx_q->produce_mask, prod_idx); 960 } 961 } 962 963 static void emac_adjust_link(struct net_device *netdev) 964 { 965 struct emac_adapter *adpt = netdev_priv(netdev); 966 struct phy_device *phydev = netdev->phydev; 967 968 if (phydev->link) 969 emac_mac_start(adpt); 970 else 971 emac_mac_stop(adpt); 972 973 phy_print_status(phydev); 974 } 975 976 /* Bringup the interface/HW */ 977 int emac_mac_up(struct emac_adapter *adpt) 978 { 979 struct net_device *netdev = adpt->netdev; 980 struct emac_irq *irq = &adpt->irq; 981 int ret; 982 983 emac_mac_rx_tx_ring_reset_all(adpt); 984 emac_mac_config(adpt); 985 986 ret = request_irq(irq->irq, emac_isr, 0, EMAC_MAC_IRQ_RES, irq); 987 if (ret) { 988 netdev_err(adpt->netdev, "could not request %s irq\n", 989 EMAC_MAC_IRQ_RES); 990 return ret; 991 } 992 993 emac_mac_rx_descs_refill(adpt, &adpt->rx_q); 994 995 ret = phy_connect_direct(netdev, adpt->phydev, emac_adjust_link, 996 PHY_INTERFACE_MODE_SGMII); 997 if (ret) { 998 netdev_err(adpt->netdev, "could not connect phy\n"); 999 free_irq(irq->irq, irq); 1000 return ret; 1001 } 1002 1003 /* enable mac irq */ 1004 writel((u32)~DIS_INT, adpt->base + EMAC_INT_STATUS); 1005 writel(adpt->irq.mask, adpt->base + EMAC_INT_MASK); 1006 1007 /* Enable pause frames. Without this feature, the EMAC has been shown 1008 * to receive (and drop) frames with FCS errors at gigabit connections. 1009 */ 1010 adpt->phydev->supported |= SUPPORTED_Pause | SUPPORTED_Asym_Pause; 1011 adpt->phydev->advertising |= SUPPORTED_Pause | SUPPORTED_Asym_Pause; 1012 1013 adpt->phydev->irq = PHY_IGNORE_INTERRUPT; 1014 phy_start(adpt->phydev); 1015 1016 napi_enable(&adpt->rx_q.napi); 1017 netif_start_queue(netdev); 1018 1019 return 0; 1020 } 1021 1022 /* Bring down the interface/HW */ 1023 void emac_mac_down(struct emac_adapter *adpt) 1024 { 1025 struct net_device *netdev = adpt->netdev; 1026 1027 netif_stop_queue(netdev); 1028 napi_disable(&adpt->rx_q.napi); 1029 1030 phy_stop(adpt->phydev); 1031 1032 /* Interrupts must be disabled before the PHY is disconnected, to 1033 * avoid a race condition where adjust_link is null when we get 1034 * an interrupt. 1035 */ 1036 writel(DIS_INT, adpt->base + EMAC_INT_STATUS); 1037 writel(0, adpt->base + EMAC_INT_MASK); 1038 synchronize_irq(adpt->irq.irq); 1039 free_irq(adpt->irq.irq, &adpt->irq); 1040 1041 phy_disconnect(adpt->phydev); 1042 1043 emac_mac_reset(adpt); 1044 1045 emac_tx_q_descs_free(adpt); 1046 netdev_reset_queue(adpt->netdev); 1047 emac_rx_q_free_descs(adpt); 1048 } 1049 1050 /* Consume next received packet descriptor */ 1051 static bool emac_rx_process_rrd(struct emac_adapter *adpt, 1052 struct emac_rx_queue *rx_q, 1053 struct emac_rrd *rrd) 1054 { 1055 u32 *hw_rrd = EMAC_RRD(rx_q, adpt->rrd_size, rx_q->rrd.consume_idx); 1056 1057 rrd->word[3] = *(hw_rrd + 3); 1058 1059 if (!RRD_UPDT(rrd)) 1060 return false; 1061 1062 rrd->word[4] = 0; 1063 rrd->word[5] = 0; 1064 1065 rrd->word[0] = *(hw_rrd++); 1066 rrd->word[1] = *(hw_rrd++); 1067 rrd->word[2] = *(hw_rrd++); 1068 1069 if (unlikely(RRD_NOR(rrd) != 1)) { 1070 netdev_err(adpt->netdev, 1071 "error: multi-RFD not support yet! nor:%lu\n", 1072 RRD_NOR(rrd)); 1073 } 1074 1075 /* mark rrd as processed */ 1076 RRD_UPDT_SET(rrd, 0); 1077 *hw_rrd = rrd->word[3]; 1078 1079 if (++rx_q->rrd.consume_idx == rx_q->rrd.count) 1080 rx_q->rrd.consume_idx = 0; 1081 1082 return true; 1083 } 1084 1085 /* Produce new transmit descriptor */ 1086 static void emac_tx_tpd_create(struct emac_adapter *adpt, 1087 struct emac_tx_queue *tx_q, struct emac_tpd *tpd) 1088 { 1089 u32 *hw_tpd; 1090 1091 tx_q->tpd.last_produce_idx = tx_q->tpd.produce_idx; 1092 hw_tpd = EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.produce_idx); 1093 1094 if (++tx_q->tpd.produce_idx == tx_q->tpd.count) 1095 tx_q->tpd.produce_idx = 0; 1096 1097 *(hw_tpd++) = tpd->word[0]; 1098 *(hw_tpd++) = tpd->word[1]; 1099 *(hw_tpd++) = tpd->word[2]; 1100 *hw_tpd = tpd->word[3]; 1101 } 1102 1103 /* Mark the last transmit descriptor as such (for the transmit packet) */ 1104 static void emac_tx_tpd_mark_last(struct emac_adapter *adpt, 1105 struct emac_tx_queue *tx_q) 1106 { 1107 u32 *hw_tpd = 1108 EMAC_TPD(tx_q, adpt->tpd_size, tx_q->tpd.last_produce_idx); 1109 u32 tmp_tpd; 1110 1111 tmp_tpd = *(hw_tpd + 1); 1112 tmp_tpd |= EMAC_TPD_LAST_FRAGMENT; 1113 *(hw_tpd + 1) = tmp_tpd; 1114 } 1115 1116 static void emac_rx_rfd_clean(struct emac_rx_queue *rx_q, struct emac_rrd *rrd) 1117 { 1118 struct emac_buffer *rfbuf = rx_q->rfd.rfbuff; 1119 u32 consume_idx = RRD_SI(rrd); 1120 unsigned int i; 1121 1122 for (i = 0; i < RRD_NOR(rrd); i++) { 1123 rfbuf[consume_idx].skb = NULL; 1124 if (++consume_idx == rx_q->rfd.count) 1125 consume_idx = 0; 1126 } 1127 1128 rx_q->rfd.consume_idx = consume_idx; 1129 rx_q->rfd.process_idx = consume_idx; 1130 } 1131 1132 /* Push the received skb to upper layers */ 1133 static void emac_receive_skb(struct emac_rx_queue *rx_q, 1134 struct sk_buff *skb, 1135 u16 vlan_tag, bool vlan_flag) 1136 { 1137 if (vlan_flag) { 1138 u16 vlan; 1139 1140 EMAC_TAG_TO_VLAN(vlan_tag, vlan); 1141 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan); 1142 } 1143 1144 napi_gro_receive(&rx_q->napi, skb); 1145 } 1146 1147 /* Process receive event */ 1148 void emac_mac_rx_process(struct emac_adapter *adpt, struct emac_rx_queue *rx_q, 1149 int *num_pkts, int max_pkts) 1150 { 1151 u32 proc_idx, hw_consume_idx, num_consume_pkts; 1152 struct net_device *netdev = adpt->netdev; 1153 struct emac_buffer *rfbuf; 1154 unsigned int count = 0; 1155 struct emac_rrd rrd; 1156 struct sk_buff *skb; 1157 u32 reg; 1158 1159 reg = readl_relaxed(adpt->base + rx_q->consume_reg); 1160 1161 hw_consume_idx = (reg & rx_q->consume_mask) >> rx_q->consume_shift; 1162 num_consume_pkts = (hw_consume_idx >= rx_q->rrd.consume_idx) ? 1163 (hw_consume_idx - rx_q->rrd.consume_idx) : 1164 (hw_consume_idx + rx_q->rrd.count - rx_q->rrd.consume_idx); 1165 1166 do { 1167 if (!num_consume_pkts) 1168 break; 1169 1170 if (!emac_rx_process_rrd(adpt, rx_q, &rrd)) 1171 break; 1172 1173 if (likely(RRD_NOR(&rrd) == 1)) { 1174 /* good receive */ 1175 rfbuf = GET_RFD_BUFFER(rx_q, RRD_SI(&rrd)); 1176 dma_unmap_single(adpt->netdev->dev.parent, 1177 rfbuf->dma_addr, rfbuf->length, 1178 DMA_FROM_DEVICE); 1179 rfbuf->dma_addr = 0; 1180 skb = rfbuf->skb; 1181 } else { 1182 netdev_err(adpt->netdev, 1183 "error: multi-RFD not support yet!\n"); 1184 break; 1185 } 1186 emac_rx_rfd_clean(rx_q, &rrd); 1187 num_consume_pkts--; 1188 count++; 1189 1190 /* Due to a HW issue in L4 check sum detection (UDP/TCP frags 1191 * with DF set are marked as error), drop packets based on the 1192 * error mask rather than the summary bit (ignoring L4F errors) 1193 */ 1194 if (rrd.word[EMAC_RRD_STATS_DW_IDX] & EMAC_RRD_ERROR) { 1195 netif_dbg(adpt, rx_status, adpt->netdev, 1196 "Drop error packet[RRD: 0x%x:0x%x:0x%x:0x%x]\n", 1197 rrd.word[0], rrd.word[1], 1198 rrd.word[2], rrd.word[3]); 1199 1200 dev_kfree_skb(skb); 1201 continue; 1202 } 1203 1204 skb_put(skb, RRD_PKT_SIZE(&rrd) - ETH_FCS_LEN); 1205 skb->dev = netdev; 1206 skb->protocol = eth_type_trans(skb, skb->dev); 1207 if (netdev->features & NETIF_F_RXCSUM) 1208 skb->ip_summed = RRD_L4F(&rrd) ? 1209 CHECKSUM_NONE : CHECKSUM_UNNECESSARY; 1210 else 1211 skb_checksum_none_assert(skb); 1212 1213 emac_receive_skb(rx_q, skb, (u16)RRD_CVALN_TAG(&rrd), 1214 (bool)RRD_CVTAG(&rrd)); 1215 1216 netdev->last_rx = jiffies; 1217 (*num_pkts)++; 1218 } while (*num_pkts < max_pkts); 1219 1220 if (count) { 1221 proc_idx = (rx_q->rfd.process_idx << rx_q->process_shft) & 1222 rx_q->process_mask; 1223 emac_reg_update32(adpt->base + rx_q->process_reg, 1224 rx_q->process_mask, proc_idx); 1225 emac_mac_rx_descs_refill(adpt, rx_q); 1226 } 1227 } 1228 1229 /* get the number of free transmit descriptors */ 1230 static unsigned int emac_tpd_num_free_descs(struct emac_tx_queue *tx_q) 1231 { 1232 u32 produce_idx = tx_q->tpd.produce_idx; 1233 u32 consume_idx = tx_q->tpd.consume_idx; 1234 1235 return (consume_idx > produce_idx) ? 1236 (consume_idx - produce_idx - 1) : 1237 (tx_q->tpd.count + consume_idx - produce_idx - 1); 1238 } 1239 1240 /* Process transmit event */ 1241 void emac_mac_tx_process(struct emac_adapter *adpt, struct emac_tx_queue *tx_q) 1242 { 1243 u32 reg = readl_relaxed(adpt->base + tx_q->consume_reg); 1244 u32 hw_consume_idx, pkts_compl = 0, bytes_compl = 0; 1245 struct emac_buffer *tpbuf; 1246 1247 hw_consume_idx = (reg & tx_q->consume_mask) >> tx_q->consume_shift; 1248 1249 while (tx_q->tpd.consume_idx != hw_consume_idx) { 1250 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.consume_idx); 1251 if (tpbuf->dma_addr) { 1252 dma_unmap_single(adpt->netdev->dev.parent, 1253 tpbuf->dma_addr, tpbuf->length, 1254 DMA_TO_DEVICE); 1255 tpbuf->dma_addr = 0; 1256 } 1257 1258 if (tpbuf->skb) { 1259 pkts_compl++; 1260 bytes_compl += tpbuf->skb->len; 1261 dev_kfree_skb_irq(tpbuf->skb); 1262 tpbuf->skb = NULL; 1263 } 1264 1265 if (++tx_q->tpd.consume_idx == tx_q->tpd.count) 1266 tx_q->tpd.consume_idx = 0; 1267 } 1268 1269 netdev_completed_queue(adpt->netdev, pkts_compl, bytes_compl); 1270 1271 if (netif_queue_stopped(adpt->netdev)) 1272 if (emac_tpd_num_free_descs(tx_q) > (MAX_SKB_FRAGS + 1)) 1273 netif_wake_queue(adpt->netdev); 1274 } 1275 1276 /* Initialize all queue data structures */ 1277 void emac_mac_rx_tx_ring_init_all(struct platform_device *pdev, 1278 struct emac_adapter *adpt) 1279 { 1280 adpt->rx_q.netdev = adpt->netdev; 1281 1282 adpt->rx_q.produce_reg = EMAC_MAILBOX_0; 1283 adpt->rx_q.produce_mask = RFD0_PROD_IDX_BMSK; 1284 adpt->rx_q.produce_shift = RFD0_PROD_IDX_SHFT; 1285 1286 adpt->rx_q.process_reg = EMAC_MAILBOX_0; 1287 adpt->rx_q.process_mask = RFD0_PROC_IDX_BMSK; 1288 adpt->rx_q.process_shft = RFD0_PROC_IDX_SHFT; 1289 1290 adpt->rx_q.consume_reg = EMAC_MAILBOX_3; 1291 adpt->rx_q.consume_mask = RFD0_CONS_IDX_BMSK; 1292 adpt->rx_q.consume_shift = RFD0_CONS_IDX_SHFT; 1293 1294 adpt->rx_q.irq = &adpt->irq; 1295 adpt->rx_q.intr = adpt->irq.mask & ISR_RX_PKT; 1296 1297 adpt->tx_q.produce_reg = EMAC_MAILBOX_15; 1298 adpt->tx_q.produce_mask = NTPD_PROD_IDX_BMSK; 1299 adpt->tx_q.produce_shift = NTPD_PROD_IDX_SHFT; 1300 1301 adpt->tx_q.consume_reg = EMAC_MAILBOX_2; 1302 adpt->tx_q.consume_mask = NTPD_CONS_IDX_BMSK; 1303 adpt->tx_q.consume_shift = NTPD_CONS_IDX_SHFT; 1304 } 1305 1306 /* Fill up transmit descriptors with TSO and Checksum offload information */ 1307 static int emac_tso_csum(struct emac_adapter *adpt, 1308 struct emac_tx_queue *tx_q, 1309 struct sk_buff *skb, 1310 struct emac_tpd *tpd) 1311 { 1312 unsigned int hdr_len; 1313 int ret; 1314 1315 if (skb_is_gso(skb)) { 1316 if (skb_header_cloned(skb)) { 1317 ret = pskb_expand_head(skb, 0, 0, GFP_ATOMIC); 1318 if (unlikely(ret)) 1319 return ret; 1320 } 1321 1322 if (skb->protocol == htons(ETH_P_IP)) { 1323 u32 pkt_len = ((unsigned char *)ip_hdr(skb) - skb->data) 1324 + ntohs(ip_hdr(skb)->tot_len); 1325 if (skb->len > pkt_len) 1326 pskb_trim(skb, pkt_len); 1327 } 1328 1329 hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 1330 if (unlikely(skb->len == hdr_len)) { 1331 /* we only need to do csum */ 1332 netif_warn(adpt, tx_err, adpt->netdev, 1333 "tso not needed for packet with 0 data\n"); 1334 goto do_csum; 1335 } 1336 1337 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV4) { 1338 ip_hdr(skb)->check = 0; 1339 tcp_hdr(skb)->check = 1340 ~csum_tcpudp_magic(ip_hdr(skb)->saddr, 1341 ip_hdr(skb)->daddr, 1342 0, IPPROTO_TCP, 0); 1343 TPD_IPV4_SET(tpd, 1); 1344 } 1345 1346 if (skb_shinfo(skb)->gso_type & SKB_GSO_TCPV6) { 1347 /* ipv6 tso need an extra tpd */ 1348 struct emac_tpd extra_tpd; 1349 1350 memset(tpd, 0, sizeof(*tpd)); 1351 memset(&extra_tpd, 0, sizeof(extra_tpd)); 1352 1353 ipv6_hdr(skb)->payload_len = 0; 1354 tcp_hdr(skb)->check = 1355 ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr, 1356 &ipv6_hdr(skb)->daddr, 1357 0, IPPROTO_TCP, 0); 1358 TPD_PKT_LEN_SET(&extra_tpd, skb->len); 1359 TPD_LSO_SET(&extra_tpd, 1); 1360 TPD_LSOV_SET(&extra_tpd, 1); 1361 emac_tx_tpd_create(adpt, tx_q, &extra_tpd); 1362 TPD_LSOV_SET(tpd, 1); 1363 } 1364 1365 TPD_LSO_SET(tpd, 1); 1366 TPD_TCPHDR_OFFSET_SET(tpd, skb_transport_offset(skb)); 1367 TPD_MSS_SET(tpd, skb_shinfo(skb)->gso_size); 1368 return 0; 1369 } 1370 1371 do_csum: 1372 if (likely(skb->ip_summed == CHECKSUM_PARTIAL)) { 1373 unsigned int css, cso; 1374 1375 cso = skb_transport_offset(skb); 1376 if (unlikely(cso & 0x1)) { 1377 netdev_err(adpt->netdev, 1378 "error: payload offset should be even\n"); 1379 return -EINVAL; 1380 } 1381 css = cso + skb->csum_offset; 1382 1383 TPD_PAYLOAD_OFFSET_SET(tpd, cso >> 1); 1384 TPD_CXSUM_OFFSET_SET(tpd, css >> 1); 1385 TPD_CSX_SET(tpd, 1); 1386 } 1387 1388 return 0; 1389 } 1390 1391 /* Fill up transmit descriptors */ 1392 static void emac_tx_fill_tpd(struct emac_adapter *adpt, 1393 struct emac_tx_queue *tx_q, struct sk_buff *skb, 1394 struct emac_tpd *tpd) 1395 { 1396 unsigned int nr_frags = skb_shinfo(skb)->nr_frags; 1397 unsigned int first = tx_q->tpd.produce_idx; 1398 unsigned int len = skb_headlen(skb); 1399 struct emac_buffer *tpbuf = NULL; 1400 unsigned int mapped_len = 0; 1401 unsigned int i; 1402 int count = 0; 1403 int ret; 1404 1405 /* if Large Segment Offload is (in TCP Segmentation Offload struct) */ 1406 if (TPD_LSO(tpd)) { 1407 mapped_len = skb_transport_offset(skb) + tcp_hdrlen(skb); 1408 1409 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); 1410 tpbuf->length = mapped_len; 1411 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent, 1412 skb->data, tpbuf->length, 1413 DMA_TO_DEVICE); 1414 ret = dma_mapping_error(adpt->netdev->dev.parent, 1415 tpbuf->dma_addr); 1416 if (ret) 1417 goto error; 1418 1419 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); 1420 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); 1421 TPD_BUF_LEN_SET(tpd, tpbuf->length); 1422 emac_tx_tpd_create(adpt, tx_q, tpd); 1423 count++; 1424 } 1425 1426 if (mapped_len < len) { 1427 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); 1428 tpbuf->length = len - mapped_len; 1429 tpbuf->dma_addr = dma_map_single(adpt->netdev->dev.parent, 1430 skb->data + mapped_len, 1431 tpbuf->length, DMA_TO_DEVICE); 1432 ret = dma_mapping_error(adpt->netdev->dev.parent, 1433 tpbuf->dma_addr); 1434 if (ret) 1435 goto error; 1436 1437 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); 1438 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); 1439 TPD_BUF_LEN_SET(tpd, tpbuf->length); 1440 emac_tx_tpd_create(adpt, tx_q, tpd); 1441 count++; 1442 } 1443 1444 for (i = 0; i < nr_frags; i++) { 1445 struct skb_frag_struct *frag; 1446 1447 frag = &skb_shinfo(skb)->frags[i]; 1448 1449 tpbuf = GET_TPD_BUFFER(tx_q, tx_q->tpd.produce_idx); 1450 tpbuf->length = frag->size; 1451 tpbuf->dma_addr = dma_map_page(adpt->netdev->dev.parent, 1452 frag->page.p, frag->page_offset, 1453 tpbuf->length, DMA_TO_DEVICE); 1454 ret = dma_mapping_error(adpt->netdev->dev.parent, 1455 tpbuf->dma_addr); 1456 if (ret) 1457 goto error; 1458 1459 TPD_BUFFER_ADDR_L_SET(tpd, lower_32_bits(tpbuf->dma_addr)); 1460 TPD_BUFFER_ADDR_H_SET(tpd, upper_32_bits(tpbuf->dma_addr)); 1461 TPD_BUF_LEN_SET(tpd, tpbuf->length); 1462 emac_tx_tpd_create(adpt, tx_q, tpd); 1463 count++; 1464 } 1465 1466 /* The last tpd */ 1467 wmb(); 1468 emac_tx_tpd_mark_last(adpt, tx_q); 1469 1470 /* The last buffer info contain the skb address, 1471 * so it will be freed after unmap 1472 */ 1473 tpbuf->skb = skb; 1474 1475 return; 1476 1477 error: 1478 /* One of the memory mappings failed, so undo everything */ 1479 tx_q->tpd.produce_idx = first; 1480 1481 while (count--) { 1482 tpbuf = GET_TPD_BUFFER(tx_q, first); 1483 dma_unmap_page(adpt->netdev->dev.parent, tpbuf->dma_addr, 1484 tpbuf->length, DMA_TO_DEVICE); 1485 tpbuf->dma_addr = 0; 1486 tpbuf->length = 0; 1487 1488 if (++first == tx_q->tpd.count) 1489 first = 0; 1490 } 1491 1492 dev_kfree_skb(skb); 1493 } 1494 1495 /* Transmit the packet using specified transmit queue */ 1496 int emac_mac_tx_buf_send(struct emac_adapter *adpt, struct emac_tx_queue *tx_q, 1497 struct sk_buff *skb) 1498 { 1499 struct emac_tpd tpd; 1500 u32 prod_idx; 1501 1502 memset(&tpd, 0, sizeof(tpd)); 1503 1504 if (emac_tso_csum(adpt, tx_q, skb, &tpd) != 0) { 1505 dev_kfree_skb_any(skb); 1506 return NETDEV_TX_OK; 1507 } 1508 1509 if (skb_vlan_tag_present(skb)) { 1510 u16 tag; 1511 1512 EMAC_VLAN_TO_TAG(skb_vlan_tag_get(skb), tag); 1513 TPD_CVLAN_TAG_SET(&tpd, tag); 1514 TPD_INSTC_SET(&tpd, 1); 1515 } 1516 1517 if (skb_network_offset(skb) != ETH_HLEN) 1518 TPD_TYP_SET(&tpd, 1); 1519 1520 emac_tx_fill_tpd(adpt, tx_q, skb, &tpd); 1521 1522 netdev_sent_queue(adpt->netdev, skb->len); 1523 1524 /* Make sure the are enough free descriptors to hold one 1525 * maximum-sized SKB. We need one desc for each fragment, 1526 * one for the checksum (emac_tso_csum), one for TSO, and 1527 * and one for the SKB header. 1528 */ 1529 if (emac_tpd_num_free_descs(tx_q) < (MAX_SKB_FRAGS + 3)) 1530 netif_stop_queue(adpt->netdev); 1531 1532 /* update produce idx */ 1533 prod_idx = (tx_q->tpd.produce_idx << tx_q->produce_shift) & 1534 tx_q->produce_mask; 1535 emac_reg_update32(adpt->base + tx_q->produce_reg, 1536 tx_q->produce_mask, prod_idx); 1537 1538 return NETDEV_TX_OK; 1539 } 1540