1 /* 2 * This file is based on code from OCTEON SDK by Cavium Networks. 3 * 4 * Copyright (c) 2003-2010 Cavium Networks 5 * 6 * This file is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License, Version 2, as 8 * published by the Free Software Foundation. 9 */ 10 11 #include <linux/module.h> 12 #include <linux/kernel.h> 13 #include <linux/cache.h> 14 #include <linux/cpumask.h> 15 #include <linux/netdevice.h> 16 #include <linux/etherdevice.h> 17 #include <linux/ip.h> 18 #include <linux/string.h> 19 #include <linux/prefetch.h> 20 #include <linux/ratelimit.h> 21 #include <linux/smp.h> 22 #include <linux/interrupt.h> 23 #include <net/dst.h> 24 #ifdef CONFIG_XFRM 25 #include <linux/xfrm.h> 26 #include <net/xfrm.h> 27 #endif /* CONFIG_XFRM */ 28 29 #include <asm/octeon/octeon.h> 30 31 #include "ethernet-defines.h" 32 #include "ethernet-mem.h" 33 #include "ethernet-rx.h" 34 #include "octeon-ethernet.h" 35 #include "ethernet-util.h" 36 37 #include <asm/octeon/cvmx-helper.h> 38 #include <asm/octeon/cvmx-wqe.h> 39 #include <asm/octeon/cvmx-fau.h> 40 #include <asm/octeon/cvmx-pow.h> 41 #include <asm/octeon/cvmx-pip.h> 42 #include <asm/octeon/cvmx-scratch.h> 43 44 #include <asm/octeon/cvmx-gmxx-defs.h> 45 46 static atomic_t oct_rx_ready = ATOMIC_INIT(0); 47 48 static struct oct_rx_group { 49 int irq; 50 int group; 51 struct napi_struct napi; 52 } oct_rx_group[16]; 53 54 /** 55 * cvm_oct_do_interrupt - interrupt handler. 56 * @irq: Interrupt number. 57 * @napi_id: Cookie to identify the NAPI instance. 58 * 59 * The interrupt occurs whenever the POW has packets in our group. 60 * 61 */ 62 static irqreturn_t cvm_oct_do_interrupt(int irq, void *napi_id) 63 { 64 /* Disable the IRQ and start napi_poll. */ 65 disable_irq_nosync(irq); 66 napi_schedule(napi_id); 67 68 return IRQ_HANDLED; 69 } 70 71 /** 72 * cvm_oct_check_rcv_error - process receive errors 73 * @work: Work queue entry pointing to the packet. 74 * 75 * Returns Non-zero if the packet can be dropped, zero otherwise. 76 */ 77 static inline int cvm_oct_check_rcv_error(cvmx_wqe_t *work) 78 { 79 int port; 80 81 if (octeon_has_feature(OCTEON_FEATURE_PKND)) 82 port = work->word0.pip.cn68xx.pknd; 83 else 84 port = work->word1.cn38xx.ipprt; 85 86 if ((work->word2.snoip.err_code == 10) && (work->word1.len <= 64)) { 87 /* 88 * Ignore length errors on min size packets. Some 89 * equipment incorrectly pads packets to 64+4FCS 90 * instead of 60+4FCS. Note these packets still get 91 * counted as frame errors. 92 */ 93 } else if (work->word2.snoip.err_code == 5 || 94 work->word2.snoip.err_code == 7) { 95 /* 96 * We received a packet with either an alignment error 97 * or a FCS error. This may be signalling that we are 98 * running 10Mbps with GMXX_RXX_FRM_CTL[PRE_CHK] 99 * off. If this is the case we need to parse the 100 * packet to determine if we can remove a non spec 101 * preamble and generate a correct packet. 102 */ 103 int interface = cvmx_helper_get_interface_num(port); 104 int index = cvmx_helper_get_interface_index_num(port); 105 union cvmx_gmxx_rxx_frm_ctl gmxx_rxx_frm_ctl; 106 107 gmxx_rxx_frm_ctl.u64 = 108 cvmx_read_csr(CVMX_GMXX_RXX_FRM_CTL(index, interface)); 109 if (gmxx_rxx_frm_ctl.s.pre_chk == 0) { 110 u8 *ptr = 111 cvmx_phys_to_ptr(work->packet_ptr.s.addr); 112 int i = 0; 113 114 while (i < work->word1.len - 1) { 115 if (*ptr != 0x55) 116 break; 117 ptr++; 118 i++; 119 } 120 121 if (*ptr == 0xd5) { 122 /* Port received 0xd5 preamble */ 123 work->packet_ptr.s.addr += i + 1; 124 work->word1.len -= i + 5; 125 } else if ((*ptr & 0xf) == 0xd) { 126 /* Port received 0xd preamble */ 127 work->packet_ptr.s.addr += i; 128 work->word1.len -= i + 4; 129 for (i = 0; i < work->word1.len; i++) { 130 *ptr = 131 ((*ptr & 0xf0) >> 4) | 132 ((*(ptr + 1) & 0xf) << 4); 133 ptr++; 134 } 135 } else { 136 printk_ratelimited("Port %d unknown preamble, packet dropped\n", 137 port); 138 cvm_oct_free_work(work); 139 return 1; 140 } 141 } 142 } else { 143 printk_ratelimited("Port %d receive error code %d, packet dropped\n", 144 port, work->word2.snoip.err_code); 145 cvm_oct_free_work(work); 146 return 1; 147 } 148 149 return 0; 150 } 151 152 static int cvm_oct_poll(struct oct_rx_group *rx_group, int budget) 153 { 154 const int coreid = cvmx_get_core_num(); 155 u64 old_group_mask; 156 u64 old_scratch; 157 int rx_count = 0; 158 int did_work_request = 0; 159 int packet_not_copied; 160 161 /* Prefetch cvm_oct_device since we know we need it soon */ 162 prefetch(cvm_oct_device); 163 164 if (USE_ASYNC_IOBDMA) { 165 /* Save scratch in case userspace is using it */ 166 CVMX_SYNCIOBDMA; 167 old_scratch = cvmx_scratch_read64(CVMX_SCR_SCRATCH); 168 } 169 170 /* Only allow work for our group (and preserve priorities) */ 171 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { 172 old_group_mask = cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); 173 cvmx_write_csr(CVMX_SSO_PPX_GRP_MSK(coreid), 174 BIT(rx_group->group)); 175 cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); /* Flush */ 176 } else { 177 old_group_mask = cvmx_read_csr(CVMX_POW_PP_GRP_MSKX(coreid)); 178 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), 179 (old_group_mask & ~0xFFFFull) | 180 BIT(rx_group->group)); 181 } 182 183 if (USE_ASYNC_IOBDMA) { 184 cvmx_pow_work_request_async(CVMX_SCR_SCRATCH, CVMX_POW_NO_WAIT); 185 did_work_request = 1; 186 } 187 188 while (rx_count < budget) { 189 struct sk_buff *skb = NULL; 190 struct sk_buff **pskb = NULL; 191 int skb_in_hw; 192 cvmx_wqe_t *work; 193 int port; 194 195 if (USE_ASYNC_IOBDMA && did_work_request) 196 work = cvmx_pow_work_response_async(CVMX_SCR_SCRATCH); 197 else 198 work = cvmx_pow_work_request_sync(CVMX_POW_NO_WAIT); 199 200 prefetch(work); 201 did_work_request = 0; 202 if (!work) { 203 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { 204 cvmx_write_csr(CVMX_SSO_WQ_IQ_DIS, 205 BIT(rx_group->group)); 206 cvmx_write_csr(CVMX_SSO_WQ_INT, 207 BIT(rx_group->group)); 208 } else { 209 union cvmx_pow_wq_int wq_int; 210 211 wq_int.u64 = 0; 212 wq_int.s.iq_dis = BIT(rx_group->group); 213 wq_int.s.wq_int = BIT(rx_group->group); 214 cvmx_write_csr(CVMX_POW_WQ_INT, wq_int.u64); 215 } 216 break; 217 } 218 pskb = (struct sk_buff **) 219 (cvm_oct_get_buffer_ptr(work->packet_ptr) - 220 sizeof(void *)); 221 prefetch(pskb); 222 223 if (USE_ASYNC_IOBDMA && rx_count < (budget - 1)) { 224 cvmx_pow_work_request_async_nocheck(CVMX_SCR_SCRATCH, 225 CVMX_POW_NO_WAIT); 226 did_work_request = 1; 227 } 228 rx_count++; 229 230 skb_in_hw = work->word2.s.bufs == 1; 231 if (likely(skb_in_hw)) { 232 skb = *pskb; 233 prefetch(&skb->head); 234 prefetch(&skb->len); 235 } 236 237 if (octeon_has_feature(OCTEON_FEATURE_PKND)) 238 port = work->word0.pip.cn68xx.pknd; 239 else 240 port = work->word1.cn38xx.ipprt; 241 242 prefetch(cvm_oct_device[port]); 243 244 /* Immediately throw away all packets with receive errors */ 245 if (unlikely(work->word2.snoip.rcv_error)) { 246 if (cvm_oct_check_rcv_error(work)) 247 continue; 248 } 249 250 /* 251 * We can only use the zero copy path if skbuffs are 252 * in the FPA pool and the packet fits in a single 253 * buffer. 254 */ 255 if (likely(skb_in_hw)) { 256 skb->data = skb->head + work->packet_ptr.s.addr - 257 cvmx_ptr_to_phys(skb->head); 258 prefetch(skb->data); 259 skb->len = work->word1.len; 260 skb_set_tail_pointer(skb, skb->len); 261 packet_not_copied = 1; 262 } else { 263 /* 264 * We have to copy the packet. First allocate 265 * an skbuff for it. 266 */ 267 skb = dev_alloc_skb(work->word1.len); 268 if (!skb) { 269 cvm_oct_free_work(work); 270 continue; 271 } 272 273 /* 274 * Check if we've received a packet that was 275 * entirely stored in the work entry. 276 */ 277 if (unlikely(work->word2.s.bufs == 0)) { 278 u8 *ptr = work->packet_data; 279 280 if (likely(!work->word2.s.not_IP)) { 281 /* 282 * The beginning of the packet 283 * moves for IP packets. 284 */ 285 if (work->word2.s.is_v6) 286 ptr += 2; 287 else 288 ptr += 6; 289 } 290 memcpy(skb_put(skb, work->word1.len), ptr, 291 work->word1.len); 292 /* No packet buffers to free */ 293 } else { 294 int segments = work->word2.s.bufs; 295 union cvmx_buf_ptr segment_ptr = 296 work->packet_ptr; 297 int len = work->word1.len; 298 299 while (segments--) { 300 union cvmx_buf_ptr next_ptr = 301 *(union cvmx_buf_ptr *) 302 cvmx_phys_to_ptr( 303 segment_ptr.s.addr - 8); 304 305 /* 306 * Octeon Errata PKI-100: The segment size is 307 * wrong. Until it is fixed, calculate the 308 * segment size based on the packet pool 309 * buffer size. When it is fixed, the 310 * following line should be replaced with this 311 * one: int segment_size = 312 * segment_ptr.s.size; 313 */ 314 int segment_size = 315 CVMX_FPA_PACKET_POOL_SIZE - 316 (segment_ptr.s.addr - 317 (((segment_ptr.s.addr >> 7) - 318 segment_ptr.s.back) << 7)); 319 /* 320 * Don't copy more than what 321 * is left in the packet. 322 */ 323 if (segment_size > len) 324 segment_size = len; 325 /* Copy the data into the packet */ 326 memcpy(skb_put(skb, segment_size), 327 cvmx_phys_to_ptr( 328 segment_ptr.s.addr), 329 segment_size); 330 len -= segment_size; 331 segment_ptr = next_ptr; 332 } 333 } 334 packet_not_copied = 0; 335 } 336 if (likely((port < TOTAL_NUMBER_OF_PORTS) && 337 cvm_oct_device[port])) { 338 struct net_device *dev = cvm_oct_device[port]; 339 struct octeon_ethernet *priv = netdev_priv(dev); 340 341 /* 342 * Only accept packets for devices that are 343 * currently up. 344 */ 345 if (likely(dev->flags & IFF_UP)) { 346 skb->protocol = eth_type_trans(skb, dev); 347 skb->dev = dev; 348 349 if (unlikely(work->word2.s.not_IP || 350 work->word2.s.IP_exc || 351 work->word2.s.L4_error || 352 !work->word2.s.tcp_or_udp)) 353 skb->ip_summed = CHECKSUM_NONE; 354 else 355 skb->ip_summed = CHECKSUM_UNNECESSARY; 356 357 /* Increment RX stats for virtual ports */ 358 if (port >= CVMX_PIP_NUM_INPUT_PORTS) { 359 priv->stats.rx_packets++; 360 priv->stats.rx_bytes += skb->len; 361 } 362 netif_receive_skb(skb); 363 } else { 364 /* 365 * Drop any packet received for a device that 366 * isn't up. 367 */ 368 priv->stats.rx_dropped++; 369 dev_kfree_skb_irq(skb); 370 } 371 } else { 372 /* 373 * Drop any packet received for a device that 374 * doesn't exist. 375 */ 376 printk_ratelimited("Port %d not controlled by Linux, packet dropped\n", 377 port); 378 dev_kfree_skb_irq(skb); 379 } 380 /* 381 * Check to see if the skbuff and work share the same 382 * packet buffer. 383 */ 384 if (likely(packet_not_copied)) { 385 /* 386 * This buffer needs to be replaced, increment 387 * the number of buffers we need to free by 388 * one. 389 */ 390 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 391 1); 392 393 cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 1); 394 } else { 395 cvm_oct_free_work(work); 396 } 397 } 398 /* Restore the original POW group mask */ 399 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { 400 cvmx_write_csr(CVMX_SSO_PPX_GRP_MSK(coreid), old_group_mask); 401 cvmx_read_csr(CVMX_SSO_PPX_GRP_MSK(coreid)); /* Flush */ 402 } else { 403 cvmx_write_csr(CVMX_POW_PP_GRP_MSKX(coreid), old_group_mask); 404 } 405 406 if (USE_ASYNC_IOBDMA) { 407 /* Restore the scratch area */ 408 cvmx_scratch_write64(CVMX_SCR_SCRATCH, old_scratch); 409 } 410 cvm_oct_rx_refill_pool(0); 411 412 return rx_count; 413 } 414 415 /** 416 * cvm_oct_napi_poll - the NAPI poll function. 417 * @napi: The NAPI instance. 418 * @budget: Maximum number of packets to receive. 419 * 420 * Returns the number of packets processed. 421 */ 422 static int cvm_oct_napi_poll(struct napi_struct *napi, int budget) 423 { 424 struct oct_rx_group *rx_group = container_of(napi, struct oct_rx_group, 425 napi); 426 int rx_count; 427 428 rx_count = cvm_oct_poll(rx_group, budget); 429 430 if (rx_count < budget) { 431 /* No more work */ 432 napi_complete(napi); 433 enable_irq(rx_group->irq); 434 } 435 return rx_count; 436 } 437 438 #ifdef CONFIG_NET_POLL_CONTROLLER 439 /** 440 * cvm_oct_poll_controller - poll for receive packets 441 * device. 442 * 443 * @dev: Device to poll. Unused 444 */ 445 void cvm_oct_poll_controller(struct net_device *dev) 446 { 447 int i; 448 449 if (!atomic_read(&oct_rx_ready)) 450 return; 451 452 for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) { 453 if (!(pow_receive_groups & BIT(i))) 454 continue; 455 456 cvm_oct_poll(&oct_rx_group[i], 16); 457 } 458 } 459 #endif 460 461 void cvm_oct_rx_initialize(void) 462 { 463 int i; 464 struct net_device *dev_for_napi = NULL; 465 466 for (i = 0; i < TOTAL_NUMBER_OF_PORTS; i++) { 467 if (cvm_oct_device[i]) { 468 dev_for_napi = cvm_oct_device[i]; 469 break; 470 } 471 } 472 473 if (!dev_for_napi) 474 panic("No net_devices were allocated."); 475 476 for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) { 477 int ret; 478 479 if (!(pow_receive_groups & BIT(i))) 480 continue; 481 482 netif_napi_add(dev_for_napi, &oct_rx_group[i].napi, 483 cvm_oct_napi_poll, rx_napi_weight); 484 napi_enable(&oct_rx_group[i].napi); 485 486 oct_rx_group[i].irq = OCTEON_IRQ_WORKQ0 + i; 487 oct_rx_group[i].group = i; 488 489 /* Register an IRQ handler to receive POW interrupts */ 490 ret = request_irq(oct_rx_group[i].irq, cvm_oct_do_interrupt, 0, 491 "Ethernet", &oct_rx_group[i].napi); 492 if (ret) 493 panic("Could not acquire Ethernet IRQ %d\n", 494 oct_rx_group[i].irq); 495 496 disable_irq_nosync(oct_rx_group[i].irq); 497 498 /* Enable POW interrupt when our port has at least one packet */ 499 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) { 500 union cvmx_sso_wq_int_thrx int_thr; 501 union cvmx_pow_wq_int_pc int_pc; 502 503 int_thr.u64 = 0; 504 int_thr.s.tc_en = 1; 505 int_thr.s.tc_thr = 1; 506 cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(i), int_thr.u64); 507 508 int_pc.u64 = 0; 509 int_pc.s.pc_thr = 5; 510 cvmx_write_csr(CVMX_SSO_WQ_INT_PC, int_pc.u64); 511 } else { 512 union cvmx_pow_wq_int_thrx int_thr; 513 union cvmx_pow_wq_int_pc int_pc; 514 515 int_thr.u64 = 0; 516 int_thr.s.tc_en = 1; 517 int_thr.s.tc_thr = 1; 518 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), int_thr.u64); 519 520 int_pc.u64 = 0; 521 int_pc.s.pc_thr = 5; 522 cvmx_write_csr(CVMX_POW_WQ_INT_PC, int_pc.u64); 523 } 524 525 /* Schedule NAPI now. This will indirectly enable the 526 * interrupt. 527 */ 528 napi_schedule(&oct_rx_group[i].napi); 529 } 530 atomic_inc(&oct_rx_ready); 531 } 532 533 void cvm_oct_rx_shutdown(void) 534 { 535 int i; 536 537 for (i = 0; i < ARRAY_SIZE(oct_rx_group); i++) { 538 if (!(pow_receive_groups & BIT(i))) 539 continue; 540 541 /* Disable POW interrupt */ 542 if (OCTEON_IS_MODEL(OCTEON_CN68XX)) 543 cvmx_write_csr(CVMX_SSO_WQ_INT_THRX(i), 0); 544 else 545 cvmx_write_csr(CVMX_POW_WQ_INT_THRX(i), 0); 546 547 /* Free the interrupt handler */ 548 free_irq(oct_rx_group[i].irq, cvm_oct_device); 549 550 netif_napi_del(&oct_rx_group[i].napi); 551 } 552 } 553