1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * This file is based on code from OCTEON SDK by Cavium Networks. 4 * 5 * Copyright (c) 2003-2010 Cavium Networks 6 */ 7 8 #include <linux/module.h> 9 #include <linux/kernel.h> 10 #include <linux/netdevice.h> 11 #include <linux/etherdevice.h> 12 #include <linux/ip.h> 13 #include <linux/ratelimit.h> 14 #include <linux/string.h> 15 #include <linux/interrupt.h> 16 #include <net/dst.h> 17 #ifdef CONFIG_XFRM 18 #include <linux/xfrm.h> 19 #include <net/xfrm.h> 20 #endif /* CONFIG_XFRM */ 21 22 #include <linux/atomic.h> 23 #include <net/sch_generic.h> 24 25 #include "octeon-ethernet.h" 26 #include "ethernet-defines.h" 27 #include "ethernet-tx.h" 28 #include "ethernet-util.h" 29 30 #define CVM_OCT_SKB_CB(skb) ((u64 *)((skb)->cb)) 31 32 /* 33 * You can define GET_SKBUFF_QOS() to override how the skbuff output 34 * function determines which output queue is used. The default 35 * implementation always uses the base queue for the port. If, for 36 * example, you wanted to use the skb->priority field, define 37 * GET_SKBUFF_QOS as: #define GET_SKBUFF_QOS(skb) ((skb)->priority) 38 */ 39 #ifndef GET_SKBUFF_QOS 40 #define GET_SKBUFF_QOS(skb) 0 41 #endif 42 43 static void cvm_oct_tx_do_cleanup(unsigned long arg); 44 static DECLARE_TASKLET_OLD(cvm_oct_tx_cleanup_tasklet, cvm_oct_tx_do_cleanup); 45 46 /* Maximum number of SKBs to try to free per xmit packet. */ 47 #define MAX_SKB_TO_FREE (MAX_OUT_QUEUE_DEPTH * 2) 48 49 static inline int cvm_oct_adjust_skb_to_free(int skb_to_free, int fau) 50 { 51 int undo; 52 53 undo = skb_to_free > 0 ? MAX_SKB_TO_FREE : skb_to_free + 54 MAX_SKB_TO_FREE; 55 if (undo > 0) 56 cvmx_fau_atomic_add32(fau, -undo); 57 skb_to_free = -skb_to_free > MAX_SKB_TO_FREE ? MAX_SKB_TO_FREE : 58 -skb_to_free; 59 return skb_to_free; 60 } 61 62 static void cvm_oct_kick_tx_poll_watchdog(void) 63 { 64 union cvmx_ciu_timx ciu_timx; 65 66 ciu_timx.u64 = 0; 67 ciu_timx.s.one_shot = 1; 68 ciu_timx.s.len = cvm_oct_tx_poll_interval; 69 cvmx_write_csr(CVMX_CIU_TIMX(1), ciu_timx.u64); 70 } 71 72 static void cvm_oct_free_tx_skbs(struct net_device *dev) 73 { 74 int skb_to_free; 75 int qos, queues_per_port; 76 int total_remaining = 0; 77 unsigned long flags; 78 struct octeon_ethernet *priv = netdev_priv(dev); 79 80 queues_per_port = cvmx_pko_get_num_queues(priv->port); 81 /* Drain any pending packets in the free list */ 82 for (qos = 0; qos < queues_per_port; qos++) { 83 if (skb_queue_len(&priv->tx_free_list[qos]) == 0) 84 continue; 85 skb_to_free = cvmx_fau_fetch_and_add32(priv->fau + qos * 4, 86 MAX_SKB_TO_FREE); 87 skb_to_free = cvm_oct_adjust_skb_to_free(skb_to_free, 88 priv->fau + qos * 4); 89 if (skb_to_free > 0) { 90 struct sk_buff *to_free_list = NULL; 91 92 spin_lock_irqsave(&priv->tx_free_list[qos].lock, flags); 93 while (skb_to_free > 0) { 94 struct sk_buff *t; 95 96 t = __skb_dequeue(&priv->tx_free_list[qos]); 97 t->next = to_free_list; 98 to_free_list = t; 99 skb_to_free--; 100 } 101 spin_unlock_irqrestore(&priv->tx_free_list[qos].lock, 102 flags); 103 /* Do the actual freeing outside of the lock. */ 104 while (to_free_list) { 105 struct sk_buff *t = to_free_list; 106 107 to_free_list = to_free_list->next; 108 dev_kfree_skb_any(t); 109 } 110 } 111 total_remaining += skb_queue_len(&priv->tx_free_list[qos]); 112 } 113 if (total_remaining < MAX_OUT_QUEUE_DEPTH && netif_queue_stopped(dev)) 114 netif_wake_queue(dev); 115 if (total_remaining) 116 cvm_oct_kick_tx_poll_watchdog(); 117 } 118 119 /** 120 * cvm_oct_xmit - transmit a packet 121 * @skb: Packet to send 122 * @dev: Device info structure 123 * 124 * Returns Always returns NETDEV_TX_OK 125 */ 126 netdev_tx_t cvm_oct_xmit(struct sk_buff *skb, struct net_device *dev) 127 { 128 union cvmx_pko_command_word0 pko_command; 129 union cvmx_buf_ptr hw_buffer; 130 u64 old_scratch; 131 u64 old_scratch2; 132 int qos; 133 int i; 134 enum {QUEUE_CORE, QUEUE_HW, QUEUE_DROP} queue_type; 135 struct octeon_ethernet *priv = netdev_priv(dev); 136 struct sk_buff *to_free_list; 137 int skb_to_free; 138 int buffers_to_free; 139 u32 total_to_clean; 140 unsigned long flags; 141 #if REUSE_SKBUFFS_WITHOUT_FREE 142 unsigned char *fpa_head; 143 #endif 144 145 /* 146 * Prefetch the private data structure. It is larger than the 147 * one cache line. 148 */ 149 prefetch(priv); 150 151 /* 152 * The check on CVMX_PKO_QUEUES_PER_PORT_* is designed to 153 * completely remove "qos" in the event neither interface 154 * supports multiple queues per port. 155 */ 156 if ((CVMX_PKO_QUEUES_PER_PORT_INTERFACE0 > 1) || 157 (CVMX_PKO_QUEUES_PER_PORT_INTERFACE1 > 1)) { 158 qos = GET_SKBUFF_QOS(skb); 159 if (qos <= 0) 160 qos = 0; 161 else if (qos >= cvmx_pko_get_num_queues(priv->port)) 162 qos = 0; 163 } else { 164 qos = 0; 165 } 166 167 if (USE_ASYNC_IOBDMA) { 168 /* Save scratch in case userspace is using it */ 169 CVMX_SYNCIOBDMA; 170 old_scratch = cvmx_scratch_read64(CVMX_SCR_SCRATCH); 171 old_scratch2 = cvmx_scratch_read64(CVMX_SCR_SCRATCH + 8); 172 173 /* 174 * Fetch and increment the number of packets to be 175 * freed. 176 */ 177 cvmx_fau_async_fetch_and_add32(CVMX_SCR_SCRATCH + 8, 178 FAU_NUM_PACKET_BUFFERS_TO_FREE, 179 0); 180 cvmx_fau_async_fetch_and_add32(CVMX_SCR_SCRATCH, 181 priv->fau + qos * 4, 182 MAX_SKB_TO_FREE); 183 } 184 185 /* 186 * We have space for 6 segment pointers, If there will be more 187 * than that, we must linearize. 188 */ 189 if (unlikely(skb_shinfo(skb)->nr_frags > 5)) { 190 if (unlikely(__skb_linearize(skb))) { 191 queue_type = QUEUE_DROP; 192 if (USE_ASYNC_IOBDMA) { 193 /* 194 * Get the number of skbuffs in use 195 * by the hardware 196 */ 197 CVMX_SYNCIOBDMA; 198 skb_to_free = 199 cvmx_scratch_read64(CVMX_SCR_SCRATCH); 200 } else { 201 /* 202 * Get the number of skbuffs in use 203 * by the hardware 204 */ 205 skb_to_free = 206 cvmx_fau_fetch_and_add32(priv->fau + 207 qos * 4, 208 MAX_SKB_TO_FREE); 209 } 210 skb_to_free = cvm_oct_adjust_skb_to_free(skb_to_free, 211 priv->fau + 212 qos * 4); 213 spin_lock_irqsave(&priv->tx_free_list[qos].lock, flags); 214 goto skip_xmit; 215 } 216 } 217 218 /* 219 * The CN3XXX series of parts has an errata (GMX-401) which 220 * causes the GMX block to hang if a collision occurs towards 221 * the end of a <68 byte packet. As a workaround for this, we 222 * pad packets to be 68 bytes whenever we are in half duplex 223 * mode. We don't handle the case of having a small packet but 224 * no room to add the padding. The kernel should always give 225 * us at least a cache line 226 */ 227 if ((skb->len < 64) && OCTEON_IS_MODEL(OCTEON_CN3XXX)) { 228 union cvmx_gmxx_prtx_cfg gmx_prt_cfg; 229 int interface = INTERFACE(priv->port); 230 int index = INDEX(priv->port); 231 232 if (interface < 2) { 233 /* We only need to pad packet in half duplex mode */ 234 gmx_prt_cfg.u64 = 235 cvmx_read_csr(CVMX_GMXX_PRTX_CFG(index, interface)); 236 if (gmx_prt_cfg.s.duplex == 0) { 237 int add_bytes = 64 - skb->len; 238 239 if ((skb_tail_pointer(skb) + add_bytes) <= 240 skb_end_pointer(skb)) 241 __skb_put_zero(skb, add_bytes); 242 } 243 } 244 } 245 246 /* Build the PKO command */ 247 pko_command.u64 = 0; 248 #ifdef __LITTLE_ENDIAN 249 pko_command.s.le = 1; 250 #endif 251 pko_command.s.n2 = 1; /* Don't pollute L2 with the outgoing packet */ 252 pko_command.s.segs = 1; 253 pko_command.s.total_bytes = skb->len; 254 pko_command.s.size0 = CVMX_FAU_OP_SIZE_32; 255 pko_command.s.subone0 = 1; 256 257 pko_command.s.dontfree = 1; 258 259 /* Build the PKO buffer pointer */ 260 hw_buffer.u64 = 0; 261 if (skb_shinfo(skb)->nr_frags == 0) { 262 hw_buffer.s.addr = XKPHYS_TO_PHYS((uintptr_t)skb->data); 263 hw_buffer.s.pool = 0; 264 hw_buffer.s.size = skb->len; 265 } else { 266 hw_buffer.s.addr = XKPHYS_TO_PHYS((uintptr_t)skb->data); 267 hw_buffer.s.pool = 0; 268 hw_buffer.s.size = skb_headlen(skb); 269 CVM_OCT_SKB_CB(skb)[0] = hw_buffer.u64; 270 for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { 271 skb_frag_t *fs = skb_shinfo(skb)->frags + i; 272 273 hw_buffer.s.addr = 274 XKPHYS_TO_PHYS((uintptr_t)skb_frag_address(fs)); 275 hw_buffer.s.size = skb_frag_size(fs); 276 CVM_OCT_SKB_CB(skb)[i + 1] = hw_buffer.u64; 277 } 278 hw_buffer.s.addr = 279 XKPHYS_TO_PHYS((uintptr_t)CVM_OCT_SKB_CB(skb)); 280 hw_buffer.s.size = skb_shinfo(skb)->nr_frags + 1; 281 pko_command.s.segs = skb_shinfo(skb)->nr_frags + 1; 282 pko_command.s.gather = 1; 283 goto dont_put_skbuff_in_hw; 284 } 285 286 /* 287 * See if we can put this skb in the FPA pool. Any strange 288 * behavior from the Linux networking stack will most likely 289 * be caused by a bug in the following code. If some field is 290 * in use by the network stack and gets carried over when a 291 * buffer is reused, bad things may happen. If in doubt and 292 * you dont need the absolute best performance, disable the 293 * define REUSE_SKBUFFS_WITHOUT_FREE. The reuse of buffers has 294 * shown a 25% increase in performance under some loads. 295 */ 296 #if REUSE_SKBUFFS_WITHOUT_FREE 297 fpa_head = skb->head + 256 - ((unsigned long)skb->head & 0x7f); 298 if (unlikely(skb->data < fpa_head)) { 299 /* TX buffer beginning can't meet FPA alignment constraints */ 300 goto dont_put_skbuff_in_hw; 301 } 302 if (unlikely 303 ((skb_end_pointer(skb) - fpa_head) < CVMX_FPA_PACKET_POOL_SIZE)) { 304 /* TX buffer isn't large enough for the FPA */ 305 goto dont_put_skbuff_in_hw; 306 } 307 if (unlikely(skb_shared(skb))) { 308 /* TX buffer sharing data with someone else */ 309 goto dont_put_skbuff_in_hw; 310 } 311 if (unlikely(skb_cloned(skb))) { 312 /* TX buffer has been cloned */ 313 goto dont_put_skbuff_in_hw; 314 } 315 if (unlikely(skb_header_cloned(skb))) { 316 /* TX buffer header has been cloned */ 317 goto dont_put_skbuff_in_hw; 318 } 319 if (unlikely(skb->destructor)) { 320 /* TX buffer has a destructor */ 321 goto dont_put_skbuff_in_hw; 322 } 323 if (unlikely(skb_shinfo(skb)->nr_frags)) { 324 /* TX buffer has fragments */ 325 goto dont_put_skbuff_in_hw; 326 } 327 if (unlikely 328 (skb->truesize != 329 sizeof(*skb) + skb_end_offset(skb))) { 330 /* TX buffer truesize has been changed */ 331 goto dont_put_skbuff_in_hw; 332 } 333 334 /* 335 * We can use this buffer in the FPA. We don't need the FAU 336 * update anymore 337 */ 338 pko_command.s.dontfree = 0; 339 340 hw_buffer.s.back = ((unsigned long)skb->data >> 7) - 341 ((unsigned long)fpa_head >> 7); 342 343 *(struct sk_buff **)(fpa_head - sizeof(void *)) = skb; 344 345 /* 346 * The skbuff will be reused without ever being freed. We must 347 * cleanup a bunch of core things. 348 */ 349 dst_release(skb_dst(skb)); 350 skb_dst_set(skb, NULL); 351 skb_ext_reset(skb); 352 nf_reset_ct(skb); 353 skb_reset_redirect(skb); 354 355 #ifdef CONFIG_NET_SCHED 356 skb->tc_index = 0; 357 #endif /* CONFIG_NET_SCHED */ 358 #endif /* REUSE_SKBUFFS_WITHOUT_FREE */ 359 360 dont_put_skbuff_in_hw: 361 362 /* Check if we can use the hardware checksumming */ 363 if ((skb->protocol == htons(ETH_P_IP)) && 364 (ip_hdr(skb)->version == 4) && 365 (ip_hdr(skb)->ihl == 5) && 366 ((ip_hdr(skb)->frag_off == 0) || 367 (ip_hdr(skb)->frag_off == htons(1 << 14))) && 368 ((ip_hdr(skb)->protocol == IPPROTO_TCP) || 369 (ip_hdr(skb)->protocol == IPPROTO_UDP))) { 370 /* Use hardware checksum calc */ 371 pko_command.s.ipoffp1 = skb_network_offset(skb) + 1; 372 } 373 374 if (USE_ASYNC_IOBDMA) { 375 /* Get the number of skbuffs in use by the hardware */ 376 CVMX_SYNCIOBDMA; 377 skb_to_free = cvmx_scratch_read64(CVMX_SCR_SCRATCH); 378 buffers_to_free = cvmx_scratch_read64(CVMX_SCR_SCRATCH + 8); 379 } else { 380 /* Get the number of skbuffs in use by the hardware */ 381 skb_to_free = cvmx_fau_fetch_and_add32(priv->fau + qos * 4, 382 MAX_SKB_TO_FREE); 383 buffers_to_free = 384 cvmx_fau_fetch_and_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, 0); 385 } 386 387 skb_to_free = cvm_oct_adjust_skb_to_free(skb_to_free, 388 priv->fau + qos * 4); 389 390 /* 391 * If we're sending faster than the receive can free them then 392 * don't do the HW free. 393 */ 394 if ((buffers_to_free < -100) && !pko_command.s.dontfree) 395 pko_command.s.dontfree = 1; 396 397 if (pko_command.s.dontfree) { 398 queue_type = QUEUE_CORE; 399 pko_command.s.reg0 = priv->fau + qos * 4; 400 } else { 401 queue_type = QUEUE_HW; 402 } 403 if (USE_ASYNC_IOBDMA) 404 cvmx_fau_async_fetch_and_add32(CVMX_SCR_SCRATCH, 405 FAU_TOTAL_TX_TO_CLEAN, 1); 406 407 spin_lock_irqsave(&priv->tx_free_list[qos].lock, flags); 408 409 /* Drop this packet if we have too many already queued to the HW */ 410 if (unlikely(skb_queue_len(&priv->tx_free_list[qos]) >= 411 MAX_OUT_QUEUE_DEPTH)) { 412 if (dev->tx_queue_len != 0) { 413 /* Drop the lock when notifying the core. */ 414 spin_unlock_irqrestore(&priv->tx_free_list[qos].lock, 415 flags); 416 netif_stop_queue(dev); 417 spin_lock_irqsave(&priv->tx_free_list[qos].lock, 418 flags); 419 } else { 420 /* If not using normal queueing. */ 421 queue_type = QUEUE_DROP; 422 goto skip_xmit; 423 } 424 } 425 426 cvmx_pko_send_packet_prepare(priv->port, priv->queue + qos, 427 CVMX_PKO_LOCK_NONE); 428 429 /* Send the packet to the output queue */ 430 if (unlikely(cvmx_pko_send_packet_finish(priv->port, 431 priv->queue + qos, 432 pko_command, hw_buffer, 433 CVMX_PKO_LOCK_NONE))) { 434 printk_ratelimited("%s: Failed to send the packet\n", 435 dev->name); 436 queue_type = QUEUE_DROP; 437 } 438 skip_xmit: 439 to_free_list = NULL; 440 441 switch (queue_type) { 442 case QUEUE_DROP: 443 skb->next = to_free_list; 444 to_free_list = skb; 445 dev->stats.tx_dropped++; 446 break; 447 case QUEUE_HW: 448 cvmx_fau_atomic_add32(FAU_NUM_PACKET_BUFFERS_TO_FREE, -1); 449 break; 450 case QUEUE_CORE: 451 __skb_queue_tail(&priv->tx_free_list[qos], skb); 452 break; 453 default: 454 BUG(); 455 } 456 457 while (skb_to_free > 0) { 458 struct sk_buff *t = __skb_dequeue(&priv->tx_free_list[qos]); 459 460 t->next = to_free_list; 461 to_free_list = t; 462 skb_to_free--; 463 } 464 465 spin_unlock_irqrestore(&priv->tx_free_list[qos].lock, flags); 466 467 /* Do the actual freeing outside of the lock. */ 468 while (to_free_list) { 469 struct sk_buff *t = to_free_list; 470 471 to_free_list = to_free_list->next; 472 dev_kfree_skb_any(t); 473 } 474 475 if (USE_ASYNC_IOBDMA) { 476 CVMX_SYNCIOBDMA; 477 total_to_clean = cvmx_scratch_read64(CVMX_SCR_SCRATCH); 478 /* Restore the scratch area */ 479 cvmx_scratch_write64(CVMX_SCR_SCRATCH, old_scratch); 480 cvmx_scratch_write64(CVMX_SCR_SCRATCH + 8, old_scratch2); 481 } else { 482 total_to_clean = 483 cvmx_fau_fetch_and_add32(FAU_TOTAL_TX_TO_CLEAN, 1); 484 } 485 486 if (total_to_clean & 0x3ff) { 487 /* 488 * Schedule the cleanup tasklet every 1024 packets for 489 * the pathological case of high traffic on one port 490 * delaying clean up of packets on a different port 491 * that is blocked waiting for the cleanup. 492 */ 493 tasklet_schedule(&cvm_oct_tx_cleanup_tasklet); 494 } 495 496 cvm_oct_kick_tx_poll_watchdog(); 497 498 return NETDEV_TX_OK; 499 } 500 501 /** 502 * cvm_oct_xmit_pow - transmit a packet to the POW 503 * @skb: Packet to send 504 * @dev: Device info structure 505 * Returns Always returns zero 506 */ 507 netdev_tx_t cvm_oct_xmit_pow(struct sk_buff *skb, struct net_device *dev) 508 { 509 struct octeon_ethernet *priv = netdev_priv(dev); 510 void *packet_buffer; 511 void *copy_location; 512 513 /* Get a work queue entry */ 514 struct cvmx_wqe *work = cvmx_fpa_alloc(CVMX_FPA_WQE_POOL); 515 516 if (unlikely(!work)) { 517 printk_ratelimited("%s: Failed to allocate a work queue entry\n", 518 dev->name); 519 dev->stats.tx_dropped++; 520 dev_kfree_skb_any(skb); 521 return 0; 522 } 523 524 /* Get a packet buffer */ 525 packet_buffer = cvmx_fpa_alloc(CVMX_FPA_PACKET_POOL); 526 if (unlikely(!packet_buffer)) { 527 printk_ratelimited("%s: Failed to allocate a packet buffer\n", 528 dev->name); 529 cvmx_fpa_free(work, CVMX_FPA_WQE_POOL, 1); 530 dev->stats.tx_dropped++; 531 dev_kfree_skb_any(skb); 532 return 0; 533 } 534 535 /* 536 * Calculate where we need to copy the data to. We need to 537 * leave 8 bytes for a next pointer (unused). We also need to 538 * include any configure skip. Then we need to align the IP 539 * packet src and dest into the same 64bit word. The below 540 * calculation may add a little extra, but that doesn't 541 * hurt. 542 */ 543 copy_location = packet_buffer + sizeof(u64); 544 copy_location += ((CVMX_HELPER_FIRST_MBUFF_SKIP + 7) & 0xfff8) + 6; 545 546 /* 547 * We have to copy the packet since whoever processes this 548 * packet will free it to a hardware pool. We can't use the 549 * trick of counting outstanding packets like in 550 * cvm_oct_xmit. 551 */ 552 memcpy(copy_location, skb->data, skb->len); 553 554 /* 555 * Fill in some of the work queue fields. We may need to add 556 * more if the software at the other end needs them. 557 */ 558 if (!OCTEON_IS_MODEL(OCTEON_CN68XX)) 559 work->word0.pip.cn38xx.hw_chksum = skb->csum; 560 work->word1.len = skb->len; 561 cvmx_wqe_set_port(work, priv->port); 562 cvmx_wqe_set_qos(work, priv->port & 0x7); 563 cvmx_wqe_set_grp(work, pow_send_group); 564 work->word1.tag_type = CVMX_HELPER_INPUT_TAG_TYPE; 565 work->word1.tag = pow_send_group; /* FIXME */ 566 /* Default to zero. Sets of zero later are commented out */ 567 work->word2.u64 = 0; 568 work->word2.s.bufs = 1; 569 work->packet_ptr.u64 = 0; 570 work->packet_ptr.s.addr = cvmx_ptr_to_phys(copy_location); 571 work->packet_ptr.s.pool = CVMX_FPA_PACKET_POOL; 572 work->packet_ptr.s.size = CVMX_FPA_PACKET_POOL_SIZE; 573 work->packet_ptr.s.back = (copy_location - packet_buffer) >> 7; 574 575 if (skb->protocol == htons(ETH_P_IP)) { 576 work->word2.s.ip_offset = 14; 577 #if 0 578 work->word2.s.vlan_valid = 0; /* FIXME */ 579 work->word2.s.vlan_cfi = 0; /* FIXME */ 580 work->word2.s.vlan_id = 0; /* FIXME */ 581 work->word2.s.dec_ipcomp = 0; /* FIXME */ 582 #endif 583 work->word2.s.tcp_or_udp = 584 (ip_hdr(skb)->protocol == IPPROTO_TCP) || 585 (ip_hdr(skb)->protocol == IPPROTO_UDP); 586 #if 0 587 /* FIXME */ 588 work->word2.s.dec_ipsec = 0; 589 /* We only support IPv4 right now */ 590 work->word2.s.is_v6 = 0; 591 /* Hardware would set to zero */ 592 work->word2.s.software = 0; 593 /* No error, packet is internal */ 594 work->word2.s.L4_error = 0; 595 #endif 596 work->word2.s.is_frag = !((ip_hdr(skb)->frag_off == 0) || 597 (ip_hdr(skb)->frag_off == 598 cpu_to_be16(1 << 14))); 599 #if 0 600 /* Assume Linux is sending a good packet */ 601 work->word2.s.IP_exc = 0; 602 #endif 603 work->word2.s.is_bcast = (skb->pkt_type == PACKET_BROADCAST); 604 work->word2.s.is_mcast = (skb->pkt_type == PACKET_MULTICAST); 605 #if 0 606 /* This is an IP packet */ 607 work->word2.s.not_IP = 0; 608 /* No error, packet is internal */ 609 work->word2.s.rcv_error = 0; 610 /* No error, packet is internal */ 611 work->word2.s.err_code = 0; 612 #endif 613 614 /* 615 * When copying the data, include 4 bytes of the 616 * ethernet header to align the same way hardware 617 * does. 618 */ 619 memcpy(work->packet_data, skb->data + 10, 620 sizeof(work->packet_data)); 621 } else { 622 #if 0 623 work->word2.snoip.vlan_valid = 0; /* FIXME */ 624 work->word2.snoip.vlan_cfi = 0; /* FIXME */ 625 work->word2.snoip.vlan_id = 0; /* FIXME */ 626 work->word2.snoip.software = 0; /* Hardware would set to zero */ 627 #endif 628 work->word2.snoip.is_rarp = skb->protocol == htons(ETH_P_RARP); 629 work->word2.snoip.is_arp = skb->protocol == htons(ETH_P_ARP); 630 work->word2.snoip.is_bcast = 631 (skb->pkt_type == PACKET_BROADCAST); 632 work->word2.snoip.is_mcast = 633 (skb->pkt_type == PACKET_MULTICAST); 634 work->word2.snoip.not_IP = 1; /* IP was done up above */ 635 #if 0 636 /* No error, packet is internal */ 637 work->word2.snoip.rcv_error = 0; 638 /* No error, packet is internal */ 639 work->word2.snoip.err_code = 0; 640 #endif 641 memcpy(work->packet_data, skb->data, sizeof(work->packet_data)); 642 } 643 644 /* Submit the packet to the POW */ 645 cvmx_pow_work_submit(work, work->word1.tag, work->word1.tag_type, 646 cvmx_wqe_get_qos(work), cvmx_wqe_get_grp(work)); 647 dev->stats.tx_packets++; 648 dev->stats.tx_bytes += skb->len; 649 dev_consume_skb_any(skb); 650 return 0; 651 } 652 653 /** 654 * cvm_oct_tx_shutdown_dev - free all skb that are currently queued for TX. 655 * @dev: Device being shutdown 656 * 657 */ 658 void cvm_oct_tx_shutdown_dev(struct net_device *dev) 659 { 660 struct octeon_ethernet *priv = netdev_priv(dev); 661 unsigned long flags; 662 int qos; 663 664 for (qos = 0; qos < 16; qos++) { 665 spin_lock_irqsave(&priv->tx_free_list[qos].lock, flags); 666 while (skb_queue_len(&priv->tx_free_list[qos])) 667 dev_kfree_skb_any(__skb_dequeue 668 (&priv->tx_free_list[qos])); 669 spin_unlock_irqrestore(&priv->tx_free_list[qos].lock, flags); 670 } 671 } 672 673 static void cvm_oct_tx_do_cleanup(unsigned long arg) 674 { 675 int port; 676 677 for (port = 0; port < TOTAL_NUMBER_OF_PORTS; port++) { 678 if (cvm_oct_device[port]) { 679 struct net_device *dev = cvm_oct_device[port]; 680 681 cvm_oct_free_tx_skbs(dev); 682 } 683 } 684 } 685 686 static irqreturn_t cvm_oct_tx_cleanup_watchdog(int cpl, void *dev_id) 687 { 688 /* Disable the interrupt. */ 689 cvmx_write_csr(CVMX_CIU_TIMX(1), 0); 690 /* Do the work in the tasklet. */ 691 tasklet_schedule(&cvm_oct_tx_cleanup_tasklet); 692 return IRQ_HANDLED; 693 } 694 695 void cvm_oct_tx_initialize(void) 696 { 697 int i; 698 699 /* Disable the interrupt. */ 700 cvmx_write_csr(CVMX_CIU_TIMX(1), 0); 701 /* Register an IRQ handler to receive CIU_TIMX(1) interrupts */ 702 i = request_irq(OCTEON_IRQ_TIMER1, 703 cvm_oct_tx_cleanup_watchdog, 0, 704 "Ethernet", cvm_oct_device); 705 706 if (i) 707 panic("Could not acquire Ethernet IRQ %d\n", OCTEON_IRQ_TIMER1); 708 } 709 710 void cvm_oct_tx_shutdown(void) 711 { 712 /* Free the interrupt handler */ 713 free_irq(OCTEON_IRQ_TIMER1, cvm_oct_device); 714 } 715