1 /********************************************************************** 2 * Author: Cavium, Inc. 3 * 4 * Contact: support@cavium.com 5 * Please include "LiquidIO" in the subject. 6 * 7 * Copyright (c) 2003-2016 Cavium, Inc. 8 * 9 * This file is free software; you can redistribute it and/or modify 10 * it under the terms of the GNU General Public License, Version 2, as 11 * published by the Free Software Foundation. 12 * 13 * This file is distributed in the hope that it will be useful, but 14 * AS-IS and WITHOUT ANY WARRANTY; without even the implied warranty 15 * of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE, TITLE, or 16 * NONINFRINGEMENT. See the GNU General Public License for more 17 * details. 18 **********************************************************************/ 19 20 /*! \file octeon_network.h 21 * \brief Host NIC Driver: Structure and Macro definitions used by NIC Module. 22 */ 23 24 #ifndef __OCTEON_NETWORK_H__ 25 #define __OCTEON_NETWORK_H__ 26 #include <linux/ptp_clock_kernel.h> 27 28 #define LIO_MAX_MTU_SIZE (OCTNET_MAX_FRM_SIZE - OCTNET_FRM_HEADER_SIZE) 29 #define LIO_MIN_MTU_SIZE ETH_MIN_MTU 30 31 /* Bit mask values for lio->ifstate */ 32 #define LIO_IFSTATE_DROQ_OPS 0x01 33 #define LIO_IFSTATE_REGISTERED 0x02 34 #define LIO_IFSTATE_RUNNING 0x04 35 #define LIO_IFSTATE_RX_TIMESTAMP_ENABLED 0x08 36 #define LIO_IFSTATE_RESETTING 0x10 37 38 struct liquidio_if_cfg_context { 39 u32 octeon_id; 40 wait_queue_head_t wc; 41 int cond; 42 }; 43 44 struct liquidio_if_cfg_resp { 45 u64 rh; 46 struct liquidio_if_cfg_info cfg_info; 47 u64 status; 48 }; 49 50 #define LIO_IFCFG_WAIT_TIME 3000 /* In milli seconds */ 51 52 /* Structure of a node in list of gather components maintained by 53 * NIC driver for each network device. 54 */ 55 struct octnic_gather { 56 /* List manipulation. Next and prev pointers. */ 57 struct list_head list; 58 59 /* Size of the gather component at sg in bytes. */ 60 int sg_size; 61 62 /* Number of bytes that sg was adjusted to make it 8B-aligned. */ 63 int adjust; 64 65 /* Gather component that can accommodate max sized fragment list 66 * received from the IP layer. 67 */ 68 struct octeon_sg_entry *sg; 69 70 dma_addr_t sg_dma_ptr; 71 }; 72 73 struct oct_nic_stats_resp { 74 u64 rh; 75 struct oct_link_stats stats; 76 u64 status; 77 }; 78 79 struct oct_nic_stats_ctrl { 80 struct completion complete; 81 struct net_device *netdev; 82 }; 83 84 struct oct_nic_seapi_resp { 85 u64 rh; 86 u32 speed; 87 u64 status; 88 }; 89 90 struct liquidio_nic_seapi_ctl_context { 91 int octeon_id; 92 u32 status; 93 struct completion complete; 94 }; 95 96 /** LiquidIO per-interface network private data */ 97 struct lio { 98 /** State of the interface. Rx/Tx happens only in the RUNNING state. */ 99 atomic_t ifstate; 100 101 /** Octeon Interface index number. This device will be represented as 102 * oct<ifidx> in the system. 103 */ 104 int ifidx; 105 106 /** Octeon Input queue to use to transmit for this network interface. */ 107 int txq; 108 109 /** Octeon Output queue from which pkts arrive 110 * for this network interface. 111 */ 112 int rxq; 113 114 /** Guards each glist */ 115 spinlock_t *glist_lock; 116 117 /** Array of gather component linked lists */ 118 struct list_head *glist; 119 void **glists_virt_base; 120 dma_addr_t *glists_dma_base; 121 u32 glist_entry_size; 122 123 /** Pointer to the NIC properties for the Octeon device this network 124 * interface is associated with. 125 */ 126 struct octdev_props *octprops; 127 128 /** Pointer to the octeon device structure. */ 129 struct octeon_device *oct_dev; 130 131 struct net_device *netdev; 132 133 /** Link information sent by the core application for this interface. */ 134 struct oct_link_info linfo; 135 136 /** counter of link changes */ 137 u64 link_changes; 138 139 /** Size of Tx queue for this octeon device. */ 140 u32 tx_qsize; 141 142 /** Size of Rx queue for this octeon device. */ 143 u32 rx_qsize; 144 145 /** Size of MTU this octeon device. */ 146 u32 mtu; 147 148 /** msg level flag per interface. */ 149 u32 msg_enable; 150 151 /** Copy of Interface capabilities: TSO, TSO6, LRO, Chescksums . */ 152 u64 dev_capability; 153 154 /* Copy of transmit encapsulation capabilities: 155 * TSO, TSO6, Checksums for this device for Kernel 156 * 3.10.0 onwards 157 */ 158 u64 enc_dev_capability; 159 160 /** Copy of beacaon reg in phy */ 161 u32 phy_beacon_val; 162 163 /** Copy of ctrl reg in phy */ 164 u32 led_ctrl_val; 165 166 /* PTP clock information */ 167 struct ptp_clock_info ptp_info; 168 struct ptp_clock *ptp_clock; 169 s64 ptp_adjust; 170 171 /* for atomic access to Octeon PTP reg and data struct */ 172 spinlock_t ptp_lock; 173 174 /* Interface info */ 175 u32 intf_open; 176 177 /* work queue for txq status */ 178 struct cavium_wq txq_status_wq; 179 180 /* work queue for rxq oom status */ 181 struct cavium_wq rxq_status_wq; 182 183 /* work queue for link status */ 184 struct cavium_wq link_status_wq; 185 186 /* work queue to regularly send local time to octeon firmware */ 187 struct cavium_wq sync_octeon_time_wq; 188 189 int netdev_uc_count; 190 }; 191 192 #define LIO_SIZE (sizeof(struct lio)) 193 #define GET_LIO(netdev) ((struct lio *)netdev_priv(netdev)) 194 195 #define LIO_MAX_CORES 16 196 197 /** 198 * \brief Enable or disable feature 199 * @param netdev pointer to network device 200 * @param cmd Command that just requires acknowledgment 201 * @param param1 Parameter to command 202 */ 203 int liquidio_set_feature(struct net_device *netdev, int cmd, u16 param1); 204 205 int setup_rx_oom_poll_fn(struct net_device *netdev); 206 207 void cleanup_rx_oom_poll_fn(struct net_device *netdev); 208 209 /** 210 * \brief Link control command completion callback 211 * @param nctrl_ptr pointer to control packet structure 212 * 213 * This routine is called by the callback function when a ctrl pkt sent to 214 * core app completes. The nctrl_ptr contains a copy of the command type 215 * and data sent to the core app. This routine is only called if the ctrl 216 * pkt was sent successfully to the core app. 217 */ 218 void liquidio_link_ctrl_cmd_completion(void *nctrl_ptr); 219 220 int liquidio_setup_io_queues(struct octeon_device *octeon_dev, int ifidx, 221 u32 num_iqs, u32 num_oqs); 222 223 irqreturn_t liquidio_msix_intr_handler(int irq __attribute__((unused)), 224 void *dev); 225 226 int octeon_setup_interrupt(struct octeon_device *oct, u32 num_ioqs); 227 228 int octnet_get_link_stats(struct net_device *netdev); 229 230 int lio_wait_for_clean_oq(struct octeon_device *oct); 231 /** 232 * \brief Register ethtool operations 233 * @param netdev pointer to network device 234 */ 235 void liquidio_set_ethtool_ops(struct net_device *netdev); 236 237 void lio_if_cfg_callback(struct octeon_device *oct, 238 u32 status __attribute__((unused)), 239 void *buf); 240 241 void lio_delete_glists(struct lio *lio); 242 243 int lio_setup_glists(struct octeon_device *oct, struct lio *lio, int num_qs); 244 245 int liquidio_get_speed(struct lio *lio); 246 int liquidio_set_speed(struct lio *lio, int speed); 247 248 /** 249 * \brief Net device change_mtu 250 * @param netdev network device 251 */ 252 int liquidio_change_mtu(struct net_device *netdev, int new_mtu); 253 #define LIO_CHANGE_MTU_SUCCESS 1 254 #define LIO_CHANGE_MTU_FAIL 2 255 256 #define SKB_ADJ_MASK 0x3F 257 #define SKB_ADJ (SKB_ADJ_MASK + 1) 258 259 #define MIN_SKB_SIZE 256 /* 8 bytes and more - 8 bytes for PTP */ 260 #define LIO_RXBUFFER_SZ 2048 261 262 static inline void 263 *recv_buffer_alloc(struct octeon_device *oct, 264 struct octeon_skb_page_info *pg_info) 265 { 266 struct page *page; 267 struct sk_buff *skb; 268 struct octeon_skb_page_info *skb_pg_info; 269 270 page = alloc_page(GFP_ATOMIC); 271 if (unlikely(!page)) 272 return NULL; 273 274 skb = dev_alloc_skb(MIN_SKB_SIZE + SKB_ADJ); 275 if (unlikely(!skb)) { 276 __free_page(page); 277 pg_info->page = NULL; 278 return NULL; 279 } 280 281 if ((unsigned long)skb->data & SKB_ADJ_MASK) { 282 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK); 283 284 skb_reserve(skb, r); 285 } 286 287 skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 288 /* Get DMA info */ 289 pg_info->dma = dma_map_page(&oct->pci_dev->dev, page, 0, 290 PAGE_SIZE, DMA_FROM_DEVICE); 291 292 /* Mapping failed!! */ 293 if (dma_mapping_error(&oct->pci_dev->dev, pg_info->dma)) { 294 __free_page(page); 295 dev_kfree_skb_any((struct sk_buff *)skb); 296 pg_info->page = NULL; 297 return NULL; 298 } 299 300 pg_info->page = page; 301 pg_info->page_offset = 0; 302 skb_pg_info->page = page; 303 skb_pg_info->page_offset = 0; 304 skb_pg_info->dma = pg_info->dma; 305 306 return (void *)skb; 307 } 308 309 static inline void 310 *recv_buffer_fast_alloc(u32 size) 311 { 312 struct sk_buff *skb; 313 struct octeon_skb_page_info *skb_pg_info; 314 315 skb = dev_alloc_skb(size + SKB_ADJ); 316 if (unlikely(!skb)) 317 return NULL; 318 319 if ((unsigned long)skb->data & SKB_ADJ_MASK) { 320 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK); 321 322 skb_reserve(skb, r); 323 } 324 325 skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 326 skb_pg_info->page = NULL; 327 skb_pg_info->page_offset = 0; 328 skb_pg_info->dma = 0; 329 330 return skb; 331 } 332 333 static inline int 334 recv_buffer_recycle(struct octeon_device *oct, void *buf) 335 { 336 struct octeon_skb_page_info *pg_info = buf; 337 338 if (!pg_info->page) { 339 dev_err(&oct->pci_dev->dev, "%s: pg_info->page NULL\n", 340 __func__); 341 return -ENOMEM; 342 } 343 344 if (unlikely(page_count(pg_info->page) != 1) || 345 unlikely(page_to_nid(pg_info->page) != numa_node_id())) { 346 dma_unmap_page(&oct->pci_dev->dev, 347 pg_info->dma, (PAGE_SIZE << 0), 348 DMA_FROM_DEVICE); 349 pg_info->dma = 0; 350 pg_info->page = NULL; 351 pg_info->page_offset = 0; 352 return -ENOMEM; 353 } 354 355 /* Flip to other half of the buffer */ 356 if (pg_info->page_offset == 0) 357 pg_info->page_offset = LIO_RXBUFFER_SZ; 358 else 359 pg_info->page_offset = 0; 360 page_ref_inc(pg_info->page); 361 362 return 0; 363 } 364 365 static inline void 366 *recv_buffer_reuse(struct octeon_device *oct, void *buf) 367 { 368 struct octeon_skb_page_info *pg_info = buf, *skb_pg_info; 369 struct sk_buff *skb; 370 371 skb = dev_alloc_skb(MIN_SKB_SIZE + SKB_ADJ); 372 if (unlikely(!skb)) { 373 dma_unmap_page(&oct->pci_dev->dev, 374 pg_info->dma, (PAGE_SIZE << 0), 375 DMA_FROM_DEVICE); 376 return NULL; 377 } 378 379 if ((unsigned long)skb->data & SKB_ADJ_MASK) { 380 u32 r = SKB_ADJ - ((unsigned long)skb->data & SKB_ADJ_MASK); 381 382 skb_reserve(skb, r); 383 } 384 385 skb_pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 386 skb_pg_info->page = pg_info->page; 387 skb_pg_info->page_offset = pg_info->page_offset; 388 skb_pg_info->dma = pg_info->dma; 389 390 return skb; 391 } 392 393 static inline void 394 recv_buffer_destroy(void *buffer, struct octeon_skb_page_info *pg_info) 395 { 396 struct sk_buff *skb = (struct sk_buff *)buffer; 397 398 put_page(pg_info->page); 399 pg_info->dma = 0; 400 pg_info->page = NULL; 401 pg_info->page_offset = 0; 402 403 if (skb) 404 dev_kfree_skb_any(skb); 405 } 406 407 static inline void recv_buffer_free(void *buffer) 408 { 409 struct sk_buff *skb = (struct sk_buff *)buffer; 410 struct octeon_skb_page_info *pg_info; 411 412 pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 413 414 if (pg_info->page) { 415 put_page(pg_info->page); 416 pg_info->dma = 0; 417 pg_info->page = NULL; 418 pg_info->page_offset = 0; 419 } 420 421 dev_kfree_skb_any((struct sk_buff *)buffer); 422 } 423 424 static inline void 425 recv_buffer_fast_free(void *buffer) 426 { 427 dev_kfree_skb_any((struct sk_buff *)buffer); 428 } 429 430 static inline void tx_buffer_free(void *buffer) 431 { 432 dev_kfree_skb_any((struct sk_buff *)buffer); 433 } 434 435 #define lio_dma_alloc(oct, size, dma_addr) \ 436 dma_alloc_coherent(&(oct)->pci_dev->dev, size, dma_addr, GFP_KERNEL) 437 #define lio_dma_free(oct, size, virt_addr, dma_addr) \ 438 dma_free_coherent(&(oct)->pci_dev->dev, size, virt_addr, dma_addr) 439 440 static inline 441 void *get_rbd(struct sk_buff *skb) 442 { 443 struct octeon_skb_page_info *pg_info; 444 unsigned char *va; 445 446 pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 447 va = page_address(pg_info->page) + pg_info->page_offset; 448 449 return va; 450 } 451 452 static inline u64 453 lio_map_ring(void *buf) 454 { 455 dma_addr_t dma_addr; 456 457 struct sk_buff *skb = (struct sk_buff *)buf; 458 struct octeon_skb_page_info *pg_info; 459 460 pg_info = ((struct octeon_skb_page_info *)(skb->cb)); 461 if (!pg_info->page) { 462 pr_err("%s: pg_info->page NULL\n", __func__); 463 WARN_ON(1); 464 } 465 466 /* Get DMA info */ 467 dma_addr = pg_info->dma; 468 if (!pg_info->dma) { 469 pr_err("%s: ERROR it should be already available\n", 470 __func__); 471 WARN_ON(1); 472 } 473 dma_addr += pg_info->page_offset; 474 475 return (u64)dma_addr; 476 } 477 478 static inline void 479 lio_unmap_ring(struct pci_dev *pci_dev, 480 u64 buf_ptr) 481 482 { 483 dma_unmap_page(&pci_dev->dev, 484 buf_ptr, (PAGE_SIZE << 0), 485 DMA_FROM_DEVICE); 486 } 487 488 static inline void *octeon_fast_packet_alloc(u32 size) 489 { 490 return recv_buffer_fast_alloc(size); 491 } 492 493 static inline void octeon_fast_packet_next(struct octeon_droq *droq, 494 struct sk_buff *nicbuf, 495 int copy_len, 496 int idx) 497 { 498 skb_put_data(nicbuf, get_rbd(droq->recv_buf_list[idx].buffer), 499 copy_len); 500 } 501 502 /** 503 * \brief check interface state 504 * @param lio per-network private data 505 * @param state_flag flag state to check 506 */ 507 static inline int ifstate_check(struct lio *lio, int state_flag) 508 { 509 return atomic_read(&lio->ifstate) & state_flag; 510 } 511 512 /** 513 * \brief set interface state 514 * @param lio per-network private data 515 * @param state_flag flag state to set 516 */ 517 static inline void ifstate_set(struct lio *lio, int state_flag) 518 { 519 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) | state_flag)); 520 } 521 522 /** 523 * \brief clear interface state 524 * @param lio per-network private data 525 * @param state_flag flag state to clear 526 */ 527 static inline void ifstate_reset(struct lio *lio, int state_flag) 528 { 529 atomic_set(&lio->ifstate, (atomic_read(&lio->ifstate) & ~(state_flag))); 530 } 531 532 /** 533 * \brief wait for all pending requests to complete 534 * @param oct Pointer to Octeon device 535 * 536 * Called during shutdown sequence 537 */ 538 static inline int wait_for_pending_requests(struct octeon_device *oct) 539 { 540 int i, pcount = 0; 541 542 for (i = 0; i < MAX_IO_PENDING_PKT_COUNT; i++) { 543 pcount = atomic_read( 544 &oct->response_list[OCTEON_ORDERED_SC_LIST] 545 .pending_req_count); 546 if (pcount) 547 schedule_timeout_uninterruptible(HZ / 10); 548 else 549 break; 550 } 551 552 if (pcount) 553 return 1; 554 555 return 0; 556 } 557 558 /** 559 * \brief Stop Tx queues 560 * @param netdev network device 561 */ 562 static inline void stop_txqs(struct net_device *netdev) 563 { 564 int i; 565 566 for (i = 0; i < netdev->real_num_tx_queues; i++) 567 netif_stop_subqueue(netdev, i); 568 } 569 570 /** 571 * \brief Wake Tx queues 572 * @param netdev network device 573 */ 574 static inline void wake_txqs(struct net_device *netdev) 575 { 576 struct lio *lio = GET_LIO(netdev); 577 int i, qno; 578 579 for (i = 0; i < netdev->real_num_tx_queues; i++) { 580 qno = lio->linfo.txpciq[i % lio->oct_dev->num_iqs].s.q_no; 581 582 if (__netif_subqueue_stopped(netdev, i)) { 583 INCR_INSTRQUEUE_PKT_COUNT(lio->oct_dev, qno, 584 tx_restart, 1); 585 netif_wake_subqueue(netdev, i); 586 } 587 } 588 } 589 590 /** 591 * \brief Start Tx queues 592 * @param netdev network device 593 */ 594 static inline void start_txqs(struct net_device *netdev) 595 { 596 struct lio *lio = GET_LIO(netdev); 597 int i; 598 599 if (lio->linfo.link.s.link_up) { 600 for (i = 0; i < netdev->real_num_tx_queues; i++) 601 netif_start_subqueue(netdev, i); 602 } 603 } 604 605 static inline int skb_iq(struct octeon_device *oct, struct sk_buff *skb) 606 { 607 return skb->queue_mapping % oct->num_iqs; 608 } 609 610 /** 611 * Remove the node at the head of the list. The list would be empty at 612 * the end of this call if there are no more nodes in the list. 613 */ 614 static inline struct list_head *lio_list_delete_head(struct list_head *root) 615 { 616 struct list_head *node; 617 618 if (root->prev == root && root->next == root) 619 node = NULL; 620 else 621 node = root->next; 622 623 if (node) 624 list_del(node); 625 626 return node; 627 } 628 629 #endif 630