1 /* 2 * Copyright (c) 2006-2008 Chelsio, Inc. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/list.h> 34 #include <linux/slab.h> 35 #include <net/neighbour.h> 36 #include <linux/notifier.h> 37 #include <linux/atomic.h> 38 #include <linux/proc_fs.h> 39 #include <linux/if_vlan.h> 40 #include <net/netevent.h> 41 #include <linux/highmem.h> 42 #include <linux/vmalloc.h> 43 #include <linux/export.h> 44 45 #include "common.h" 46 #include "regs.h" 47 #include "cxgb3_ioctl.h" 48 #include "cxgb3_ctl_defs.h" 49 #include "cxgb3_defs.h" 50 #include "l2t.h" 51 #include "firmware_exports.h" 52 #include "cxgb3_offload.h" 53 54 static LIST_HEAD(client_list); 55 static LIST_HEAD(ofld_dev_list); 56 static DEFINE_MUTEX(cxgb3_db_lock); 57 58 static DEFINE_RWLOCK(adapter_list_lock); 59 static LIST_HEAD(adapter_list); 60 61 static const unsigned int MAX_ATIDS = 64 * 1024; 62 static const unsigned int ATID_BASE = 0x10000; 63 64 static void cxgb_neigh_update(struct neighbour *neigh); 65 static void cxgb_redirect(struct dst_entry *old, struct dst_entry *new); 66 67 static inline int offload_activated(struct t3cdev *tdev) 68 { 69 const struct adapter *adapter = tdev2adap(tdev); 70 71 return test_bit(OFFLOAD_DEVMAP_BIT, &adapter->open_device_map); 72 } 73 74 /** 75 * cxgb3_register_client - register an offload client 76 * @client: the client 77 * 78 * Add the client to the client list, 79 * and call backs the client for each activated offload device 80 */ 81 void cxgb3_register_client(struct cxgb3_client *client) 82 { 83 struct t3cdev *tdev; 84 85 mutex_lock(&cxgb3_db_lock); 86 list_add_tail(&client->client_list, &client_list); 87 88 if (client->add) { 89 list_for_each_entry(tdev, &ofld_dev_list, ofld_dev_list) { 90 if (offload_activated(tdev)) 91 client->add(tdev); 92 } 93 } 94 mutex_unlock(&cxgb3_db_lock); 95 } 96 97 EXPORT_SYMBOL(cxgb3_register_client); 98 99 /** 100 * cxgb3_unregister_client - unregister an offload client 101 * @client: the client 102 * 103 * Remove the client to the client list, 104 * and call backs the client for each activated offload device. 105 */ 106 void cxgb3_unregister_client(struct cxgb3_client *client) 107 { 108 struct t3cdev *tdev; 109 110 mutex_lock(&cxgb3_db_lock); 111 list_del(&client->client_list); 112 113 if (client->remove) { 114 list_for_each_entry(tdev, &ofld_dev_list, ofld_dev_list) { 115 if (offload_activated(tdev)) 116 client->remove(tdev); 117 } 118 } 119 mutex_unlock(&cxgb3_db_lock); 120 } 121 122 EXPORT_SYMBOL(cxgb3_unregister_client); 123 124 /** 125 * cxgb3_add_clients - activate registered clients for an offload device 126 * @tdev: the offload device 127 * 128 * Call backs all registered clients once a offload device is activated 129 */ 130 void cxgb3_add_clients(struct t3cdev *tdev) 131 { 132 struct cxgb3_client *client; 133 134 mutex_lock(&cxgb3_db_lock); 135 list_for_each_entry(client, &client_list, client_list) { 136 if (client->add) 137 client->add(tdev); 138 } 139 mutex_unlock(&cxgb3_db_lock); 140 } 141 142 /** 143 * cxgb3_remove_clients - deactivates registered clients 144 * for an offload device 145 * @tdev: the offload device 146 * 147 * Call backs all registered clients once a offload device is deactivated 148 */ 149 void cxgb3_remove_clients(struct t3cdev *tdev) 150 { 151 struct cxgb3_client *client; 152 153 mutex_lock(&cxgb3_db_lock); 154 list_for_each_entry(client, &client_list, client_list) { 155 if (client->remove) 156 client->remove(tdev); 157 } 158 mutex_unlock(&cxgb3_db_lock); 159 } 160 161 void cxgb3_event_notify(struct t3cdev *tdev, u32 event, u32 port) 162 { 163 struct cxgb3_client *client; 164 165 mutex_lock(&cxgb3_db_lock); 166 list_for_each_entry(client, &client_list, client_list) { 167 if (client->event_handler) 168 client->event_handler(tdev, event, port); 169 } 170 mutex_unlock(&cxgb3_db_lock); 171 } 172 173 static struct net_device *get_iff_from_mac(struct adapter *adapter, 174 const unsigned char *mac, 175 unsigned int vlan) 176 { 177 int i; 178 179 for_each_port(adapter, i) { 180 struct net_device *dev = adapter->port[i]; 181 182 if (!memcmp(dev->dev_addr, mac, ETH_ALEN)) { 183 if (vlan && vlan != VLAN_VID_MASK) { 184 rcu_read_lock(); 185 dev = __vlan_find_dev_deep(dev, vlan); 186 rcu_read_unlock(); 187 } else if (netif_is_bond_slave(dev)) { 188 while (dev->master) 189 dev = dev->master; 190 } 191 return dev; 192 } 193 } 194 return NULL; 195 } 196 197 static int cxgb_ulp_iscsi_ctl(struct adapter *adapter, unsigned int req, 198 void *data) 199 { 200 int i; 201 int ret = 0; 202 unsigned int val = 0; 203 struct ulp_iscsi_info *uiip = data; 204 205 switch (req) { 206 case ULP_ISCSI_GET_PARAMS: 207 uiip->pdev = adapter->pdev; 208 uiip->llimit = t3_read_reg(adapter, A_ULPRX_ISCSI_LLIMIT); 209 uiip->ulimit = t3_read_reg(adapter, A_ULPRX_ISCSI_ULIMIT); 210 uiip->tagmask = t3_read_reg(adapter, A_ULPRX_ISCSI_TAGMASK); 211 212 val = t3_read_reg(adapter, A_ULPRX_ISCSI_PSZ); 213 for (i = 0; i < 4; i++, val >>= 8) 214 uiip->pgsz_factor[i] = val & 0xFF; 215 216 val = t3_read_reg(adapter, A_TP_PARA_REG7); 217 uiip->max_txsz = 218 uiip->max_rxsz = min((val >> S_PMMAXXFERLEN0)&M_PMMAXXFERLEN0, 219 (val >> S_PMMAXXFERLEN1)&M_PMMAXXFERLEN1); 220 /* 221 * On tx, the iscsi pdu has to be <= tx page size and has to 222 * fit into the Tx PM FIFO. 223 */ 224 val = min(adapter->params.tp.tx_pg_size, 225 t3_read_reg(adapter, A_PM1_TX_CFG) >> 17); 226 uiip->max_txsz = min(val, uiip->max_txsz); 227 228 /* set MaxRxData to 16224 */ 229 val = t3_read_reg(adapter, A_TP_PARA_REG2); 230 if ((val >> S_MAXRXDATA) != 0x3f60) { 231 val &= (M_RXCOALESCESIZE << S_RXCOALESCESIZE); 232 val |= V_MAXRXDATA(0x3f60); 233 printk(KERN_INFO 234 "%s, iscsi set MaxRxData to 16224 (0x%x).\n", 235 adapter->name, val); 236 t3_write_reg(adapter, A_TP_PARA_REG2, val); 237 } 238 239 /* 240 * on rx, the iscsi pdu has to be < rx page size and the 241 * the max rx data length programmed in TP 242 */ 243 val = min(adapter->params.tp.rx_pg_size, 244 ((t3_read_reg(adapter, A_TP_PARA_REG2)) >> 245 S_MAXRXDATA) & M_MAXRXDATA); 246 uiip->max_rxsz = min(val, uiip->max_rxsz); 247 break; 248 case ULP_ISCSI_SET_PARAMS: 249 t3_write_reg(adapter, A_ULPRX_ISCSI_TAGMASK, uiip->tagmask); 250 /* program the ddp page sizes */ 251 for (i = 0; i < 4; i++) 252 val |= (uiip->pgsz_factor[i] & 0xF) << (8 * i); 253 if (val && (val != t3_read_reg(adapter, A_ULPRX_ISCSI_PSZ))) { 254 printk(KERN_INFO 255 "%s, setting iscsi pgsz 0x%x, %u,%u,%u,%u.\n", 256 adapter->name, val, uiip->pgsz_factor[0], 257 uiip->pgsz_factor[1], uiip->pgsz_factor[2], 258 uiip->pgsz_factor[3]); 259 t3_write_reg(adapter, A_ULPRX_ISCSI_PSZ, val); 260 } 261 break; 262 default: 263 ret = -EOPNOTSUPP; 264 } 265 return ret; 266 } 267 268 /* Response queue used for RDMA events. */ 269 #define ASYNC_NOTIF_RSPQ 0 270 271 static int cxgb_rdma_ctl(struct adapter *adapter, unsigned int req, void *data) 272 { 273 int ret = 0; 274 275 switch (req) { 276 case RDMA_GET_PARAMS: { 277 struct rdma_info *rdma = data; 278 struct pci_dev *pdev = adapter->pdev; 279 280 rdma->udbell_physbase = pci_resource_start(pdev, 2); 281 rdma->udbell_len = pci_resource_len(pdev, 2); 282 rdma->tpt_base = 283 t3_read_reg(adapter, A_ULPTX_TPT_LLIMIT); 284 rdma->tpt_top = t3_read_reg(adapter, A_ULPTX_TPT_ULIMIT); 285 rdma->pbl_base = 286 t3_read_reg(adapter, A_ULPTX_PBL_LLIMIT); 287 rdma->pbl_top = t3_read_reg(adapter, A_ULPTX_PBL_ULIMIT); 288 rdma->rqt_base = t3_read_reg(adapter, A_ULPRX_RQ_LLIMIT); 289 rdma->rqt_top = t3_read_reg(adapter, A_ULPRX_RQ_ULIMIT); 290 rdma->kdb_addr = adapter->regs + A_SG_KDOORBELL; 291 rdma->pdev = pdev; 292 break; 293 } 294 case RDMA_CQ_OP:{ 295 unsigned long flags; 296 struct rdma_cq_op *rdma = data; 297 298 /* may be called in any context */ 299 spin_lock_irqsave(&adapter->sge.reg_lock, flags); 300 ret = t3_sge_cqcntxt_op(adapter, rdma->id, rdma->op, 301 rdma->credits); 302 spin_unlock_irqrestore(&adapter->sge.reg_lock, flags); 303 break; 304 } 305 case RDMA_GET_MEM:{ 306 struct ch_mem_range *t = data; 307 struct mc7 *mem; 308 309 if ((t->addr & 7) || (t->len & 7)) 310 return -EINVAL; 311 if (t->mem_id == MEM_CM) 312 mem = &adapter->cm; 313 else if (t->mem_id == MEM_PMRX) 314 mem = &adapter->pmrx; 315 else if (t->mem_id == MEM_PMTX) 316 mem = &adapter->pmtx; 317 else 318 return -EINVAL; 319 320 ret = 321 t3_mc7_bd_read(mem, t->addr / 8, t->len / 8, 322 (u64 *) t->buf); 323 if (ret) 324 return ret; 325 break; 326 } 327 case RDMA_CQ_SETUP:{ 328 struct rdma_cq_setup *rdma = data; 329 330 spin_lock_irq(&adapter->sge.reg_lock); 331 ret = 332 t3_sge_init_cqcntxt(adapter, rdma->id, 333 rdma->base_addr, rdma->size, 334 ASYNC_NOTIF_RSPQ, 335 rdma->ovfl_mode, rdma->credits, 336 rdma->credit_thres); 337 spin_unlock_irq(&adapter->sge.reg_lock); 338 break; 339 } 340 case RDMA_CQ_DISABLE: 341 spin_lock_irq(&adapter->sge.reg_lock); 342 ret = t3_sge_disable_cqcntxt(adapter, *(unsigned int *)data); 343 spin_unlock_irq(&adapter->sge.reg_lock); 344 break; 345 case RDMA_CTRL_QP_SETUP:{ 346 struct rdma_ctrlqp_setup *rdma = data; 347 348 spin_lock_irq(&adapter->sge.reg_lock); 349 ret = t3_sge_init_ecntxt(adapter, FW_RI_SGEEC_START, 0, 350 SGE_CNTXT_RDMA, 351 ASYNC_NOTIF_RSPQ, 352 rdma->base_addr, rdma->size, 353 FW_RI_TID_START, 1, 0); 354 spin_unlock_irq(&adapter->sge.reg_lock); 355 break; 356 } 357 case RDMA_GET_MIB: { 358 spin_lock(&adapter->stats_lock); 359 t3_tp_get_mib_stats(adapter, (struct tp_mib_stats *)data); 360 spin_unlock(&adapter->stats_lock); 361 break; 362 } 363 default: 364 ret = -EOPNOTSUPP; 365 } 366 return ret; 367 } 368 369 static int cxgb_offload_ctl(struct t3cdev *tdev, unsigned int req, void *data) 370 { 371 struct adapter *adapter = tdev2adap(tdev); 372 struct tid_range *tid; 373 struct mtutab *mtup; 374 struct iff_mac *iffmacp; 375 struct ddp_params *ddpp; 376 struct adap_ports *ports; 377 struct ofld_page_info *rx_page_info; 378 struct tp_params *tp = &adapter->params.tp; 379 int i; 380 381 switch (req) { 382 case GET_MAX_OUTSTANDING_WR: 383 *(unsigned int *)data = FW_WR_NUM; 384 break; 385 case GET_WR_LEN: 386 *(unsigned int *)data = WR_FLITS; 387 break; 388 case GET_TX_MAX_CHUNK: 389 *(unsigned int *)data = 1 << 20; /* 1MB */ 390 break; 391 case GET_TID_RANGE: 392 tid = data; 393 tid->num = t3_mc5_size(&adapter->mc5) - 394 adapter->params.mc5.nroutes - 395 adapter->params.mc5.nfilters - adapter->params.mc5.nservers; 396 tid->base = 0; 397 break; 398 case GET_STID_RANGE: 399 tid = data; 400 tid->num = adapter->params.mc5.nservers; 401 tid->base = t3_mc5_size(&adapter->mc5) - tid->num - 402 adapter->params.mc5.nfilters - adapter->params.mc5.nroutes; 403 break; 404 case GET_L2T_CAPACITY: 405 *(unsigned int *)data = 2048; 406 break; 407 case GET_MTUS: 408 mtup = data; 409 mtup->size = NMTUS; 410 mtup->mtus = adapter->params.mtus; 411 break; 412 case GET_IFF_FROM_MAC: 413 iffmacp = data; 414 iffmacp->dev = get_iff_from_mac(adapter, iffmacp->mac_addr, 415 iffmacp->vlan_tag & 416 VLAN_VID_MASK); 417 break; 418 case GET_DDP_PARAMS: 419 ddpp = data; 420 ddpp->llimit = t3_read_reg(adapter, A_ULPRX_TDDP_LLIMIT); 421 ddpp->ulimit = t3_read_reg(adapter, A_ULPRX_TDDP_ULIMIT); 422 ddpp->tag_mask = t3_read_reg(adapter, A_ULPRX_TDDP_TAGMASK); 423 break; 424 case GET_PORTS: 425 ports = data; 426 ports->nports = adapter->params.nports; 427 for_each_port(adapter, i) 428 ports->lldevs[i] = adapter->port[i]; 429 break; 430 case ULP_ISCSI_GET_PARAMS: 431 case ULP_ISCSI_SET_PARAMS: 432 if (!offload_running(adapter)) 433 return -EAGAIN; 434 return cxgb_ulp_iscsi_ctl(adapter, req, data); 435 case RDMA_GET_PARAMS: 436 case RDMA_CQ_OP: 437 case RDMA_CQ_SETUP: 438 case RDMA_CQ_DISABLE: 439 case RDMA_CTRL_QP_SETUP: 440 case RDMA_GET_MEM: 441 case RDMA_GET_MIB: 442 if (!offload_running(adapter)) 443 return -EAGAIN; 444 return cxgb_rdma_ctl(adapter, req, data); 445 case GET_RX_PAGE_INFO: 446 rx_page_info = data; 447 rx_page_info->page_size = tp->rx_pg_size; 448 rx_page_info->num = tp->rx_num_pgs; 449 break; 450 case GET_ISCSI_IPV4ADDR: { 451 struct iscsi_ipv4addr *p = data; 452 struct port_info *pi = netdev_priv(p->dev); 453 p->ipv4addr = pi->iscsi_ipv4addr; 454 break; 455 } 456 case GET_EMBEDDED_INFO: { 457 struct ch_embedded_info *e = data; 458 459 spin_lock(&adapter->stats_lock); 460 t3_get_fw_version(adapter, &e->fw_vers); 461 t3_get_tp_version(adapter, &e->tp_vers); 462 spin_unlock(&adapter->stats_lock); 463 break; 464 } 465 default: 466 return -EOPNOTSUPP; 467 } 468 return 0; 469 } 470 471 /* 472 * Dummy handler for Rx offload packets in case we get an offload packet before 473 * proper processing is setup. This complains and drops the packet as it isn't 474 * normal to get offload packets at this stage. 475 */ 476 static int rx_offload_blackhole(struct t3cdev *dev, struct sk_buff **skbs, 477 int n) 478 { 479 while (n--) 480 dev_kfree_skb_any(skbs[n]); 481 return 0; 482 } 483 484 static void dummy_neigh_update(struct t3cdev *dev, struct neighbour *neigh) 485 { 486 } 487 488 void cxgb3_set_dummy_ops(struct t3cdev *dev) 489 { 490 dev->recv = rx_offload_blackhole; 491 dev->neigh_update = dummy_neigh_update; 492 } 493 494 /* 495 * Free an active-open TID. 496 */ 497 void *cxgb3_free_atid(struct t3cdev *tdev, int atid) 498 { 499 struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; 500 union active_open_entry *p = atid2entry(t, atid); 501 void *ctx = p->t3c_tid.ctx; 502 503 spin_lock_bh(&t->atid_lock); 504 p->next = t->afree; 505 t->afree = p; 506 t->atids_in_use--; 507 spin_unlock_bh(&t->atid_lock); 508 509 return ctx; 510 } 511 512 EXPORT_SYMBOL(cxgb3_free_atid); 513 514 /* 515 * Free a server TID and return it to the free pool. 516 */ 517 void cxgb3_free_stid(struct t3cdev *tdev, int stid) 518 { 519 struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; 520 union listen_entry *p = stid2entry(t, stid); 521 522 spin_lock_bh(&t->stid_lock); 523 p->next = t->sfree; 524 t->sfree = p; 525 t->stids_in_use--; 526 spin_unlock_bh(&t->stid_lock); 527 } 528 529 EXPORT_SYMBOL(cxgb3_free_stid); 530 531 void cxgb3_insert_tid(struct t3cdev *tdev, struct cxgb3_client *client, 532 void *ctx, unsigned int tid) 533 { 534 struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; 535 536 t->tid_tab[tid].client = client; 537 t->tid_tab[tid].ctx = ctx; 538 atomic_inc(&t->tids_in_use); 539 } 540 541 EXPORT_SYMBOL(cxgb3_insert_tid); 542 543 /* 544 * Populate a TID_RELEASE WR. The skb must be already propely sized. 545 */ 546 static inline void mk_tid_release(struct sk_buff *skb, unsigned int tid) 547 { 548 struct cpl_tid_release *req; 549 550 skb->priority = CPL_PRIORITY_SETUP; 551 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req)); 552 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 553 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid)); 554 } 555 556 static void t3_process_tid_release_list(struct work_struct *work) 557 { 558 struct t3c_data *td = container_of(work, struct t3c_data, 559 tid_release_task); 560 struct sk_buff *skb; 561 struct t3cdev *tdev = td->dev; 562 563 564 spin_lock_bh(&td->tid_release_lock); 565 while (td->tid_release_list) { 566 struct t3c_tid_entry *p = td->tid_release_list; 567 568 td->tid_release_list = p->ctx; 569 spin_unlock_bh(&td->tid_release_lock); 570 571 skb = alloc_skb(sizeof(struct cpl_tid_release), 572 GFP_KERNEL); 573 if (!skb) 574 skb = td->nofail_skb; 575 if (!skb) { 576 spin_lock_bh(&td->tid_release_lock); 577 p->ctx = (void *)td->tid_release_list; 578 td->tid_release_list = (struct t3c_tid_entry *)p; 579 break; 580 } 581 mk_tid_release(skb, p - td->tid_maps.tid_tab); 582 cxgb3_ofld_send(tdev, skb); 583 p->ctx = NULL; 584 if (skb == td->nofail_skb) 585 td->nofail_skb = 586 alloc_skb(sizeof(struct cpl_tid_release), 587 GFP_KERNEL); 588 spin_lock_bh(&td->tid_release_lock); 589 } 590 td->release_list_incomplete = (td->tid_release_list == NULL) ? 0 : 1; 591 spin_unlock_bh(&td->tid_release_lock); 592 593 if (!td->nofail_skb) 594 td->nofail_skb = 595 alloc_skb(sizeof(struct cpl_tid_release), 596 GFP_KERNEL); 597 } 598 599 /* use ctx as a next pointer in the tid release list */ 600 void cxgb3_queue_tid_release(struct t3cdev *tdev, unsigned int tid) 601 { 602 struct t3c_data *td = T3C_DATA(tdev); 603 struct t3c_tid_entry *p = &td->tid_maps.tid_tab[tid]; 604 605 spin_lock_bh(&td->tid_release_lock); 606 p->ctx = (void *)td->tid_release_list; 607 p->client = NULL; 608 td->tid_release_list = p; 609 if (!p->ctx || td->release_list_incomplete) 610 schedule_work(&td->tid_release_task); 611 spin_unlock_bh(&td->tid_release_lock); 612 } 613 614 EXPORT_SYMBOL(cxgb3_queue_tid_release); 615 616 /* 617 * Remove a tid from the TID table. A client may defer processing its last 618 * CPL message if it is locked at the time it arrives, and while the message 619 * sits in the client's backlog the TID may be reused for another connection. 620 * To handle this we atomically switch the TID association if it still points 621 * to the original client context. 622 */ 623 void cxgb3_remove_tid(struct t3cdev *tdev, void *ctx, unsigned int tid) 624 { 625 struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; 626 627 BUG_ON(tid >= t->ntids); 628 if (tdev->type == T3A) 629 (void)cmpxchg(&t->tid_tab[tid].ctx, ctx, NULL); 630 else { 631 struct sk_buff *skb; 632 633 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC); 634 if (likely(skb)) { 635 mk_tid_release(skb, tid); 636 cxgb3_ofld_send(tdev, skb); 637 t->tid_tab[tid].ctx = NULL; 638 } else 639 cxgb3_queue_tid_release(tdev, tid); 640 } 641 atomic_dec(&t->tids_in_use); 642 } 643 644 EXPORT_SYMBOL(cxgb3_remove_tid); 645 646 int cxgb3_alloc_atid(struct t3cdev *tdev, struct cxgb3_client *client, 647 void *ctx) 648 { 649 int atid = -1; 650 struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; 651 652 spin_lock_bh(&t->atid_lock); 653 if (t->afree && 654 t->atids_in_use + atomic_read(&t->tids_in_use) + MC5_MIN_TIDS <= 655 t->ntids) { 656 union active_open_entry *p = t->afree; 657 658 atid = (p - t->atid_tab) + t->atid_base; 659 t->afree = p->next; 660 p->t3c_tid.ctx = ctx; 661 p->t3c_tid.client = client; 662 t->atids_in_use++; 663 } 664 spin_unlock_bh(&t->atid_lock); 665 return atid; 666 } 667 668 EXPORT_SYMBOL(cxgb3_alloc_atid); 669 670 int cxgb3_alloc_stid(struct t3cdev *tdev, struct cxgb3_client *client, 671 void *ctx) 672 { 673 int stid = -1; 674 struct tid_info *t = &(T3C_DATA(tdev))->tid_maps; 675 676 spin_lock_bh(&t->stid_lock); 677 if (t->sfree) { 678 union listen_entry *p = t->sfree; 679 680 stid = (p - t->stid_tab) + t->stid_base; 681 t->sfree = p->next; 682 p->t3c_tid.ctx = ctx; 683 p->t3c_tid.client = client; 684 t->stids_in_use++; 685 } 686 spin_unlock_bh(&t->stid_lock); 687 return stid; 688 } 689 690 EXPORT_SYMBOL(cxgb3_alloc_stid); 691 692 /* Get the t3cdev associated with a net_device */ 693 struct t3cdev *dev2t3cdev(struct net_device *dev) 694 { 695 const struct port_info *pi = netdev_priv(dev); 696 697 return (struct t3cdev *)pi->adapter; 698 } 699 700 EXPORT_SYMBOL(dev2t3cdev); 701 702 static int do_smt_write_rpl(struct t3cdev *dev, struct sk_buff *skb) 703 { 704 struct cpl_smt_write_rpl *rpl = cplhdr(skb); 705 706 if (rpl->status != CPL_ERR_NONE) 707 printk(KERN_ERR 708 "Unexpected SMT_WRITE_RPL status %u for entry %u\n", 709 rpl->status, GET_TID(rpl)); 710 711 return CPL_RET_BUF_DONE; 712 } 713 714 static int do_l2t_write_rpl(struct t3cdev *dev, struct sk_buff *skb) 715 { 716 struct cpl_l2t_write_rpl *rpl = cplhdr(skb); 717 718 if (rpl->status != CPL_ERR_NONE) 719 printk(KERN_ERR 720 "Unexpected L2T_WRITE_RPL status %u for entry %u\n", 721 rpl->status, GET_TID(rpl)); 722 723 return CPL_RET_BUF_DONE; 724 } 725 726 static int do_rte_write_rpl(struct t3cdev *dev, struct sk_buff *skb) 727 { 728 struct cpl_rte_write_rpl *rpl = cplhdr(skb); 729 730 if (rpl->status != CPL_ERR_NONE) 731 printk(KERN_ERR 732 "Unexpected RTE_WRITE_RPL status %u for entry %u\n", 733 rpl->status, GET_TID(rpl)); 734 735 return CPL_RET_BUF_DONE; 736 } 737 738 static int do_act_open_rpl(struct t3cdev *dev, struct sk_buff *skb) 739 { 740 struct cpl_act_open_rpl *rpl = cplhdr(skb); 741 unsigned int atid = G_TID(ntohl(rpl->atid)); 742 struct t3c_tid_entry *t3c_tid; 743 744 t3c_tid = lookup_atid(&(T3C_DATA(dev))->tid_maps, atid); 745 if (t3c_tid && t3c_tid->ctx && t3c_tid->client && 746 t3c_tid->client->handlers && 747 t3c_tid->client->handlers[CPL_ACT_OPEN_RPL]) { 748 return t3c_tid->client->handlers[CPL_ACT_OPEN_RPL] (dev, skb, 749 t3c_tid-> 750 ctx); 751 } else { 752 printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", 753 dev->name, CPL_ACT_OPEN_RPL); 754 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 755 } 756 } 757 758 static int do_stid_rpl(struct t3cdev *dev, struct sk_buff *skb) 759 { 760 union opcode_tid *p = cplhdr(skb); 761 unsigned int stid = G_TID(ntohl(p->opcode_tid)); 762 struct t3c_tid_entry *t3c_tid; 763 764 t3c_tid = lookup_stid(&(T3C_DATA(dev))->tid_maps, stid); 765 if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && 766 t3c_tid->client->handlers[p->opcode]) { 767 return t3c_tid->client->handlers[p->opcode] (dev, skb, 768 t3c_tid->ctx); 769 } else { 770 printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", 771 dev->name, p->opcode); 772 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 773 } 774 } 775 776 static int do_hwtid_rpl(struct t3cdev *dev, struct sk_buff *skb) 777 { 778 union opcode_tid *p = cplhdr(skb); 779 unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); 780 struct t3c_tid_entry *t3c_tid; 781 782 t3c_tid = lookup_tid(&(T3C_DATA(dev))->tid_maps, hwtid); 783 if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && 784 t3c_tid->client->handlers[p->opcode]) { 785 return t3c_tid->client->handlers[p->opcode] 786 (dev, skb, t3c_tid->ctx); 787 } else { 788 printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", 789 dev->name, p->opcode); 790 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 791 } 792 } 793 794 static int do_cr(struct t3cdev *dev, struct sk_buff *skb) 795 { 796 struct cpl_pass_accept_req *req = cplhdr(skb); 797 unsigned int stid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 798 struct tid_info *t = &(T3C_DATA(dev))->tid_maps; 799 struct t3c_tid_entry *t3c_tid; 800 unsigned int tid = GET_TID(req); 801 802 if (unlikely(tid >= t->ntids)) { 803 printk("%s: passive open TID %u too large\n", 804 dev->name, tid); 805 t3_fatal_err(tdev2adap(dev)); 806 return CPL_RET_BUF_DONE; 807 } 808 809 t3c_tid = lookup_stid(t, stid); 810 if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && 811 t3c_tid->client->handlers[CPL_PASS_ACCEPT_REQ]) { 812 return t3c_tid->client->handlers[CPL_PASS_ACCEPT_REQ] 813 (dev, skb, t3c_tid->ctx); 814 } else { 815 printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", 816 dev->name, CPL_PASS_ACCEPT_REQ); 817 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 818 } 819 } 820 821 /* 822 * Returns an sk_buff for a reply CPL message of size len. If the input 823 * sk_buff has no other users it is trimmed and reused, otherwise a new buffer 824 * is allocated. The input skb must be of size at least len. Note that this 825 * operation does not destroy the original skb data even if it decides to reuse 826 * the buffer. 827 */ 828 static struct sk_buff *cxgb3_get_cpl_reply_skb(struct sk_buff *skb, size_t len, 829 gfp_t gfp) 830 { 831 if (likely(!skb_cloned(skb))) { 832 BUG_ON(skb->len < len); 833 __skb_trim(skb, len); 834 skb_get(skb); 835 } else { 836 skb = alloc_skb(len, gfp); 837 if (skb) 838 __skb_put(skb, len); 839 } 840 return skb; 841 } 842 843 static int do_abort_req_rss(struct t3cdev *dev, struct sk_buff *skb) 844 { 845 union opcode_tid *p = cplhdr(skb); 846 unsigned int hwtid = G_TID(ntohl(p->opcode_tid)); 847 struct t3c_tid_entry *t3c_tid; 848 849 t3c_tid = lookup_tid(&(T3C_DATA(dev))->tid_maps, hwtid); 850 if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && 851 t3c_tid->client->handlers[p->opcode]) { 852 return t3c_tid->client->handlers[p->opcode] 853 (dev, skb, t3c_tid->ctx); 854 } else { 855 struct cpl_abort_req_rss *req = cplhdr(skb); 856 struct cpl_abort_rpl *rpl; 857 struct sk_buff *reply_skb; 858 unsigned int tid = GET_TID(req); 859 u8 cmd = req->status; 860 861 if (req->status == CPL_ERR_RTX_NEG_ADVICE || 862 req->status == CPL_ERR_PERSIST_NEG_ADVICE) 863 goto out; 864 865 reply_skb = cxgb3_get_cpl_reply_skb(skb, 866 sizeof(struct 867 cpl_abort_rpl), 868 GFP_ATOMIC); 869 870 if (!reply_skb) { 871 printk("do_abort_req_rss: couldn't get skb!\n"); 872 goto out; 873 } 874 reply_skb->priority = CPL_PRIORITY_DATA; 875 __skb_put(reply_skb, sizeof(struct cpl_abort_rpl)); 876 rpl = cplhdr(reply_skb); 877 rpl->wr.wr_hi = 878 htonl(V_WR_OP(FW_WROPCODE_OFLD_HOST_ABORT_CON_RPL)); 879 rpl->wr.wr_lo = htonl(V_WR_TID(tid)); 880 OPCODE_TID(rpl) = htonl(MK_OPCODE_TID(CPL_ABORT_RPL, tid)); 881 rpl->cmd = cmd; 882 cxgb3_ofld_send(dev, reply_skb); 883 out: 884 return CPL_RET_BUF_DONE; 885 } 886 } 887 888 static int do_act_establish(struct t3cdev *dev, struct sk_buff *skb) 889 { 890 struct cpl_act_establish *req = cplhdr(skb); 891 unsigned int atid = G_PASS_OPEN_TID(ntohl(req->tos_tid)); 892 struct tid_info *t = &(T3C_DATA(dev))->tid_maps; 893 struct t3c_tid_entry *t3c_tid; 894 unsigned int tid = GET_TID(req); 895 896 if (unlikely(tid >= t->ntids)) { 897 printk("%s: active establish TID %u too large\n", 898 dev->name, tid); 899 t3_fatal_err(tdev2adap(dev)); 900 return CPL_RET_BUF_DONE; 901 } 902 903 t3c_tid = lookup_atid(t, atid); 904 if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && 905 t3c_tid->client->handlers[CPL_ACT_ESTABLISH]) { 906 return t3c_tid->client->handlers[CPL_ACT_ESTABLISH] 907 (dev, skb, t3c_tid->ctx); 908 } else { 909 printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", 910 dev->name, CPL_ACT_ESTABLISH); 911 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 912 } 913 } 914 915 static int do_trace(struct t3cdev *dev, struct sk_buff *skb) 916 { 917 struct cpl_trace_pkt *p = cplhdr(skb); 918 919 skb->protocol = htons(0xffff); 920 skb->dev = dev->lldev; 921 skb_pull(skb, sizeof(*p)); 922 skb_reset_mac_header(skb); 923 netif_receive_skb(skb); 924 return 0; 925 } 926 927 /* 928 * That skb would better have come from process_responses() where we abuse 929 * ->priority and ->csum to carry our data. NB: if we get to per-arch 930 * ->csum, the things might get really interesting here. 931 */ 932 933 static inline u32 get_hwtid(struct sk_buff *skb) 934 { 935 return ntohl((__force __be32)skb->priority) >> 8 & 0xfffff; 936 } 937 938 static inline u32 get_opcode(struct sk_buff *skb) 939 { 940 return G_OPCODE(ntohl((__force __be32)skb->csum)); 941 } 942 943 static int do_term(struct t3cdev *dev, struct sk_buff *skb) 944 { 945 unsigned int hwtid = get_hwtid(skb); 946 unsigned int opcode = get_opcode(skb); 947 struct t3c_tid_entry *t3c_tid; 948 949 t3c_tid = lookup_tid(&(T3C_DATA(dev))->tid_maps, hwtid); 950 if (t3c_tid && t3c_tid->ctx && t3c_tid->client->handlers && 951 t3c_tid->client->handlers[opcode]) { 952 return t3c_tid->client->handlers[opcode] (dev, skb, 953 t3c_tid->ctx); 954 } else { 955 printk(KERN_ERR "%s: received clientless CPL command 0x%x\n", 956 dev->name, opcode); 957 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 958 } 959 } 960 961 static int nb_callback(struct notifier_block *self, unsigned long event, 962 void *ctx) 963 { 964 switch (event) { 965 case (NETEVENT_NEIGH_UPDATE):{ 966 cxgb_neigh_update((struct neighbour *)ctx); 967 break; 968 } 969 case (NETEVENT_REDIRECT):{ 970 struct netevent_redirect *nr = ctx; 971 cxgb_redirect(nr->old, nr->new); 972 cxgb_neigh_update(dst_get_neighbour_noref(nr->new)); 973 break; 974 } 975 default: 976 break; 977 } 978 return 0; 979 } 980 981 static struct notifier_block nb = { 982 .notifier_call = nb_callback 983 }; 984 985 /* 986 * Process a received packet with an unknown/unexpected CPL opcode. 987 */ 988 static int do_bad_cpl(struct t3cdev *dev, struct sk_buff *skb) 989 { 990 printk(KERN_ERR "%s: received bad CPL command 0x%x\n", dev->name, 991 *skb->data); 992 return CPL_RET_BUF_DONE | CPL_RET_BAD_MSG; 993 } 994 995 /* 996 * Handlers for each CPL opcode 997 */ 998 static cpl_handler_func cpl_handlers[NUM_CPL_CMDS]; 999 1000 /* 1001 * Add a new handler to the CPL dispatch table. A NULL handler may be supplied 1002 * to unregister an existing handler. 1003 */ 1004 void t3_register_cpl_handler(unsigned int opcode, cpl_handler_func h) 1005 { 1006 if (opcode < NUM_CPL_CMDS) 1007 cpl_handlers[opcode] = h ? h : do_bad_cpl; 1008 else 1009 printk(KERN_ERR "T3C: handler registration for " 1010 "opcode %x failed\n", opcode); 1011 } 1012 1013 EXPORT_SYMBOL(t3_register_cpl_handler); 1014 1015 /* 1016 * T3CDEV's receive method. 1017 */ 1018 static int process_rx(struct t3cdev *dev, struct sk_buff **skbs, int n) 1019 { 1020 while (n--) { 1021 struct sk_buff *skb = *skbs++; 1022 unsigned int opcode = get_opcode(skb); 1023 int ret = cpl_handlers[opcode] (dev, skb); 1024 1025 #if VALIDATE_TID 1026 if (ret & CPL_RET_UNKNOWN_TID) { 1027 union opcode_tid *p = cplhdr(skb); 1028 1029 printk(KERN_ERR "%s: CPL message (opcode %u) had " 1030 "unknown TID %u\n", dev->name, opcode, 1031 G_TID(ntohl(p->opcode_tid))); 1032 } 1033 #endif 1034 if (ret & CPL_RET_BUF_DONE) 1035 kfree_skb(skb); 1036 } 1037 return 0; 1038 } 1039 1040 /* 1041 * Sends an sk_buff to a T3C driver after dealing with any active network taps. 1042 */ 1043 int cxgb3_ofld_send(struct t3cdev *dev, struct sk_buff *skb) 1044 { 1045 int r; 1046 1047 local_bh_disable(); 1048 r = dev->send(dev, skb); 1049 local_bh_enable(); 1050 return r; 1051 } 1052 1053 EXPORT_SYMBOL(cxgb3_ofld_send); 1054 1055 static int is_offloading(struct net_device *dev) 1056 { 1057 struct adapter *adapter; 1058 int i; 1059 1060 read_lock_bh(&adapter_list_lock); 1061 list_for_each_entry(adapter, &adapter_list, adapter_list) { 1062 for_each_port(adapter, i) { 1063 if (dev == adapter->port[i]) { 1064 read_unlock_bh(&adapter_list_lock); 1065 return 1; 1066 } 1067 } 1068 } 1069 read_unlock_bh(&adapter_list_lock); 1070 return 0; 1071 } 1072 1073 static void cxgb_neigh_update(struct neighbour *neigh) 1074 { 1075 struct net_device *dev = neigh->dev; 1076 1077 if (dev && (is_offloading(dev))) { 1078 struct t3cdev *tdev = dev2t3cdev(dev); 1079 1080 BUG_ON(!tdev); 1081 t3_l2t_update(tdev, neigh); 1082 } 1083 } 1084 1085 static void set_l2t_ix(struct t3cdev *tdev, u32 tid, struct l2t_entry *e) 1086 { 1087 struct sk_buff *skb; 1088 struct cpl_set_tcb_field *req; 1089 1090 skb = alloc_skb(sizeof(*req), GFP_ATOMIC); 1091 if (!skb) { 1092 printk(KERN_ERR "%s: cannot allocate skb!\n", __func__); 1093 return; 1094 } 1095 skb->priority = CPL_PRIORITY_CONTROL; 1096 req = (struct cpl_set_tcb_field *)skb_put(skb, sizeof(*req)); 1097 req->wr.wr_hi = htonl(V_WR_OP(FW_WROPCODE_FORWARD)); 1098 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); 1099 req->reply = 0; 1100 req->cpu_idx = 0; 1101 req->word = htons(W_TCB_L2T_IX); 1102 req->mask = cpu_to_be64(V_TCB_L2T_IX(M_TCB_L2T_IX)); 1103 req->val = cpu_to_be64(V_TCB_L2T_IX(e->idx)); 1104 tdev->send(tdev, skb); 1105 } 1106 1107 static void cxgb_redirect(struct dst_entry *old, struct dst_entry *new) 1108 { 1109 struct net_device *olddev, *newdev; 1110 struct tid_info *ti; 1111 struct t3cdev *tdev; 1112 u32 tid; 1113 int update_tcb; 1114 struct l2t_entry *e; 1115 struct t3c_tid_entry *te; 1116 1117 olddev = dst_get_neighbour_noref(old)->dev; 1118 newdev = dst_get_neighbour_noref(new)->dev; 1119 if (!is_offloading(olddev)) 1120 return; 1121 if (!is_offloading(newdev)) { 1122 printk(KERN_WARNING "%s: Redirect to non-offload " 1123 "device ignored.\n", __func__); 1124 return; 1125 } 1126 tdev = dev2t3cdev(olddev); 1127 BUG_ON(!tdev); 1128 if (tdev != dev2t3cdev(newdev)) { 1129 printk(KERN_WARNING "%s: Redirect to different " 1130 "offload device ignored.\n", __func__); 1131 return; 1132 } 1133 1134 /* Add new L2T entry */ 1135 e = t3_l2t_get(tdev, dst_get_neighbour_noref(new), newdev); 1136 if (!e) { 1137 printk(KERN_ERR "%s: couldn't allocate new l2t entry!\n", 1138 __func__); 1139 return; 1140 } 1141 1142 /* Walk tid table and notify clients of dst change. */ 1143 ti = &(T3C_DATA(tdev))->tid_maps; 1144 for (tid = 0; tid < ti->ntids; tid++) { 1145 te = lookup_tid(ti, tid); 1146 BUG_ON(!te); 1147 if (te && te->ctx && te->client && te->client->redirect) { 1148 update_tcb = te->client->redirect(te->ctx, old, new, e); 1149 if (update_tcb) { 1150 rcu_read_lock(); 1151 l2t_hold(L2DATA(tdev), e); 1152 rcu_read_unlock(); 1153 set_l2t_ix(tdev, tid, e); 1154 } 1155 } 1156 } 1157 l2t_release(tdev, e); 1158 } 1159 1160 /* 1161 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc. 1162 * The allocated memory is cleared. 1163 */ 1164 void *cxgb_alloc_mem(unsigned long size) 1165 { 1166 void *p = kzalloc(size, GFP_KERNEL); 1167 1168 if (!p) 1169 p = vzalloc(size); 1170 return p; 1171 } 1172 1173 /* 1174 * Free memory allocated through t3_alloc_mem(). 1175 */ 1176 void cxgb_free_mem(void *addr) 1177 { 1178 if (is_vmalloc_addr(addr)) 1179 vfree(addr); 1180 else 1181 kfree(addr); 1182 } 1183 1184 /* 1185 * Allocate and initialize the TID tables. Returns 0 on success. 1186 */ 1187 static int init_tid_tabs(struct tid_info *t, unsigned int ntids, 1188 unsigned int natids, unsigned int nstids, 1189 unsigned int atid_base, unsigned int stid_base) 1190 { 1191 unsigned long size = ntids * sizeof(*t->tid_tab) + 1192 natids * sizeof(*t->atid_tab) + nstids * sizeof(*t->stid_tab); 1193 1194 t->tid_tab = cxgb_alloc_mem(size); 1195 if (!t->tid_tab) 1196 return -ENOMEM; 1197 1198 t->stid_tab = (union listen_entry *)&t->tid_tab[ntids]; 1199 t->atid_tab = (union active_open_entry *)&t->stid_tab[nstids]; 1200 t->ntids = ntids; 1201 t->nstids = nstids; 1202 t->stid_base = stid_base; 1203 t->sfree = NULL; 1204 t->natids = natids; 1205 t->atid_base = atid_base; 1206 t->afree = NULL; 1207 t->stids_in_use = t->atids_in_use = 0; 1208 atomic_set(&t->tids_in_use, 0); 1209 spin_lock_init(&t->stid_lock); 1210 spin_lock_init(&t->atid_lock); 1211 1212 /* 1213 * Setup the free lists for stid_tab and atid_tab. 1214 */ 1215 if (nstids) { 1216 while (--nstids) 1217 t->stid_tab[nstids - 1].next = &t->stid_tab[nstids]; 1218 t->sfree = t->stid_tab; 1219 } 1220 if (natids) { 1221 while (--natids) 1222 t->atid_tab[natids - 1].next = &t->atid_tab[natids]; 1223 t->afree = t->atid_tab; 1224 } 1225 return 0; 1226 } 1227 1228 static void free_tid_maps(struct tid_info *t) 1229 { 1230 cxgb_free_mem(t->tid_tab); 1231 } 1232 1233 static inline void add_adapter(struct adapter *adap) 1234 { 1235 write_lock_bh(&adapter_list_lock); 1236 list_add_tail(&adap->adapter_list, &adapter_list); 1237 write_unlock_bh(&adapter_list_lock); 1238 } 1239 1240 static inline void remove_adapter(struct adapter *adap) 1241 { 1242 write_lock_bh(&adapter_list_lock); 1243 list_del(&adap->adapter_list); 1244 write_unlock_bh(&adapter_list_lock); 1245 } 1246 1247 int cxgb3_offload_activate(struct adapter *adapter) 1248 { 1249 struct t3cdev *dev = &adapter->tdev; 1250 int natids, err; 1251 struct t3c_data *t; 1252 struct tid_range stid_range, tid_range; 1253 struct mtutab mtutab; 1254 unsigned int l2t_capacity; 1255 1256 t = kzalloc(sizeof(*t), GFP_KERNEL); 1257 if (!t) 1258 return -ENOMEM; 1259 1260 err = -EOPNOTSUPP; 1261 if (dev->ctl(dev, GET_TX_MAX_CHUNK, &t->tx_max_chunk) < 0 || 1262 dev->ctl(dev, GET_MAX_OUTSTANDING_WR, &t->max_wrs) < 0 || 1263 dev->ctl(dev, GET_L2T_CAPACITY, &l2t_capacity) < 0 || 1264 dev->ctl(dev, GET_MTUS, &mtutab) < 0 || 1265 dev->ctl(dev, GET_TID_RANGE, &tid_range) < 0 || 1266 dev->ctl(dev, GET_STID_RANGE, &stid_range) < 0) 1267 goto out_free; 1268 1269 err = -ENOMEM; 1270 RCU_INIT_POINTER(dev->l2opt, t3_init_l2t(l2t_capacity)); 1271 if (!L2DATA(dev)) 1272 goto out_free; 1273 1274 natids = min(tid_range.num / 2, MAX_ATIDS); 1275 err = init_tid_tabs(&t->tid_maps, tid_range.num, natids, 1276 stid_range.num, ATID_BASE, stid_range.base); 1277 if (err) 1278 goto out_free_l2t; 1279 1280 t->mtus = mtutab.mtus; 1281 t->nmtus = mtutab.size; 1282 1283 INIT_WORK(&t->tid_release_task, t3_process_tid_release_list); 1284 spin_lock_init(&t->tid_release_lock); 1285 INIT_LIST_HEAD(&t->list_node); 1286 t->dev = dev; 1287 1288 T3C_DATA(dev) = t; 1289 dev->recv = process_rx; 1290 dev->neigh_update = t3_l2t_update; 1291 1292 /* Register netevent handler once */ 1293 if (list_empty(&adapter_list)) 1294 register_netevent_notifier(&nb); 1295 1296 t->nofail_skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_KERNEL); 1297 t->release_list_incomplete = 0; 1298 1299 add_adapter(adapter); 1300 return 0; 1301 1302 out_free_l2t: 1303 t3_free_l2t(L2DATA(dev)); 1304 RCU_INIT_POINTER(dev->l2opt, NULL); 1305 out_free: 1306 kfree(t); 1307 return err; 1308 } 1309 1310 static void clean_l2_data(struct rcu_head *head) 1311 { 1312 struct l2t_data *d = container_of(head, struct l2t_data, rcu_head); 1313 t3_free_l2t(d); 1314 } 1315 1316 1317 void cxgb3_offload_deactivate(struct adapter *adapter) 1318 { 1319 struct t3cdev *tdev = &adapter->tdev; 1320 struct t3c_data *t = T3C_DATA(tdev); 1321 struct l2t_data *d; 1322 1323 remove_adapter(adapter); 1324 if (list_empty(&adapter_list)) 1325 unregister_netevent_notifier(&nb); 1326 1327 free_tid_maps(&t->tid_maps); 1328 T3C_DATA(tdev) = NULL; 1329 rcu_read_lock(); 1330 d = L2DATA(tdev); 1331 rcu_read_unlock(); 1332 RCU_INIT_POINTER(tdev->l2opt, NULL); 1333 call_rcu(&d->rcu_head, clean_l2_data); 1334 if (t->nofail_skb) 1335 kfree_skb(t->nofail_skb); 1336 kfree(t); 1337 } 1338 1339 static inline void register_tdev(struct t3cdev *tdev) 1340 { 1341 static int unit; 1342 1343 mutex_lock(&cxgb3_db_lock); 1344 snprintf(tdev->name, sizeof(tdev->name), "ofld_dev%d", unit++); 1345 list_add_tail(&tdev->ofld_dev_list, &ofld_dev_list); 1346 mutex_unlock(&cxgb3_db_lock); 1347 } 1348 1349 static inline void unregister_tdev(struct t3cdev *tdev) 1350 { 1351 mutex_lock(&cxgb3_db_lock); 1352 list_del(&tdev->ofld_dev_list); 1353 mutex_unlock(&cxgb3_db_lock); 1354 } 1355 1356 static inline int adap2type(struct adapter *adapter) 1357 { 1358 int type = 0; 1359 1360 switch (adapter->params.rev) { 1361 case T3_REV_A: 1362 type = T3A; 1363 break; 1364 case T3_REV_B: 1365 case T3_REV_B2: 1366 type = T3B; 1367 break; 1368 case T3_REV_C: 1369 type = T3C; 1370 break; 1371 } 1372 return type; 1373 } 1374 1375 void __devinit cxgb3_adapter_ofld(struct adapter *adapter) 1376 { 1377 struct t3cdev *tdev = &adapter->tdev; 1378 1379 INIT_LIST_HEAD(&tdev->ofld_dev_list); 1380 1381 cxgb3_set_dummy_ops(tdev); 1382 tdev->send = t3_offload_tx; 1383 tdev->ctl = cxgb_offload_ctl; 1384 tdev->type = adap2type(adapter); 1385 1386 register_tdev(tdev); 1387 } 1388 1389 void __devexit cxgb3_adapter_unofld(struct adapter *adapter) 1390 { 1391 struct t3cdev *tdev = &adapter->tdev; 1392 1393 tdev->recv = NULL; 1394 tdev->neigh_update = NULL; 1395 1396 unregister_tdev(tdev); 1397 } 1398 1399 void __init cxgb3_offload_init(void) 1400 { 1401 int i; 1402 1403 for (i = 0; i < NUM_CPL_CMDS; ++i) 1404 cpl_handlers[i] = do_bad_cpl; 1405 1406 t3_register_cpl_handler(CPL_SMT_WRITE_RPL, do_smt_write_rpl); 1407 t3_register_cpl_handler(CPL_L2T_WRITE_RPL, do_l2t_write_rpl); 1408 t3_register_cpl_handler(CPL_RTE_WRITE_RPL, do_rte_write_rpl); 1409 t3_register_cpl_handler(CPL_PASS_OPEN_RPL, do_stid_rpl); 1410 t3_register_cpl_handler(CPL_CLOSE_LISTSRV_RPL, do_stid_rpl); 1411 t3_register_cpl_handler(CPL_PASS_ACCEPT_REQ, do_cr); 1412 t3_register_cpl_handler(CPL_PASS_ESTABLISH, do_hwtid_rpl); 1413 t3_register_cpl_handler(CPL_ABORT_RPL_RSS, do_hwtid_rpl); 1414 t3_register_cpl_handler(CPL_ABORT_RPL, do_hwtid_rpl); 1415 t3_register_cpl_handler(CPL_RX_URG_NOTIFY, do_hwtid_rpl); 1416 t3_register_cpl_handler(CPL_RX_DATA, do_hwtid_rpl); 1417 t3_register_cpl_handler(CPL_TX_DATA_ACK, do_hwtid_rpl); 1418 t3_register_cpl_handler(CPL_TX_DMA_ACK, do_hwtid_rpl); 1419 t3_register_cpl_handler(CPL_ACT_OPEN_RPL, do_act_open_rpl); 1420 t3_register_cpl_handler(CPL_PEER_CLOSE, do_hwtid_rpl); 1421 t3_register_cpl_handler(CPL_CLOSE_CON_RPL, do_hwtid_rpl); 1422 t3_register_cpl_handler(CPL_ABORT_REQ_RSS, do_abort_req_rss); 1423 t3_register_cpl_handler(CPL_ACT_ESTABLISH, do_act_establish); 1424 t3_register_cpl_handler(CPL_SET_TCB_RPL, do_hwtid_rpl); 1425 t3_register_cpl_handler(CPL_GET_TCB_RPL, do_hwtid_rpl); 1426 t3_register_cpl_handler(CPL_RDMA_TERMINATE, do_term); 1427 t3_register_cpl_handler(CPL_RDMA_EC_STATUS, do_hwtid_rpl); 1428 t3_register_cpl_handler(CPL_TRACE_PKT, do_trace); 1429 t3_register_cpl_handler(CPL_RX_DATA_DDP, do_hwtid_rpl); 1430 t3_register_cpl_handler(CPL_RX_DDP_COMPLETE, do_hwtid_rpl); 1431 t3_register_cpl_handler(CPL_ISCSI_HDR, do_hwtid_rpl); 1432 } 1433