1 /* 2 * libcxgbi.c: Chelsio common library for T3/T4 iSCSI driver. 3 * 4 * Copyright (c) 2010-2015 Chelsio Communications, Inc. 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License as published by 8 * the Free Software Foundation. 9 * 10 * Written by: Karen Xie (kxie@chelsio.com) 11 * Written by: Rakesh Ranjan (rranjan@chelsio.com) 12 */ 13 14 #define pr_fmt(fmt) KBUILD_MODNAME ":%s: " fmt, __func__ 15 16 #include <linux/skbuff.h> 17 #include <linux/crypto.h> 18 #include <linux/scatterlist.h> 19 #include <linux/pci.h> 20 #include <scsi/scsi.h> 21 #include <scsi/scsi_cmnd.h> 22 #include <scsi/scsi_host.h> 23 #include <linux/if_vlan.h> 24 #include <linux/inet.h> 25 #include <net/dst.h> 26 #include <net/route.h> 27 #include <net/ipv6.h> 28 #include <net/ip6_route.h> 29 #include <net/addrconf.h> 30 31 #include <linux/inetdevice.h> /* ip_dev_find */ 32 #include <linux/module.h> 33 #include <net/tcp.h> 34 35 static unsigned int dbg_level; 36 37 #include "libcxgbi.h" 38 39 #define DRV_MODULE_NAME "libcxgbi" 40 #define DRV_MODULE_DESC "Chelsio iSCSI driver library" 41 #define DRV_MODULE_VERSION "0.9.1-ko" 42 #define DRV_MODULE_RELDATE "Apr. 2015" 43 44 static char version[] = 45 DRV_MODULE_DESC " " DRV_MODULE_NAME 46 " v" DRV_MODULE_VERSION " (" DRV_MODULE_RELDATE ")\n"; 47 48 MODULE_AUTHOR("Chelsio Communications, Inc."); 49 MODULE_DESCRIPTION(DRV_MODULE_DESC); 50 MODULE_VERSION(DRV_MODULE_VERSION); 51 MODULE_LICENSE("GPL"); 52 53 module_param(dbg_level, uint, 0644); 54 MODULE_PARM_DESC(dbg_level, "libiscsi debug level (default=0)"); 55 56 57 /* 58 * cxgbi device management 59 * maintains a list of the cxgbi devices 60 */ 61 static LIST_HEAD(cdev_list); 62 static DEFINE_MUTEX(cdev_mutex); 63 64 static LIST_HEAD(cdev_rcu_list); 65 static DEFINE_SPINLOCK(cdev_rcu_lock); 66 67 static inline void cxgbi_decode_sw_tag(u32 sw_tag, int *idx, int *age) 68 { 69 if (age) 70 *age = sw_tag & 0x7FFF; 71 if (idx) 72 *idx = (sw_tag >> 16) & 0x7FFF; 73 } 74 75 int cxgbi_device_portmap_create(struct cxgbi_device *cdev, unsigned int base, 76 unsigned int max_conn) 77 { 78 struct cxgbi_ports_map *pmap = &cdev->pmap; 79 80 pmap->port_csk = kvzalloc(array_size(max_conn, 81 sizeof(struct cxgbi_sock *)), 82 GFP_KERNEL | __GFP_NOWARN); 83 if (!pmap->port_csk) { 84 pr_warn("cdev 0x%p, portmap OOM %u.\n", cdev, max_conn); 85 return -ENOMEM; 86 } 87 88 pmap->max_connect = max_conn; 89 pmap->sport_base = base; 90 spin_lock_init(&pmap->lock); 91 return 0; 92 } 93 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_create); 94 95 void cxgbi_device_portmap_cleanup(struct cxgbi_device *cdev) 96 { 97 struct cxgbi_ports_map *pmap = &cdev->pmap; 98 struct cxgbi_sock *csk; 99 int i; 100 101 for (i = 0; i < pmap->max_connect; i++) { 102 if (pmap->port_csk[i]) { 103 csk = pmap->port_csk[i]; 104 pmap->port_csk[i] = NULL; 105 log_debug(1 << CXGBI_DBG_SOCK, 106 "csk 0x%p, cdev 0x%p, offload down.\n", 107 csk, cdev); 108 spin_lock_bh(&csk->lock); 109 cxgbi_sock_set_flag(csk, CTPF_OFFLOAD_DOWN); 110 cxgbi_sock_closed(csk); 111 spin_unlock_bh(&csk->lock); 112 cxgbi_sock_put(csk); 113 } 114 } 115 } 116 EXPORT_SYMBOL_GPL(cxgbi_device_portmap_cleanup); 117 118 static inline void cxgbi_device_destroy(struct cxgbi_device *cdev) 119 { 120 log_debug(1 << CXGBI_DBG_DEV, 121 "cdev 0x%p, p# %u.\n", cdev, cdev->nports); 122 cxgbi_hbas_remove(cdev); 123 cxgbi_device_portmap_cleanup(cdev); 124 if (cdev->cdev2ppm) 125 cxgbi_ppm_release(cdev->cdev2ppm(cdev)); 126 if (cdev->pmap.max_connect) 127 kvfree(cdev->pmap.port_csk); 128 kfree(cdev); 129 } 130 131 struct cxgbi_device *cxgbi_device_register(unsigned int extra, 132 unsigned int nports) 133 { 134 struct cxgbi_device *cdev; 135 136 cdev = kzalloc(sizeof(*cdev) + extra + nports * 137 (sizeof(struct cxgbi_hba *) + 138 sizeof(struct net_device *)), 139 GFP_KERNEL); 140 if (!cdev) { 141 pr_warn("nport %d, OOM.\n", nports); 142 return NULL; 143 } 144 cdev->ports = (struct net_device **)(cdev + 1); 145 cdev->hbas = (struct cxgbi_hba **)(((char*)cdev->ports) + nports * 146 sizeof(struct net_device *)); 147 if (extra) 148 cdev->dd_data = ((char *)cdev->hbas) + 149 nports * sizeof(struct cxgbi_hba *); 150 spin_lock_init(&cdev->pmap.lock); 151 152 mutex_lock(&cdev_mutex); 153 list_add_tail(&cdev->list_head, &cdev_list); 154 mutex_unlock(&cdev_mutex); 155 156 spin_lock(&cdev_rcu_lock); 157 list_add_tail_rcu(&cdev->rcu_node, &cdev_rcu_list); 158 spin_unlock(&cdev_rcu_lock); 159 160 log_debug(1 << CXGBI_DBG_DEV, 161 "cdev 0x%p, p# %u.\n", cdev, nports); 162 return cdev; 163 } 164 EXPORT_SYMBOL_GPL(cxgbi_device_register); 165 166 void cxgbi_device_unregister(struct cxgbi_device *cdev) 167 { 168 log_debug(1 << CXGBI_DBG_DEV, 169 "cdev 0x%p, p# %u,%s.\n", 170 cdev, cdev->nports, cdev->nports ? cdev->ports[0]->name : ""); 171 172 mutex_lock(&cdev_mutex); 173 list_del(&cdev->list_head); 174 mutex_unlock(&cdev_mutex); 175 176 spin_lock(&cdev_rcu_lock); 177 list_del_rcu(&cdev->rcu_node); 178 spin_unlock(&cdev_rcu_lock); 179 synchronize_rcu(); 180 181 cxgbi_device_destroy(cdev); 182 } 183 EXPORT_SYMBOL_GPL(cxgbi_device_unregister); 184 185 void cxgbi_device_unregister_all(unsigned int flag) 186 { 187 struct cxgbi_device *cdev, *tmp; 188 189 mutex_lock(&cdev_mutex); 190 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 191 if ((cdev->flags & flag) == flag) { 192 mutex_unlock(&cdev_mutex); 193 cxgbi_device_unregister(cdev); 194 mutex_lock(&cdev_mutex); 195 } 196 } 197 mutex_unlock(&cdev_mutex); 198 } 199 EXPORT_SYMBOL_GPL(cxgbi_device_unregister_all); 200 201 struct cxgbi_device *cxgbi_device_find_by_lldev(void *lldev) 202 { 203 struct cxgbi_device *cdev, *tmp; 204 205 mutex_lock(&cdev_mutex); 206 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 207 if (cdev->lldev == lldev) { 208 mutex_unlock(&cdev_mutex); 209 return cdev; 210 } 211 } 212 mutex_unlock(&cdev_mutex); 213 214 log_debug(1 << CXGBI_DBG_DEV, 215 "lldev 0x%p, NO match found.\n", lldev); 216 return NULL; 217 } 218 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_lldev); 219 220 struct cxgbi_device *cxgbi_device_find_by_netdev(struct net_device *ndev, 221 int *port) 222 { 223 struct net_device *vdev = NULL; 224 struct cxgbi_device *cdev, *tmp; 225 int i; 226 227 if (is_vlan_dev(ndev)) { 228 vdev = ndev; 229 ndev = vlan_dev_real_dev(ndev); 230 log_debug(1 << CXGBI_DBG_DEV, 231 "vlan dev %s -> %s.\n", vdev->name, ndev->name); 232 } 233 234 mutex_lock(&cdev_mutex); 235 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 236 for (i = 0; i < cdev->nports; i++) { 237 if (ndev == cdev->ports[i]) { 238 cdev->hbas[i]->vdev = vdev; 239 mutex_unlock(&cdev_mutex); 240 if (port) 241 *port = i; 242 return cdev; 243 } 244 } 245 } 246 mutex_unlock(&cdev_mutex); 247 log_debug(1 << CXGBI_DBG_DEV, 248 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name); 249 return NULL; 250 } 251 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev); 252 253 struct cxgbi_device *cxgbi_device_find_by_netdev_rcu(struct net_device *ndev, 254 int *port) 255 { 256 struct net_device *vdev = NULL; 257 struct cxgbi_device *cdev; 258 int i; 259 260 if (is_vlan_dev(ndev)) { 261 vdev = ndev; 262 ndev = vlan_dev_real_dev(ndev); 263 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name); 264 } 265 266 rcu_read_lock(); 267 list_for_each_entry_rcu(cdev, &cdev_rcu_list, rcu_node) { 268 for (i = 0; i < cdev->nports; i++) { 269 if (ndev == cdev->ports[i]) { 270 cdev->hbas[i]->vdev = vdev; 271 rcu_read_unlock(); 272 if (port) 273 *port = i; 274 return cdev; 275 } 276 } 277 } 278 rcu_read_unlock(); 279 280 log_debug(1 << CXGBI_DBG_DEV, 281 "ndev 0x%p, %s, NO match found.\n", ndev, ndev->name); 282 return NULL; 283 } 284 EXPORT_SYMBOL_GPL(cxgbi_device_find_by_netdev_rcu); 285 286 static struct cxgbi_device *cxgbi_device_find_by_mac(struct net_device *ndev, 287 int *port) 288 { 289 struct net_device *vdev = NULL; 290 struct cxgbi_device *cdev, *tmp; 291 int i; 292 293 if (is_vlan_dev(ndev)) { 294 vdev = ndev; 295 ndev = vlan_dev_real_dev(ndev); 296 pr_info("vlan dev %s -> %s.\n", vdev->name, ndev->name); 297 } 298 299 mutex_lock(&cdev_mutex); 300 list_for_each_entry_safe(cdev, tmp, &cdev_list, list_head) { 301 for (i = 0; i < cdev->nports; i++) { 302 if (!memcmp(ndev->dev_addr, cdev->ports[i]->dev_addr, 303 MAX_ADDR_LEN)) { 304 cdev->hbas[i]->vdev = vdev; 305 mutex_unlock(&cdev_mutex); 306 if (port) 307 *port = i; 308 return cdev; 309 } 310 } 311 } 312 mutex_unlock(&cdev_mutex); 313 log_debug(1 << CXGBI_DBG_DEV, 314 "ndev 0x%p, %s, NO match mac found.\n", 315 ndev, ndev->name); 316 return NULL; 317 } 318 319 void cxgbi_hbas_remove(struct cxgbi_device *cdev) 320 { 321 int i; 322 struct cxgbi_hba *chba; 323 324 log_debug(1 << CXGBI_DBG_DEV, 325 "cdev 0x%p, p#%u.\n", cdev, cdev->nports); 326 327 for (i = 0; i < cdev->nports; i++) { 328 chba = cdev->hbas[i]; 329 if (chba) { 330 cdev->hbas[i] = NULL; 331 iscsi_host_remove(chba->shost, false); 332 pci_dev_put(cdev->pdev); 333 iscsi_host_free(chba->shost); 334 } 335 } 336 } 337 EXPORT_SYMBOL_GPL(cxgbi_hbas_remove); 338 339 int cxgbi_hbas_add(struct cxgbi_device *cdev, u64 max_lun, 340 unsigned int max_conns, const struct scsi_host_template *sht, 341 struct scsi_transport_template *stt) 342 { 343 struct cxgbi_hba *chba; 344 struct Scsi_Host *shost; 345 int i, err; 346 347 log_debug(1 << CXGBI_DBG_DEV, "cdev 0x%p, p#%u.\n", cdev, cdev->nports); 348 349 for (i = 0; i < cdev->nports; i++) { 350 shost = iscsi_host_alloc(sht, sizeof(*chba), 1); 351 if (!shost) { 352 pr_info("0x%p, p%d, %s, host alloc failed.\n", 353 cdev, i, cdev->ports[i]->name); 354 err = -ENOMEM; 355 goto err_out; 356 } 357 358 shost->transportt = stt; 359 shost->max_lun = max_lun; 360 shost->max_id = max_conns - 1; 361 shost->max_channel = 0; 362 shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE; 363 364 chba = iscsi_host_priv(shost); 365 chba->cdev = cdev; 366 chba->ndev = cdev->ports[i]; 367 chba->shost = shost; 368 369 shost->can_queue = sht->can_queue - ISCSI_MGMT_CMDS_MAX; 370 371 log_debug(1 << CXGBI_DBG_DEV, 372 "cdev 0x%p, p#%d %s: chba 0x%p.\n", 373 cdev, i, cdev->ports[i]->name, chba); 374 375 pci_dev_get(cdev->pdev); 376 err = iscsi_host_add(shost, &cdev->pdev->dev); 377 if (err) { 378 pr_info("cdev 0x%p, p#%d %s, host add failed.\n", 379 cdev, i, cdev->ports[i]->name); 380 pci_dev_put(cdev->pdev); 381 scsi_host_put(shost); 382 goto err_out; 383 } 384 385 cdev->hbas[i] = chba; 386 } 387 388 return 0; 389 390 err_out: 391 cxgbi_hbas_remove(cdev); 392 return err; 393 } 394 EXPORT_SYMBOL_GPL(cxgbi_hbas_add); 395 396 /* 397 * iSCSI offload 398 * 399 * - source port management 400 * To find a free source port in the port allocation map we use a very simple 401 * rotor scheme to look for the next free port. 402 * 403 * If a source port has been specified make sure that it doesn't collide with 404 * our normal source port allocation map. If it's outside the range of our 405 * allocation/deallocation scheme just let them use it. 406 * 407 * If the source port is outside our allocation range, the caller is 408 * responsible for keeping track of their port usage. 409 */ 410 411 static struct cxgbi_sock *find_sock_on_port(struct cxgbi_device *cdev, 412 unsigned char port_id) 413 { 414 struct cxgbi_ports_map *pmap = &cdev->pmap; 415 unsigned int i; 416 unsigned int used; 417 418 if (!pmap->max_connect || !pmap->used) 419 return NULL; 420 421 spin_lock_bh(&pmap->lock); 422 used = pmap->used; 423 for (i = 0; used && i < pmap->max_connect; i++) { 424 struct cxgbi_sock *csk = pmap->port_csk[i]; 425 426 if (csk) { 427 if (csk->port_id == port_id) { 428 spin_unlock_bh(&pmap->lock); 429 return csk; 430 } 431 used--; 432 } 433 } 434 spin_unlock_bh(&pmap->lock); 435 436 return NULL; 437 } 438 439 static int sock_get_port(struct cxgbi_sock *csk) 440 { 441 struct cxgbi_device *cdev = csk->cdev; 442 struct cxgbi_ports_map *pmap = &cdev->pmap; 443 unsigned int start; 444 int idx; 445 __be16 *port; 446 447 if (!pmap->max_connect) { 448 pr_err("cdev 0x%p, p#%u %s, NO port map.\n", 449 cdev, csk->port_id, cdev->ports[csk->port_id]->name); 450 return -EADDRNOTAVAIL; 451 } 452 453 if (csk->csk_family == AF_INET) 454 port = &csk->saddr.sin_port; 455 else /* ipv6 */ 456 port = &csk->saddr6.sin6_port; 457 458 if (*port) { 459 pr_err("source port NON-ZERO %u.\n", 460 ntohs(*port)); 461 return -EADDRINUSE; 462 } 463 464 spin_lock_bh(&pmap->lock); 465 if (pmap->used >= pmap->max_connect) { 466 spin_unlock_bh(&pmap->lock); 467 pr_info("cdev 0x%p, p#%u %s, ALL ports used.\n", 468 cdev, csk->port_id, cdev->ports[csk->port_id]->name); 469 return -EADDRNOTAVAIL; 470 } 471 472 start = idx = pmap->next; 473 do { 474 if (++idx >= pmap->max_connect) 475 idx = 0; 476 if (!pmap->port_csk[idx]) { 477 pmap->used++; 478 *port = htons(pmap->sport_base + idx); 479 pmap->next = idx; 480 pmap->port_csk[idx] = csk; 481 spin_unlock_bh(&pmap->lock); 482 cxgbi_sock_get(csk); 483 log_debug(1 << CXGBI_DBG_SOCK, 484 "cdev 0x%p, p#%u %s, p %u, %u.\n", 485 cdev, csk->port_id, 486 cdev->ports[csk->port_id]->name, 487 pmap->sport_base + idx, pmap->next); 488 return 0; 489 } 490 } while (idx != start); 491 spin_unlock_bh(&pmap->lock); 492 493 /* should not happen */ 494 pr_warn("cdev 0x%p, p#%u %s, next %u?\n", 495 cdev, csk->port_id, cdev->ports[csk->port_id]->name, 496 pmap->next); 497 return -EADDRNOTAVAIL; 498 } 499 500 static void sock_put_port(struct cxgbi_sock *csk) 501 { 502 struct cxgbi_device *cdev = csk->cdev; 503 struct cxgbi_ports_map *pmap = &cdev->pmap; 504 __be16 *port; 505 506 if (csk->csk_family == AF_INET) 507 port = &csk->saddr.sin_port; 508 else /* ipv6 */ 509 port = &csk->saddr6.sin6_port; 510 511 if (*port) { 512 int idx = ntohs(*port) - pmap->sport_base; 513 514 *port = 0; 515 if (idx < 0 || idx >= pmap->max_connect) { 516 pr_err("cdev 0x%p, p#%u %s, port %u OOR.\n", 517 cdev, csk->port_id, 518 cdev->ports[csk->port_id]->name, 519 ntohs(*port)); 520 return; 521 } 522 523 spin_lock_bh(&pmap->lock); 524 pmap->port_csk[idx] = NULL; 525 pmap->used--; 526 spin_unlock_bh(&pmap->lock); 527 528 log_debug(1 << CXGBI_DBG_SOCK, 529 "cdev 0x%p, p#%u %s, release %u.\n", 530 cdev, csk->port_id, cdev->ports[csk->port_id]->name, 531 pmap->sport_base + idx); 532 533 cxgbi_sock_put(csk); 534 } 535 } 536 537 /* 538 * iscsi tcp connection 539 */ 540 void cxgbi_sock_free_cpl_skbs(struct cxgbi_sock *csk) 541 { 542 if (csk->cpl_close) { 543 kfree_skb(csk->cpl_close); 544 csk->cpl_close = NULL; 545 } 546 if (csk->cpl_abort_req) { 547 kfree_skb(csk->cpl_abort_req); 548 csk->cpl_abort_req = NULL; 549 } 550 if (csk->cpl_abort_rpl) { 551 kfree_skb(csk->cpl_abort_rpl); 552 csk->cpl_abort_rpl = NULL; 553 } 554 } 555 EXPORT_SYMBOL_GPL(cxgbi_sock_free_cpl_skbs); 556 557 static struct cxgbi_sock *cxgbi_sock_create(struct cxgbi_device *cdev) 558 { 559 struct cxgbi_sock *csk = kzalloc(sizeof(*csk), GFP_NOIO); 560 561 if (!csk) { 562 pr_info("alloc csk %zu failed.\n", sizeof(*csk)); 563 return NULL; 564 } 565 566 if (cdev->csk_alloc_cpls(csk) < 0) { 567 pr_info("csk 0x%p, alloc cpls failed.\n", csk); 568 kfree(csk); 569 return NULL; 570 } 571 572 spin_lock_init(&csk->lock); 573 kref_init(&csk->refcnt); 574 skb_queue_head_init(&csk->receive_queue); 575 skb_queue_head_init(&csk->write_queue); 576 timer_setup(&csk->retry_timer, NULL, 0); 577 init_completion(&csk->cmpl); 578 rwlock_init(&csk->callback_lock); 579 csk->cdev = cdev; 580 csk->flags = 0; 581 cxgbi_sock_set_state(csk, CTP_CLOSED); 582 583 log_debug(1 << CXGBI_DBG_SOCK, "cdev 0x%p, new csk 0x%p.\n", cdev, csk); 584 585 return csk; 586 } 587 588 static struct rtable *find_route_ipv4(struct flowi4 *fl4, 589 __be32 saddr, __be32 daddr, 590 __be16 sport, __be16 dport, u8 tos, 591 int ifindex) 592 { 593 struct rtable *rt; 594 595 rt = ip_route_output_ports(&init_net, fl4, NULL, daddr, saddr, 596 dport, sport, IPPROTO_TCP, tos, ifindex); 597 if (IS_ERR(rt)) 598 return NULL; 599 600 return rt; 601 } 602 603 static struct cxgbi_sock * 604 cxgbi_check_route(struct sockaddr *dst_addr, int ifindex) 605 { 606 struct sockaddr_in *daddr = (struct sockaddr_in *)dst_addr; 607 struct dst_entry *dst; 608 struct net_device *ndev; 609 struct cxgbi_device *cdev; 610 struct rtable *rt = NULL; 611 struct neighbour *n; 612 struct flowi4 fl4; 613 struct cxgbi_sock *csk = NULL; 614 unsigned int mtu = 0; 615 int port = 0xFFFF; 616 int err = 0; 617 618 rt = find_route_ipv4(&fl4, 0, daddr->sin_addr.s_addr, 0, 619 daddr->sin_port, 0, ifindex); 620 if (!rt) { 621 pr_info("no route to ipv4 0x%x, port %u.\n", 622 be32_to_cpu(daddr->sin_addr.s_addr), 623 be16_to_cpu(daddr->sin_port)); 624 err = -ENETUNREACH; 625 goto err_out; 626 } 627 dst = &rt->dst; 628 n = dst_neigh_lookup(dst, &daddr->sin_addr.s_addr); 629 if (!n) { 630 err = -ENODEV; 631 goto rel_rt; 632 } 633 ndev = n->dev; 634 635 if (rt->rt_flags & (RTCF_MULTICAST | RTCF_BROADCAST)) { 636 pr_info("multi-cast route %pI4, port %u, dev %s.\n", 637 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port), 638 ndev->name); 639 err = -ENETUNREACH; 640 goto rel_neigh; 641 } 642 643 if (ndev->flags & IFF_LOOPBACK) { 644 ndev = ip_dev_find(&init_net, daddr->sin_addr.s_addr); 645 if (!ndev) { 646 err = -ENETUNREACH; 647 goto rel_neigh; 648 } 649 mtu = ndev->mtu; 650 pr_info("rt dev %s, loopback -> %s, mtu %u.\n", 651 n->dev->name, ndev->name, mtu); 652 } 653 654 if (!(ndev->flags & IFF_UP) || !netif_carrier_ok(ndev)) { 655 pr_info("%s interface not up.\n", ndev->name); 656 err = -ENETDOWN; 657 goto rel_neigh; 658 } 659 660 cdev = cxgbi_device_find_by_netdev(ndev, &port); 661 if (!cdev) 662 cdev = cxgbi_device_find_by_mac(ndev, &port); 663 if (!cdev) { 664 pr_info("dst %pI4, %s, NOT cxgbi device.\n", 665 &daddr->sin_addr.s_addr, ndev->name); 666 err = -ENETUNREACH; 667 goto rel_neigh; 668 } 669 log_debug(1 << CXGBI_DBG_SOCK, 670 "route to %pI4 :%u, ndev p#%d,%s, cdev 0x%p.\n", 671 &daddr->sin_addr.s_addr, ntohs(daddr->sin_port), 672 port, ndev->name, cdev); 673 674 csk = cxgbi_sock_create(cdev); 675 if (!csk) { 676 err = -ENOMEM; 677 goto rel_neigh; 678 } 679 csk->cdev = cdev; 680 csk->port_id = port; 681 csk->mtu = mtu; 682 csk->dst = dst; 683 684 csk->csk_family = AF_INET; 685 csk->daddr.sin_addr.s_addr = daddr->sin_addr.s_addr; 686 csk->daddr.sin_port = daddr->sin_port; 687 csk->daddr.sin_family = daddr->sin_family; 688 csk->saddr.sin_family = daddr->sin_family; 689 csk->saddr.sin_addr.s_addr = fl4.saddr; 690 neigh_release(n); 691 692 return csk; 693 694 rel_neigh: 695 neigh_release(n); 696 697 rel_rt: 698 ip_rt_put(rt); 699 err_out: 700 return ERR_PTR(err); 701 } 702 703 #if IS_ENABLED(CONFIG_IPV6) 704 static struct rt6_info *find_route_ipv6(const struct in6_addr *saddr, 705 const struct in6_addr *daddr, 706 int ifindex) 707 { 708 struct flowi6 fl; 709 710 memset(&fl, 0, sizeof(fl)); 711 fl.flowi6_oif = ifindex; 712 if (saddr) 713 memcpy(&fl.saddr, saddr, sizeof(struct in6_addr)); 714 if (daddr) 715 memcpy(&fl.daddr, daddr, sizeof(struct in6_addr)); 716 return (struct rt6_info *)ip6_route_output(&init_net, NULL, &fl); 717 } 718 719 static struct cxgbi_sock * 720 cxgbi_check_route6(struct sockaddr *dst_addr, int ifindex) 721 { 722 struct sockaddr_in6 *daddr6 = (struct sockaddr_in6 *)dst_addr; 723 struct dst_entry *dst; 724 struct net_device *ndev; 725 struct cxgbi_device *cdev; 726 struct rt6_info *rt = NULL; 727 struct neighbour *n; 728 struct in6_addr pref_saddr; 729 struct cxgbi_sock *csk = NULL; 730 unsigned int mtu = 0; 731 int port = 0xFFFF; 732 int err = 0; 733 734 rt = find_route_ipv6(NULL, &daddr6->sin6_addr, ifindex); 735 736 if (!rt) { 737 pr_info("no route to ipv6 %pI6 port %u\n", 738 daddr6->sin6_addr.s6_addr, 739 be16_to_cpu(daddr6->sin6_port)); 740 err = -ENETUNREACH; 741 goto err_out; 742 } 743 744 dst = &rt->dst; 745 746 n = dst_neigh_lookup(dst, &daddr6->sin6_addr); 747 748 if (!n) { 749 pr_info("%pI6, port %u, dst no neighbour.\n", 750 daddr6->sin6_addr.s6_addr, 751 be16_to_cpu(daddr6->sin6_port)); 752 err = -ENETUNREACH; 753 goto rel_rt; 754 } 755 ndev = n->dev; 756 757 if (!(ndev->flags & IFF_UP) || !netif_carrier_ok(ndev)) { 758 pr_info("%s interface not up.\n", ndev->name); 759 err = -ENETDOWN; 760 goto rel_rt; 761 } 762 763 if (ipv6_addr_is_multicast(&daddr6->sin6_addr)) { 764 pr_info("multi-cast route %pI6 port %u, dev %s.\n", 765 daddr6->sin6_addr.s6_addr, 766 ntohs(daddr6->sin6_port), ndev->name); 767 err = -ENETUNREACH; 768 goto rel_rt; 769 } 770 771 cdev = cxgbi_device_find_by_netdev(ndev, &port); 772 if (!cdev) 773 cdev = cxgbi_device_find_by_mac(ndev, &port); 774 if (!cdev) { 775 pr_info("dst %pI6 %s, NOT cxgbi device.\n", 776 daddr6->sin6_addr.s6_addr, ndev->name); 777 err = -ENETUNREACH; 778 goto rel_rt; 779 } 780 log_debug(1 << CXGBI_DBG_SOCK, 781 "route to %pI6 :%u, ndev p#%d,%s, cdev 0x%p.\n", 782 daddr6->sin6_addr.s6_addr, ntohs(daddr6->sin6_port), port, 783 ndev->name, cdev); 784 785 csk = cxgbi_sock_create(cdev); 786 if (!csk) { 787 err = -ENOMEM; 788 goto rel_rt; 789 } 790 csk->cdev = cdev; 791 csk->port_id = port; 792 csk->mtu = mtu; 793 csk->dst = dst; 794 795 rt6_get_prefsrc(rt, &pref_saddr); 796 if (ipv6_addr_any(&pref_saddr)) { 797 struct inet6_dev *idev = ip6_dst_idev((struct dst_entry *)rt); 798 799 err = ipv6_dev_get_saddr(&init_net, idev ? idev->dev : NULL, 800 &daddr6->sin6_addr, 0, &pref_saddr); 801 if (err) { 802 pr_info("failed to get source address to reach %pI6\n", 803 &daddr6->sin6_addr); 804 goto rel_rt; 805 } 806 } 807 808 csk->csk_family = AF_INET6; 809 csk->daddr6.sin6_addr = daddr6->sin6_addr; 810 csk->daddr6.sin6_port = daddr6->sin6_port; 811 csk->daddr6.sin6_family = daddr6->sin6_family; 812 csk->saddr6.sin6_family = daddr6->sin6_family; 813 csk->saddr6.sin6_addr = pref_saddr; 814 815 neigh_release(n); 816 return csk; 817 818 rel_rt: 819 if (n) 820 neigh_release(n); 821 822 ip6_rt_put(rt); 823 if (csk) 824 cxgbi_sock_closed(csk); 825 err_out: 826 return ERR_PTR(err); 827 } 828 #endif /* IS_ENABLED(CONFIG_IPV6) */ 829 830 void cxgbi_sock_established(struct cxgbi_sock *csk, unsigned int snd_isn, 831 unsigned int opt) 832 { 833 csk->write_seq = csk->snd_nxt = csk->snd_una = snd_isn; 834 dst_confirm(csk->dst); 835 smp_mb(); 836 cxgbi_sock_set_state(csk, CTP_ESTABLISHED); 837 } 838 EXPORT_SYMBOL_GPL(cxgbi_sock_established); 839 840 static void cxgbi_inform_iscsi_conn_closing(struct cxgbi_sock *csk) 841 { 842 log_debug(1 << CXGBI_DBG_SOCK, 843 "csk 0x%p, state %u, flags 0x%lx, conn 0x%p.\n", 844 csk, csk->state, csk->flags, csk->user_data); 845 846 if (csk->state != CTP_ESTABLISHED) { 847 read_lock_bh(&csk->callback_lock); 848 if (csk->user_data) 849 iscsi_conn_failure(csk->user_data, 850 ISCSI_ERR_TCP_CONN_CLOSE); 851 read_unlock_bh(&csk->callback_lock); 852 } 853 } 854 855 void cxgbi_sock_closed(struct cxgbi_sock *csk) 856 { 857 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 858 csk, (csk)->state, (csk)->flags, (csk)->tid); 859 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED); 860 if (csk->state == CTP_ACTIVE_OPEN || csk->state == CTP_CLOSED) 861 return; 862 if (csk->saddr.sin_port) 863 sock_put_port(csk); 864 if (csk->dst) 865 dst_release(csk->dst); 866 csk->cdev->csk_release_offload_resources(csk); 867 cxgbi_sock_set_state(csk, CTP_CLOSED); 868 cxgbi_inform_iscsi_conn_closing(csk); 869 cxgbi_sock_put(csk); 870 } 871 EXPORT_SYMBOL_GPL(cxgbi_sock_closed); 872 873 static void need_active_close(struct cxgbi_sock *csk) 874 { 875 int data_lost; 876 int close_req = 0; 877 878 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 879 csk, (csk)->state, (csk)->flags, (csk)->tid); 880 spin_lock_bh(&csk->lock); 881 if (csk->dst) 882 dst_confirm(csk->dst); 883 data_lost = skb_queue_len(&csk->receive_queue); 884 __skb_queue_purge(&csk->receive_queue); 885 886 if (csk->state == CTP_ACTIVE_OPEN) 887 cxgbi_sock_set_flag(csk, CTPF_ACTIVE_CLOSE_NEEDED); 888 else if (csk->state == CTP_ESTABLISHED) { 889 close_req = 1; 890 cxgbi_sock_set_state(csk, CTP_ACTIVE_CLOSE); 891 } else if (csk->state == CTP_PASSIVE_CLOSE) { 892 close_req = 1; 893 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2); 894 } 895 896 if (close_req) { 897 if (!cxgbi_sock_flag(csk, CTPF_LOGOUT_RSP_RCVD) || 898 data_lost) 899 csk->cdev->csk_send_abort_req(csk); 900 else 901 csk->cdev->csk_send_close_req(csk); 902 } 903 904 spin_unlock_bh(&csk->lock); 905 } 906 907 void cxgbi_sock_fail_act_open(struct cxgbi_sock *csk, int errno) 908 { 909 pr_info("csk 0x%p,%u,%lx, %pI4:%u-%pI4:%u, err %d.\n", 910 csk, csk->state, csk->flags, 911 &csk->saddr.sin_addr.s_addr, csk->saddr.sin_port, 912 &csk->daddr.sin_addr.s_addr, csk->daddr.sin_port, 913 errno); 914 915 cxgbi_sock_set_state(csk, CTP_CONNECTING); 916 csk->err = errno; 917 cxgbi_sock_closed(csk); 918 } 919 EXPORT_SYMBOL_GPL(cxgbi_sock_fail_act_open); 920 921 void cxgbi_sock_act_open_req_arp_failure(void *handle, struct sk_buff *skb) 922 { 923 struct cxgbi_sock *csk = (struct cxgbi_sock *)skb->sk; 924 struct module *owner = csk->cdev->owner; 925 926 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 927 csk, (csk)->state, (csk)->flags, (csk)->tid); 928 cxgbi_sock_get(csk); 929 spin_lock_bh(&csk->lock); 930 if (csk->state == CTP_ACTIVE_OPEN) 931 cxgbi_sock_fail_act_open(csk, -EHOSTUNREACH); 932 spin_unlock_bh(&csk->lock); 933 cxgbi_sock_put(csk); 934 __kfree_skb(skb); 935 936 module_put(owner); 937 } 938 EXPORT_SYMBOL_GPL(cxgbi_sock_act_open_req_arp_failure); 939 940 void cxgbi_sock_rcv_abort_rpl(struct cxgbi_sock *csk) 941 { 942 cxgbi_sock_get(csk); 943 spin_lock_bh(&csk->lock); 944 945 cxgbi_sock_set_flag(csk, CTPF_ABORT_RPL_RCVD); 946 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) { 947 cxgbi_sock_clear_flag(csk, CTPF_ABORT_RPL_PENDING); 948 if (cxgbi_sock_flag(csk, CTPF_ABORT_REQ_RCVD)) 949 pr_err("csk 0x%p,%u,0x%lx,%u,ABT_RPL_RSS.\n", 950 csk, csk->state, csk->flags, csk->tid); 951 cxgbi_sock_closed(csk); 952 } 953 954 spin_unlock_bh(&csk->lock); 955 cxgbi_sock_put(csk); 956 } 957 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_abort_rpl); 958 959 void cxgbi_sock_rcv_peer_close(struct cxgbi_sock *csk) 960 { 961 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 962 csk, (csk)->state, (csk)->flags, (csk)->tid); 963 cxgbi_sock_get(csk); 964 spin_lock_bh(&csk->lock); 965 966 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) 967 goto done; 968 969 switch (csk->state) { 970 case CTP_ESTABLISHED: 971 cxgbi_sock_set_state(csk, CTP_PASSIVE_CLOSE); 972 break; 973 case CTP_ACTIVE_CLOSE: 974 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_2); 975 break; 976 case CTP_CLOSE_WAIT_1: 977 cxgbi_sock_closed(csk); 978 break; 979 case CTP_ABORTING: 980 break; 981 default: 982 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n", 983 csk, csk->state, csk->flags, csk->tid); 984 } 985 cxgbi_inform_iscsi_conn_closing(csk); 986 done: 987 spin_unlock_bh(&csk->lock); 988 cxgbi_sock_put(csk); 989 } 990 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_peer_close); 991 992 void cxgbi_sock_rcv_close_conn_rpl(struct cxgbi_sock *csk, u32 snd_nxt) 993 { 994 log_debug(1 << CXGBI_DBG_SOCK, "csk 0x%p,%u,0x%lx,%u.\n", 995 csk, (csk)->state, (csk)->flags, (csk)->tid); 996 cxgbi_sock_get(csk); 997 spin_lock_bh(&csk->lock); 998 999 csk->snd_una = snd_nxt - 1; 1000 if (cxgbi_sock_flag(csk, CTPF_ABORT_RPL_PENDING)) 1001 goto done; 1002 1003 switch (csk->state) { 1004 case CTP_ACTIVE_CLOSE: 1005 cxgbi_sock_set_state(csk, CTP_CLOSE_WAIT_1); 1006 break; 1007 case CTP_CLOSE_WAIT_1: 1008 case CTP_CLOSE_WAIT_2: 1009 cxgbi_sock_closed(csk); 1010 break; 1011 case CTP_ABORTING: 1012 break; 1013 default: 1014 pr_err("csk 0x%p,%u,0x%lx,%u, bad state.\n", 1015 csk, csk->state, csk->flags, csk->tid); 1016 } 1017 done: 1018 spin_unlock_bh(&csk->lock); 1019 cxgbi_sock_put(csk); 1020 } 1021 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_close_conn_rpl); 1022 1023 void cxgbi_sock_rcv_wr_ack(struct cxgbi_sock *csk, unsigned int credits, 1024 unsigned int snd_una, int seq_chk) 1025 { 1026 log_debug(1 << CXGBI_DBG_TOE | 1 << CXGBI_DBG_SOCK, 1027 "csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, snd_una %u,%d.\n", 1028 csk, csk->state, csk->flags, csk->tid, credits, 1029 csk->wr_cred, csk->wr_una_cred, snd_una, seq_chk); 1030 1031 spin_lock_bh(&csk->lock); 1032 1033 csk->wr_cred += credits; 1034 if (csk->wr_una_cred > csk->wr_max_cred - csk->wr_cred) 1035 csk->wr_una_cred = csk->wr_max_cred - csk->wr_cred; 1036 1037 while (credits) { 1038 struct sk_buff *p = cxgbi_sock_peek_wr(csk); 1039 1040 if (unlikely(!p)) { 1041 pr_err("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, empty.\n", 1042 csk, csk->state, csk->flags, csk->tid, credits, 1043 csk->wr_cred, csk->wr_una_cred); 1044 break; 1045 } 1046 1047 if (unlikely(credits < p->csum)) { 1048 pr_warn("csk 0x%p,%u,0x%lx,%u, cr %u,%u+%u, < %u.\n", 1049 csk, csk->state, csk->flags, csk->tid, 1050 credits, csk->wr_cred, csk->wr_una_cred, 1051 p->csum); 1052 p->csum -= credits; 1053 break; 1054 } else { 1055 cxgbi_sock_dequeue_wr(csk); 1056 credits -= p->csum; 1057 kfree_skb(p); 1058 } 1059 } 1060 1061 cxgbi_sock_check_wr_invariants(csk); 1062 1063 if (seq_chk) { 1064 if (unlikely(before(snd_una, csk->snd_una))) { 1065 pr_warn("csk 0x%p,%u,0x%lx,%u, snd_una %u/%u.", 1066 csk, csk->state, csk->flags, csk->tid, snd_una, 1067 csk->snd_una); 1068 goto done; 1069 } 1070 1071 if (csk->snd_una != snd_una) { 1072 csk->snd_una = snd_una; 1073 dst_confirm(csk->dst); 1074 } 1075 } 1076 1077 if (skb_queue_len(&csk->write_queue)) { 1078 if (csk->cdev->csk_push_tx_frames(csk, 0)) 1079 cxgbi_conn_tx_open(csk); 1080 } else 1081 cxgbi_conn_tx_open(csk); 1082 done: 1083 spin_unlock_bh(&csk->lock); 1084 } 1085 EXPORT_SYMBOL_GPL(cxgbi_sock_rcv_wr_ack); 1086 1087 static unsigned int cxgbi_sock_find_best_mtu(struct cxgbi_sock *csk, 1088 unsigned short mtu) 1089 { 1090 int i = 0; 1091 1092 while (i < csk->cdev->nmtus - 1 && csk->cdev->mtus[i + 1] <= mtu) 1093 ++i; 1094 1095 return i; 1096 } 1097 1098 unsigned int cxgbi_sock_select_mss(struct cxgbi_sock *csk, unsigned int pmtu) 1099 { 1100 unsigned int idx; 1101 struct dst_entry *dst = csk->dst; 1102 1103 csk->advmss = dst_metric_advmss(dst); 1104 1105 if (csk->advmss > pmtu - 40) 1106 csk->advmss = pmtu - 40; 1107 if (csk->advmss < csk->cdev->mtus[0] - 40) 1108 csk->advmss = csk->cdev->mtus[0] - 40; 1109 idx = cxgbi_sock_find_best_mtu(csk, csk->advmss + 40); 1110 1111 return idx; 1112 } 1113 EXPORT_SYMBOL_GPL(cxgbi_sock_select_mss); 1114 1115 void cxgbi_sock_skb_entail(struct cxgbi_sock *csk, struct sk_buff *skb) 1116 { 1117 cxgbi_skcb_tcp_seq(skb) = csk->write_seq; 1118 __skb_queue_tail(&csk->write_queue, skb); 1119 } 1120 EXPORT_SYMBOL_GPL(cxgbi_sock_skb_entail); 1121 1122 void cxgbi_sock_purge_wr_queue(struct cxgbi_sock *csk) 1123 { 1124 struct sk_buff *skb; 1125 1126 while ((skb = cxgbi_sock_dequeue_wr(csk)) != NULL) 1127 kfree_skb(skb); 1128 } 1129 EXPORT_SYMBOL_GPL(cxgbi_sock_purge_wr_queue); 1130 1131 void cxgbi_sock_check_wr_invariants(const struct cxgbi_sock *csk) 1132 { 1133 int pending = cxgbi_sock_count_pending_wrs(csk); 1134 1135 if (unlikely(csk->wr_cred + pending != csk->wr_max_cred)) 1136 pr_err("csk 0x%p, tid %u, credit %u + %u != %u.\n", 1137 csk, csk->tid, csk->wr_cred, pending, csk->wr_max_cred); 1138 } 1139 EXPORT_SYMBOL_GPL(cxgbi_sock_check_wr_invariants); 1140 1141 static inline void 1142 scmd_get_params(struct scsi_cmnd *sc, struct scatterlist **sgl, 1143 unsigned int *sgcnt, unsigned int *dlen, 1144 unsigned int prot) 1145 { 1146 struct scsi_data_buffer *sdb = prot ? scsi_prot(sc) : &sc->sdb; 1147 1148 *sgl = sdb->table.sgl; 1149 *sgcnt = sdb->table.nents; 1150 *dlen = sdb->length; 1151 /* Caution: for protection sdb, sdb->length is invalid */ 1152 } 1153 1154 void cxgbi_ddp_set_one_ppod(struct cxgbi_pagepod *ppod, 1155 struct cxgbi_task_tag_info *ttinfo, 1156 struct scatterlist **sg_pp, unsigned int *sg_off) 1157 { 1158 struct scatterlist *sg = sg_pp ? *sg_pp : NULL; 1159 unsigned int offset = sg_off ? *sg_off : 0; 1160 dma_addr_t addr = 0UL; 1161 unsigned int len = 0; 1162 int i; 1163 1164 memcpy(ppod, &ttinfo->hdr, sizeof(struct cxgbi_pagepod_hdr)); 1165 1166 if (sg) { 1167 addr = sg_dma_address(sg); 1168 len = sg_dma_len(sg); 1169 } 1170 1171 for (i = 0; i < PPOD_PAGES_MAX; i++) { 1172 if (sg) { 1173 ppod->addr[i] = cpu_to_be64(addr + offset); 1174 offset += PAGE_SIZE; 1175 if (offset == (len + sg->offset)) { 1176 offset = 0; 1177 sg = sg_next(sg); 1178 if (sg) { 1179 addr = sg_dma_address(sg); 1180 len = sg_dma_len(sg); 1181 } 1182 } 1183 } else { 1184 ppod->addr[i] = 0ULL; 1185 } 1186 } 1187 1188 /* 1189 * the fifth address needs to be repeated in the next ppod, so do 1190 * not move sg 1191 */ 1192 if (sg_pp) { 1193 *sg_pp = sg; 1194 *sg_off = offset; 1195 } 1196 1197 if (offset == len) { 1198 offset = 0; 1199 sg = sg_next(sg); 1200 if (sg) { 1201 addr = sg_dma_address(sg); 1202 len = sg_dma_len(sg); 1203 } 1204 } 1205 ppod->addr[i] = sg ? cpu_to_be64(addr + offset) : 0ULL; 1206 } 1207 EXPORT_SYMBOL_GPL(cxgbi_ddp_set_one_ppod); 1208 1209 /* 1210 * APIs interacting with open-iscsi libraries 1211 */ 1212 1213 int cxgbi_ddp_ppm_setup(void **ppm_pp, struct cxgbi_device *cdev, 1214 struct cxgbi_tag_format *tformat, 1215 unsigned int iscsi_size, unsigned int llimit, 1216 unsigned int start, unsigned int rsvd_factor, 1217 unsigned int edram_start, unsigned int edram_size) 1218 { 1219 int err = cxgbi_ppm_init(ppm_pp, cdev->ports[0], cdev->pdev, 1220 cdev->lldev, tformat, iscsi_size, llimit, start, 1221 rsvd_factor, edram_start, edram_size); 1222 1223 if (err >= 0) { 1224 struct cxgbi_ppm *ppm = (struct cxgbi_ppm *)(*ppm_pp); 1225 1226 if (ppm->ppmax < 1024 || 1227 ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX) 1228 cdev->flags |= CXGBI_FLAG_DDP_OFF; 1229 err = 0; 1230 } else { 1231 cdev->flags |= CXGBI_FLAG_DDP_OFF; 1232 } 1233 1234 return err; 1235 } 1236 EXPORT_SYMBOL_GPL(cxgbi_ddp_ppm_setup); 1237 1238 static int cxgbi_ddp_sgl_check(struct scatterlist *sgl, int nents) 1239 { 1240 int i; 1241 int last_sgidx = nents - 1; 1242 struct scatterlist *sg = sgl; 1243 1244 for (i = 0; i < nents; i++, sg = sg_next(sg)) { 1245 unsigned int len = sg->length + sg->offset; 1246 1247 if ((sg->offset & 0x3) || (i && sg->offset) || 1248 ((i != last_sgidx) && len != PAGE_SIZE)) { 1249 log_debug(1 << CXGBI_DBG_DDP, 1250 "sg %u/%u, %u,%u, not aligned.\n", 1251 i, nents, sg->offset, sg->length); 1252 goto err_out; 1253 } 1254 } 1255 return 0; 1256 err_out: 1257 return -EINVAL; 1258 } 1259 1260 static int cxgbi_ddp_reserve(struct cxgbi_conn *cconn, 1261 struct cxgbi_task_data *tdata, u32 sw_tag, 1262 unsigned int xferlen) 1263 { 1264 struct cxgbi_sock *csk = cconn->cep->csk; 1265 struct cxgbi_device *cdev = csk->cdev; 1266 struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev); 1267 struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; 1268 struct scatterlist *sgl = ttinfo->sgl; 1269 unsigned int sgcnt = ttinfo->nents; 1270 unsigned int sg_offset = sgl->offset; 1271 int err; 1272 1273 if (cdev->flags & CXGBI_FLAG_DDP_OFF) { 1274 log_debug(1 << CXGBI_DBG_DDP, 1275 "cdev 0x%p DDP off.\n", cdev); 1276 return -EINVAL; 1277 } 1278 1279 if (!ppm || xferlen < DDP_THRESHOLD || !sgcnt || 1280 ppm->tformat.pgsz_idx_dflt >= DDP_PGIDX_MAX) { 1281 log_debug(1 << CXGBI_DBG_DDP, 1282 "ppm 0x%p, pgidx %u, xfer %u, sgcnt %u, NO ddp.\n", 1283 ppm, ppm ? ppm->tformat.pgsz_idx_dflt : DDP_PGIDX_MAX, 1284 xferlen, ttinfo->nents); 1285 return -EINVAL; 1286 } 1287 1288 /* make sure the buffer is suitable for ddp */ 1289 if (cxgbi_ddp_sgl_check(sgl, sgcnt) < 0) 1290 return -EINVAL; 1291 1292 ttinfo->nr_pages = (xferlen + sgl->offset + (1 << PAGE_SHIFT) - 1) >> 1293 PAGE_SHIFT; 1294 1295 /* 1296 * the ddp tag will be used for the itt in the outgoing pdu, 1297 * the itt genrated by libiscsi is saved in the ppm and can be 1298 * retrieved via the ddp tag 1299 */ 1300 err = cxgbi_ppm_ppods_reserve(ppm, ttinfo->nr_pages, 0, &ttinfo->idx, 1301 &ttinfo->tag, (unsigned long)sw_tag); 1302 if (err < 0) { 1303 cconn->ddp_full++; 1304 return err; 1305 } 1306 ttinfo->npods = err; 1307 1308 /* setup dma from scsi command sgl */ 1309 sgl->offset = 0; 1310 err = dma_map_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); 1311 sgl->offset = sg_offset; 1312 if (err == 0) { 1313 pr_info("%s: 0x%x, xfer %u, sgl %u dma mapping err.\n", 1314 __func__, sw_tag, xferlen, sgcnt); 1315 goto rel_ppods; 1316 } 1317 if (err != ttinfo->nr_pages) { 1318 log_debug(1 << CXGBI_DBG_DDP, 1319 "%s: sw tag 0x%x, xfer %u, sgl %u, dma count %d.\n", 1320 __func__, sw_tag, xferlen, sgcnt, err); 1321 } 1322 1323 ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_MAPPED; 1324 ttinfo->cid = csk->port_id; 1325 1326 cxgbi_ppm_make_ppod_hdr(ppm, ttinfo->tag, csk->tid, sgl->offset, 1327 xferlen, &ttinfo->hdr); 1328 1329 if (cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ) { 1330 /* write ppod from xmit_pdu (of iscsi_scsi_command pdu) */ 1331 ttinfo->flags |= CXGBI_PPOD_INFO_FLAG_VALID; 1332 } else { 1333 /* write ppod from control queue now */ 1334 err = cdev->csk_ddp_set_map(ppm, csk, ttinfo); 1335 if (err < 0) 1336 goto rel_ppods; 1337 } 1338 1339 return 0; 1340 1341 rel_ppods: 1342 cxgbi_ppm_ppod_release(ppm, ttinfo->idx); 1343 1344 if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_MAPPED) { 1345 ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_MAPPED; 1346 dma_unmap_sg(&ppm->pdev->dev, sgl, sgcnt, DMA_FROM_DEVICE); 1347 } 1348 return -EINVAL; 1349 } 1350 1351 static void task_release_itt(struct iscsi_task *task, itt_t hdr_itt) 1352 { 1353 struct scsi_cmnd *sc = task->sc; 1354 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 1355 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1356 struct cxgbi_device *cdev = cconn->chba->cdev; 1357 struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev); 1358 u32 tag = ntohl((__force u32)hdr_itt); 1359 1360 log_debug(1 << CXGBI_DBG_DDP, 1361 "cdev 0x%p, task 0x%p, release tag 0x%x.\n", 1362 cdev, task, tag); 1363 if (sc && sc->sc_data_direction == DMA_FROM_DEVICE && 1364 cxgbi_ppm_is_ddp_tag(ppm, tag)) { 1365 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1366 struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; 1367 1368 if (!(cdev->flags & CXGBI_FLAG_USE_PPOD_OFLDQ)) 1369 cdev->csk_ddp_clear_map(cdev, ppm, ttinfo); 1370 cxgbi_ppm_ppod_release(ppm, ttinfo->idx); 1371 dma_unmap_sg(&ppm->pdev->dev, ttinfo->sgl, ttinfo->nents, 1372 DMA_FROM_DEVICE); 1373 } 1374 } 1375 1376 static inline u32 cxgbi_build_sw_tag(u32 idx, u32 age) 1377 { 1378 /* assume idx and age both are < 0x7FFF (32767) */ 1379 return (idx << 16) | age; 1380 } 1381 1382 static int task_reserve_itt(struct iscsi_task *task, itt_t *hdr_itt) 1383 { 1384 struct scsi_cmnd *sc = task->sc; 1385 struct iscsi_conn *conn = task->conn; 1386 struct iscsi_session *sess = conn->session; 1387 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1388 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1389 struct cxgbi_device *cdev = cconn->chba->cdev; 1390 struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev); 1391 u32 sw_tag = cxgbi_build_sw_tag(task->itt, sess->age); 1392 u32 tag = 0; 1393 int err = -EINVAL; 1394 1395 if (sc && sc->sc_data_direction == DMA_FROM_DEVICE) { 1396 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1397 struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; 1398 1399 scmd_get_params(sc, &ttinfo->sgl, &ttinfo->nents, 1400 &tdata->dlen, 0); 1401 err = cxgbi_ddp_reserve(cconn, tdata, sw_tag, tdata->dlen); 1402 if (!err) 1403 tag = ttinfo->tag; 1404 else 1405 log_debug(1 << CXGBI_DBG_DDP, 1406 "csk 0x%p, R task 0x%p, %u,%u, no ddp.\n", 1407 cconn->cep->csk, task, tdata->dlen, 1408 ttinfo->nents); 1409 } 1410 1411 if (err < 0) { 1412 err = cxgbi_ppm_make_non_ddp_tag(ppm, sw_tag, &tag); 1413 if (err < 0) 1414 return err; 1415 } 1416 /* the itt need to sent in big-endian order */ 1417 *hdr_itt = (__force itt_t)htonl(tag); 1418 1419 log_debug(1 << CXGBI_DBG_DDP, 1420 "cdev 0x%p, task 0x%p, 0x%x(0x%x,0x%x)->0x%x/0x%x.\n", 1421 cdev, task, sw_tag, task->itt, sess->age, tag, *hdr_itt); 1422 return 0; 1423 } 1424 1425 void cxgbi_parse_pdu_itt(struct iscsi_conn *conn, itt_t itt, int *idx, int *age) 1426 { 1427 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1428 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1429 struct cxgbi_device *cdev = cconn->chba->cdev; 1430 struct cxgbi_ppm *ppm = cdev->cdev2ppm(cdev); 1431 u32 tag = ntohl((__force u32)itt); 1432 u32 sw_bits; 1433 1434 if (ppm) { 1435 if (cxgbi_ppm_is_ddp_tag(ppm, tag)) 1436 sw_bits = cxgbi_ppm_get_tag_caller_data(ppm, tag); 1437 else 1438 sw_bits = cxgbi_ppm_decode_non_ddp_tag(ppm, tag); 1439 } else { 1440 sw_bits = tag; 1441 } 1442 1443 cxgbi_decode_sw_tag(sw_bits, idx, age); 1444 log_debug(1 << CXGBI_DBG_DDP, 1445 "cdev 0x%p, tag 0x%x/0x%x, -> 0x%x(0x%x,0x%x).\n", 1446 cdev, tag, itt, sw_bits, idx ? *idx : 0xFFFFF, 1447 age ? *age : 0xFF); 1448 } 1449 EXPORT_SYMBOL_GPL(cxgbi_parse_pdu_itt); 1450 1451 void cxgbi_conn_tx_open(struct cxgbi_sock *csk) 1452 { 1453 struct iscsi_conn *conn = csk->user_data; 1454 1455 if (conn) { 1456 log_debug(1 << CXGBI_DBG_SOCK, 1457 "csk 0x%p, cid %d.\n", csk, conn->id); 1458 iscsi_conn_queue_xmit(conn); 1459 } 1460 } 1461 EXPORT_SYMBOL_GPL(cxgbi_conn_tx_open); 1462 1463 /* 1464 * pdu receive, interact with libiscsi_tcp 1465 */ 1466 static inline int read_pdu_skb(struct iscsi_conn *conn, 1467 struct sk_buff *skb, 1468 unsigned int offset, 1469 int offloaded) 1470 { 1471 int status = 0; 1472 int bytes_read; 1473 1474 bytes_read = iscsi_tcp_recv_skb(conn, skb, offset, offloaded, &status); 1475 switch (status) { 1476 case ISCSI_TCP_CONN_ERR: 1477 pr_info("skb 0x%p, off %u, %d, TCP_ERR.\n", 1478 skb, offset, offloaded); 1479 return -EIO; 1480 case ISCSI_TCP_SUSPENDED: 1481 log_debug(1 << CXGBI_DBG_PDU_RX, 1482 "skb 0x%p, off %u, %d, TCP_SUSPEND, rc %d.\n", 1483 skb, offset, offloaded, bytes_read); 1484 /* no transfer - just have caller flush queue */ 1485 return bytes_read; 1486 case ISCSI_TCP_SKB_DONE: 1487 pr_info("skb 0x%p, off %u, %d, TCP_SKB_DONE.\n", 1488 skb, offset, offloaded); 1489 /* 1490 * pdus should always fit in the skb and we should get 1491 * segment done notifcation. 1492 */ 1493 iscsi_conn_printk(KERN_ERR, conn, "Invalid pdu or skb."); 1494 return -EFAULT; 1495 case ISCSI_TCP_SEGMENT_DONE: 1496 log_debug(1 << CXGBI_DBG_PDU_RX, 1497 "skb 0x%p, off %u, %d, TCP_SEG_DONE, rc %d.\n", 1498 skb, offset, offloaded, bytes_read); 1499 return bytes_read; 1500 default: 1501 pr_info("skb 0x%p, off %u, %d, invalid status %d.\n", 1502 skb, offset, offloaded, status); 1503 return -EINVAL; 1504 } 1505 } 1506 1507 static int 1508 skb_read_pdu_bhs(struct cxgbi_sock *csk, struct iscsi_conn *conn, 1509 struct sk_buff *skb) 1510 { 1511 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1512 int err; 1513 1514 log_debug(1 << CXGBI_DBG_PDU_RX, 1515 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n", 1516 conn, skb, skb->len, cxgbi_skcb_flags(skb)); 1517 1518 if (!iscsi_tcp_recv_segment_is_hdr(tcp_conn)) { 1519 pr_info("conn 0x%p, skb 0x%p, not hdr.\n", conn, skb); 1520 iscsi_conn_failure(conn, ISCSI_ERR_PROTO); 1521 return -EIO; 1522 } 1523 1524 if (conn->hdrdgst_en && 1525 cxgbi_skcb_test_flag(skb, SKCBF_RX_HCRC_ERR)) { 1526 pr_info("conn 0x%p, skb 0x%p, hcrc.\n", conn, skb); 1527 iscsi_conn_failure(conn, ISCSI_ERR_HDR_DGST); 1528 return -EIO; 1529 } 1530 1531 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_ISCSI_COMPL) && 1532 cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA_DDPD)) { 1533 /* If completion flag is set and data is directly 1534 * placed in to the host memory then update 1535 * task->exp_datasn to the datasn in completion 1536 * iSCSI hdr as T6 adapter generates completion only 1537 * for the last pdu of a sequence. 1538 */ 1539 itt_t itt = ((struct iscsi_data *)skb->data)->itt; 1540 struct iscsi_task *task = iscsi_itt_to_ctask(conn, itt); 1541 u32 data_sn = be32_to_cpu(((struct iscsi_data *) 1542 skb->data)->datasn); 1543 if (task && task->sc) { 1544 struct iscsi_tcp_task *tcp_task = task->dd_data; 1545 1546 tcp_task->exp_datasn = data_sn; 1547 } 1548 } 1549 1550 err = read_pdu_skb(conn, skb, 0, 0); 1551 if (likely(err >= 0)) { 1552 struct iscsi_hdr *hdr = (struct iscsi_hdr *)skb->data; 1553 u8 opcode = hdr->opcode & ISCSI_OPCODE_MASK; 1554 1555 if (unlikely(opcode == ISCSI_OP_LOGOUT_RSP)) 1556 cxgbi_sock_set_flag(csk, CTPF_LOGOUT_RSP_RCVD); 1557 } 1558 1559 return err; 1560 } 1561 1562 static int skb_read_pdu_data(struct iscsi_conn *conn, struct sk_buff *lskb, 1563 struct sk_buff *skb, unsigned int offset) 1564 { 1565 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1566 bool offloaded = 0; 1567 int opcode = tcp_conn->in.hdr->opcode & ISCSI_OPCODE_MASK; 1568 1569 log_debug(1 << CXGBI_DBG_PDU_RX, 1570 "conn 0x%p, skb 0x%p, len %u, flag 0x%lx.\n", 1571 conn, skb, skb->len, cxgbi_skcb_flags(skb)); 1572 1573 if (conn->datadgst_en && 1574 cxgbi_skcb_test_flag(lskb, SKCBF_RX_DCRC_ERR)) { 1575 pr_info("conn 0x%p, skb 0x%p, dcrc 0x%lx.\n", 1576 conn, lskb, cxgbi_skcb_flags(lskb)); 1577 iscsi_conn_failure(conn, ISCSI_ERR_DATA_DGST); 1578 return -EIO; 1579 } 1580 1581 if (iscsi_tcp_recv_segment_is_hdr(tcp_conn)) 1582 return 0; 1583 1584 /* coalesced, add header digest length */ 1585 if (lskb == skb && conn->hdrdgst_en) 1586 offset += ISCSI_DIGEST_SIZE; 1587 1588 if (cxgbi_skcb_test_flag(lskb, SKCBF_RX_DATA_DDPD)) 1589 offloaded = 1; 1590 1591 if (opcode == ISCSI_OP_SCSI_DATA_IN) 1592 log_debug(1 << CXGBI_DBG_PDU_RX, 1593 "skb 0x%p, op 0x%x, itt 0x%x, %u %s ddp'ed.\n", 1594 skb, opcode, ntohl(tcp_conn->in.hdr->itt), 1595 tcp_conn->in.datalen, offloaded ? "is" : "not"); 1596 1597 return read_pdu_skb(conn, skb, offset, offloaded); 1598 } 1599 1600 static void csk_return_rx_credits(struct cxgbi_sock *csk, int copied) 1601 { 1602 struct cxgbi_device *cdev = csk->cdev; 1603 int must_send; 1604 u32 credits; 1605 1606 log_debug(1 << CXGBI_DBG_PDU_RX, 1607 "csk 0x%p,%u,0x%lx,%u, seq %u, wup %u, thre %u, %u.\n", 1608 csk, csk->state, csk->flags, csk->tid, csk->copied_seq, 1609 csk->rcv_wup, cdev->rx_credit_thres, 1610 csk->rcv_win); 1611 1612 if (!cdev->rx_credit_thres) 1613 return; 1614 1615 if (csk->state != CTP_ESTABLISHED) 1616 return; 1617 1618 credits = csk->copied_seq - csk->rcv_wup; 1619 if (unlikely(!credits)) 1620 return; 1621 must_send = credits + 16384 >= csk->rcv_win; 1622 if (must_send || credits >= cdev->rx_credit_thres) 1623 csk->rcv_wup += cdev->csk_send_rx_credits(csk, credits); 1624 } 1625 1626 void cxgbi_conn_pdu_ready(struct cxgbi_sock *csk) 1627 { 1628 struct cxgbi_device *cdev = csk->cdev; 1629 struct iscsi_conn *conn = csk->user_data; 1630 struct sk_buff *skb; 1631 unsigned int read = 0; 1632 int err = 0; 1633 1634 log_debug(1 << CXGBI_DBG_PDU_RX, 1635 "csk 0x%p, conn 0x%p.\n", csk, conn); 1636 1637 if (unlikely(!conn || test_bit(ISCSI_CONN_FLAG_SUSPEND_RX, &conn->flags))) { 1638 log_debug(1 << CXGBI_DBG_PDU_RX, 1639 "csk 0x%p, conn 0x%p, id %d, conn flags 0x%lx!\n", 1640 csk, conn, conn ? conn->id : 0xFF, 1641 conn ? conn->flags : 0xFF); 1642 return; 1643 } 1644 1645 while (!err) { 1646 skb = skb_peek(&csk->receive_queue); 1647 if (!skb || 1648 !(cxgbi_skcb_test_flag(skb, SKCBF_RX_STATUS))) { 1649 if (skb) 1650 log_debug(1 << CXGBI_DBG_PDU_RX, 1651 "skb 0x%p, NOT ready 0x%lx.\n", 1652 skb, cxgbi_skcb_flags(skb)); 1653 break; 1654 } 1655 __skb_unlink(skb, &csk->receive_queue); 1656 1657 read += cxgbi_skcb_rx_pdulen(skb); 1658 log_debug(1 << CXGBI_DBG_PDU_RX, 1659 "csk 0x%p, skb 0x%p,%u,f 0x%lx, pdu len %u.\n", 1660 csk, skb, skb->len, cxgbi_skcb_flags(skb), 1661 cxgbi_skcb_rx_pdulen(skb)); 1662 1663 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_COALESCED)) { 1664 err = skb_read_pdu_bhs(csk, conn, skb); 1665 if (err < 0) { 1666 pr_err("coalesced bhs, csk 0x%p, skb 0x%p,%u, " 1667 "f 0x%lx, plen %u.\n", 1668 csk, skb, skb->len, 1669 cxgbi_skcb_flags(skb), 1670 cxgbi_skcb_rx_pdulen(skb)); 1671 goto skb_done; 1672 } 1673 err = skb_read_pdu_data(conn, skb, skb, 1674 err + cdev->skb_rx_extra); 1675 if (err < 0) 1676 pr_err("coalesced data, csk 0x%p, skb 0x%p,%u, " 1677 "f 0x%lx, plen %u.\n", 1678 csk, skb, skb->len, 1679 cxgbi_skcb_flags(skb), 1680 cxgbi_skcb_rx_pdulen(skb)); 1681 } else { 1682 err = skb_read_pdu_bhs(csk, conn, skb); 1683 if (err < 0) { 1684 pr_err("bhs, csk 0x%p, skb 0x%p,%u, " 1685 "f 0x%lx, plen %u.\n", 1686 csk, skb, skb->len, 1687 cxgbi_skcb_flags(skb), 1688 cxgbi_skcb_rx_pdulen(skb)); 1689 goto skb_done; 1690 } 1691 1692 if (cxgbi_skcb_test_flag(skb, SKCBF_RX_DATA)) { 1693 struct sk_buff *dskb; 1694 1695 dskb = skb_peek(&csk->receive_queue); 1696 if (!dskb) { 1697 pr_err("csk 0x%p, skb 0x%p,%u, f 0x%lx," 1698 " plen %u, NO data.\n", 1699 csk, skb, skb->len, 1700 cxgbi_skcb_flags(skb), 1701 cxgbi_skcb_rx_pdulen(skb)); 1702 err = -EIO; 1703 goto skb_done; 1704 } 1705 __skb_unlink(dskb, &csk->receive_queue); 1706 1707 err = skb_read_pdu_data(conn, skb, dskb, 0); 1708 if (err < 0) 1709 pr_err("data, csk 0x%p, skb 0x%p,%u, " 1710 "f 0x%lx, plen %u, dskb 0x%p," 1711 "%u.\n", 1712 csk, skb, skb->len, 1713 cxgbi_skcb_flags(skb), 1714 cxgbi_skcb_rx_pdulen(skb), 1715 dskb, dskb->len); 1716 __kfree_skb(dskb); 1717 } else 1718 err = skb_read_pdu_data(conn, skb, skb, 0); 1719 } 1720 skb_done: 1721 __kfree_skb(skb); 1722 1723 if (err < 0) 1724 break; 1725 } 1726 1727 log_debug(1 << CXGBI_DBG_PDU_RX, "csk 0x%p, read %u.\n", csk, read); 1728 if (read) { 1729 csk->copied_seq += read; 1730 csk_return_rx_credits(csk, read); 1731 conn->rxdata_octets += read; 1732 } 1733 1734 if (err < 0) { 1735 pr_info("csk 0x%p, 0x%p, rx failed %d, read %u.\n", 1736 csk, conn, err, read); 1737 iscsi_conn_failure(conn, ISCSI_ERR_CONN_FAILED); 1738 } 1739 } 1740 EXPORT_SYMBOL_GPL(cxgbi_conn_pdu_ready); 1741 1742 static int sgl_seek_offset(struct scatterlist *sgl, unsigned int sgcnt, 1743 unsigned int offset, unsigned int *off, 1744 struct scatterlist **sgp) 1745 { 1746 int i; 1747 struct scatterlist *sg; 1748 1749 for_each_sg(sgl, sg, sgcnt, i) { 1750 if (offset < sg->length) { 1751 *off = offset; 1752 *sgp = sg; 1753 return 0; 1754 } 1755 offset -= sg->length; 1756 } 1757 return -EFAULT; 1758 } 1759 1760 static int 1761 sgl_read_to_frags(struct scatterlist *sg, unsigned int sgoffset, 1762 unsigned int dlen, struct page_frag *frags, 1763 int frag_max, u32 *dlimit) 1764 { 1765 unsigned int datalen = dlen; 1766 unsigned int sglen = sg->length - sgoffset; 1767 struct page *page = sg_page(sg); 1768 int i; 1769 1770 i = 0; 1771 do { 1772 unsigned int copy; 1773 1774 if (!sglen) { 1775 sg = sg_next(sg); 1776 if (!sg) { 1777 pr_warn("sg %d NULL, len %u/%u.\n", 1778 i, datalen, dlen); 1779 return -EINVAL; 1780 } 1781 sgoffset = 0; 1782 sglen = sg->length; 1783 page = sg_page(sg); 1784 1785 } 1786 copy = min(datalen, sglen); 1787 if (i && page == frags[i - 1].page && 1788 sgoffset + sg->offset == 1789 frags[i - 1].offset + frags[i - 1].size) { 1790 frags[i - 1].size += copy; 1791 } else { 1792 if (i >= frag_max) { 1793 pr_warn("too many pages %u, dlen %u.\n", 1794 frag_max, dlen); 1795 *dlimit = dlen - datalen; 1796 return -EINVAL; 1797 } 1798 1799 frags[i].page = page; 1800 frags[i].offset = sg->offset + sgoffset; 1801 frags[i].size = copy; 1802 i++; 1803 } 1804 datalen -= copy; 1805 sgoffset += copy; 1806 sglen -= copy; 1807 } while (datalen); 1808 1809 return i; 1810 } 1811 1812 static void cxgbi_task_data_sgl_check(struct iscsi_task *task) 1813 { 1814 struct scsi_cmnd *sc = task->sc; 1815 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1816 struct scatterlist *sg, *sgl = NULL; 1817 u32 sgcnt = 0; 1818 int i; 1819 1820 tdata->flags = CXGBI_TASK_SGL_CHECKED; 1821 if (!sc) 1822 return; 1823 1824 scmd_get_params(sc, &sgl, &sgcnt, &tdata->dlen, 0); 1825 if (!sgl || !sgcnt) { 1826 tdata->flags |= CXGBI_TASK_SGL_COPY; 1827 return; 1828 } 1829 1830 for_each_sg(sgl, sg, sgcnt, i) { 1831 if (page_count(sg_page(sg)) < 1) { 1832 tdata->flags |= CXGBI_TASK_SGL_COPY; 1833 return; 1834 } 1835 } 1836 } 1837 1838 static int 1839 cxgbi_task_data_sgl_read(struct iscsi_task *task, u32 offset, u32 count, 1840 u32 *dlimit) 1841 { 1842 struct scsi_cmnd *sc = task->sc; 1843 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1844 struct scatterlist *sgl = NULL; 1845 struct scatterlist *sg; 1846 u32 dlen = 0; 1847 u32 sgcnt; 1848 int err; 1849 1850 if (!sc) 1851 return 0; 1852 1853 scmd_get_params(sc, &sgl, &sgcnt, &dlen, 0); 1854 if (!sgl || !sgcnt) 1855 return 0; 1856 1857 err = sgl_seek_offset(sgl, sgcnt, offset, &tdata->sgoffset, &sg); 1858 if (err < 0) { 1859 pr_warn("tpdu max, sgl %u, bad offset %u/%u.\n", 1860 sgcnt, offset, tdata->dlen); 1861 return err; 1862 } 1863 err = sgl_read_to_frags(sg, tdata->sgoffset, count, 1864 tdata->frags, MAX_SKB_FRAGS, dlimit); 1865 if (err < 0) { 1866 log_debug(1 << CXGBI_DBG_ISCSI, 1867 "sgl max limit, sgl %u, offset %u, %u/%u, dlimit %u.\n", 1868 sgcnt, offset, count, tdata->dlen, *dlimit); 1869 return err; 1870 } 1871 tdata->offset = offset; 1872 tdata->count = count; 1873 tdata->nr_frags = err; 1874 tdata->total_count = count; 1875 tdata->total_offset = offset; 1876 1877 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 1878 "%s: offset %u, count %u,\n" 1879 "err %u, total_count %u, total_offset %u\n", 1880 __func__, offset, count, err, tdata->total_count, tdata->total_offset); 1881 1882 return 0; 1883 } 1884 1885 int cxgbi_conn_alloc_pdu(struct iscsi_task *task, u8 op) 1886 { 1887 struct iscsi_conn *conn = task->conn; 1888 struct iscsi_session *session = task->conn->session; 1889 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 1890 struct cxgbi_conn *cconn = tcp_conn->dd_data; 1891 struct cxgbi_device *cdev = cconn->chba->cdev; 1892 struct cxgbi_sock *csk = cconn->cep ? cconn->cep->csk : NULL; 1893 struct iscsi_tcp_task *tcp_task = task->dd_data; 1894 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 1895 struct scsi_cmnd *sc = task->sc; 1896 u32 headroom = SKB_TX_ISCSI_PDU_HEADER_MAX; 1897 u32 max_txdata_len = conn->max_xmit_dlength; 1898 u32 iso_tx_rsvd = 0, local_iso_info = 0; 1899 u32 last_tdata_offset, last_tdata_count; 1900 int err = 0; 1901 1902 if (!tcp_task) { 1903 pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p.\n", 1904 task, tcp_task, tdata); 1905 return -ENOMEM; 1906 } 1907 if (!csk) { 1908 pr_err("task 0x%p, csk gone.\n", task); 1909 return -EPIPE; 1910 } 1911 1912 op &= ISCSI_OPCODE_MASK; 1913 1914 tcp_task->dd_data = tdata; 1915 task->hdr = NULL; 1916 1917 last_tdata_count = tdata->count; 1918 last_tdata_offset = tdata->offset; 1919 1920 if ((op == ISCSI_OP_SCSI_DATA_OUT) || 1921 ((op == ISCSI_OP_SCSI_CMD) && 1922 (sc->sc_data_direction == DMA_TO_DEVICE))) { 1923 u32 remaining_data_tosend, dlimit = 0; 1924 u32 max_pdu_size, max_num_pdu, num_pdu; 1925 u32 count; 1926 1927 /* Preserve conn->max_xmit_dlength because it can get updated to 1928 * ISO data size. 1929 */ 1930 if (task->state == ISCSI_TASK_PENDING) 1931 tdata->max_xmit_dlength = conn->max_xmit_dlength; 1932 1933 if (!tdata->offset) 1934 cxgbi_task_data_sgl_check(task); 1935 1936 remaining_data_tosend = 1937 tdata->dlen - tdata->offset - tdata->count; 1938 1939 recalculate_sgl: 1940 max_txdata_len = tdata->max_xmit_dlength; 1941 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 1942 "tdata->dlen %u, remaining to send %u " 1943 "conn->max_xmit_dlength %u, " 1944 "tdata->max_xmit_dlength %u\n", 1945 tdata->dlen, remaining_data_tosend, 1946 conn->max_xmit_dlength, tdata->max_xmit_dlength); 1947 1948 if (cdev->skb_iso_txhdr && !csk->disable_iso && 1949 (remaining_data_tosend > tdata->max_xmit_dlength) && 1950 !(remaining_data_tosend % 4)) { 1951 u32 max_iso_data; 1952 1953 if ((op == ISCSI_OP_SCSI_CMD) && 1954 session->initial_r2t_en) 1955 goto no_iso; 1956 1957 max_pdu_size = tdata->max_xmit_dlength + 1958 ISCSI_PDU_NONPAYLOAD_LEN; 1959 max_iso_data = rounddown(CXGBI_MAX_ISO_DATA_IN_SKB, 1960 csk->advmss); 1961 max_num_pdu = max_iso_data / max_pdu_size; 1962 1963 num_pdu = (remaining_data_tosend + 1964 tdata->max_xmit_dlength - 1) / 1965 tdata->max_xmit_dlength; 1966 1967 if (num_pdu > max_num_pdu) 1968 num_pdu = max_num_pdu; 1969 1970 conn->max_xmit_dlength = tdata->max_xmit_dlength * num_pdu; 1971 max_txdata_len = conn->max_xmit_dlength; 1972 iso_tx_rsvd = cdev->skb_iso_txhdr; 1973 local_iso_info = sizeof(struct cxgbi_iso_info); 1974 1975 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 1976 "max_pdu_size %u, max_num_pdu %u, " 1977 "max_txdata %u, num_pdu %u\n", 1978 max_pdu_size, max_num_pdu, 1979 max_txdata_len, num_pdu); 1980 } 1981 no_iso: 1982 count = min_t(u32, max_txdata_len, remaining_data_tosend); 1983 err = cxgbi_task_data_sgl_read(task, 1984 tdata->offset + tdata->count, 1985 count, &dlimit); 1986 if (unlikely(err < 0)) { 1987 log_debug(1 << CXGBI_DBG_ISCSI, 1988 "task 0x%p, tcp_task 0x%p, tdata 0x%p, " 1989 "sgl err %d, count %u, dlimit %u\n", 1990 task, tcp_task, tdata, err, count, dlimit); 1991 if (dlimit) { 1992 remaining_data_tosend = 1993 rounddown(dlimit, 1994 tdata->max_xmit_dlength); 1995 if (!remaining_data_tosend) 1996 remaining_data_tosend = dlimit; 1997 1998 dlimit = 0; 1999 2000 conn->max_xmit_dlength = remaining_data_tosend; 2001 goto recalculate_sgl; 2002 } 2003 2004 pr_err("task 0x%p, tcp_task 0x%p, tdata 0x%p, " 2005 "sgl err %d\n", 2006 task, tcp_task, tdata, err); 2007 goto ret_err; 2008 } 2009 2010 if ((tdata->flags & CXGBI_TASK_SGL_COPY) || 2011 (tdata->nr_frags > MAX_SKB_FRAGS)) 2012 headroom += conn->max_xmit_dlength; 2013 } 2014 2015 tdata->skb = alloc_skb(local_iso_info + cdev->skb_tx_rsvd + 2016 iso_tx_rsvd + headroom, GFP_ATOMIC); 2017 if (!tdata->skb) { 2018 tdata->count = last_tdata_count; 2019 tdata->offset = last_tdata_offset; 2020 err = -ENOMEM; 2021 goto ret_err; 2022 } 2023 2024 skb_reserve(tdata->skb, local_iso_info + cdev->skb_tx_rsvd + 2025 iso_tx_rsvd); 2026 2027 if (task->sc) { 2028 task->hdr = (struct iscsi_hdr *)tdata->skb->data; 2029 } else { 2030 task->hdr = kzalloc(SKB_TX_ISCSI_PDU_HEADER_MAX, GFP_ATOMIC); 2031 if (!task->hdr) { 2032 __kfree_skb(tdata->skb); 2033 tdata->skb = NULL; 2034 return -ENOMEM; 2035 } 2036 } 2037 2038 task->hdr_max = SKB_TX_ISCSI_PDU_HEADER_MAX; 2039 2040 if (iso_tx_rsvd) 2041 cxgbi_skcb_set_flag(tdata->skb, SKCBF_TX_ISO); 2042 2043 /* data_out uses scsi_cmd's itt */ 2044 if (op != ISCSI_OP_SCSI_DATA_OUT) 2045 task_reserve_itt(task, &task->hdr->itt); 2046 2047 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2048 "task 0x%p, op 0x%x, skb 0x%p,%u+%u/%u, itt 0x%x.\n", 2049 task, op, tdata->skb, cdev->skb_tx_rsvd, headroom, 2050 conn->max_xmit_dlength, be32_to_cpu(task->hdr->itt)); 2051 2052 return 0; 2053 2054 ret_err: 2055 conn->max_xmit_dlength = tdata->max_xmit_dlength; 2056 return err; 2057 } 2058 EXPORT_SYMBOL_GPL(cxgbi_conn_alloc_pdu); 2059 2060 static int 2061 cxgbi_prep_iso_info(struct iscsi_task *task, struct sk_buff *skb, 2062 u32 count) 2063 { 2064 struct cxgbi_iso_info *iso_info = (struct cxgbi_iso_info *)skb->head; 2065 struct iscsi_r2t_info *r2t; 2066 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2067 struct iscsi_conn *conn = task->conn; 2068 struct iscsi_session *session = conn->session; 2069 struct iscsi_tcp_task *tcp_task = task->dd_data; 2070 u32 burst_size = 0, r2t_dlength = 0, dlength; 2071 u32 max_pdu_len = tdata->max_xmit_dlength; 2072 u32 segment_offset = 0; 2073 u32 num_pdu; 2074 2075 if (unlikely(!cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))) 2076 return 0; 2077 2078 memset(iso_info, 0, sizeof(struct cxgbi_iso_info)); 2079 2080 if (task->hdr->opcode == ISCSI_OP_SCSI_CMD && session->imm_data_en) { 2081 iso_info->flags |= CXGBI_ISO_INFO_IMM_ENABLE; 2082 burst_size = count; 2083 } 2084 2085 dlength = ntoh24(task->hdr->dlength); 2086 dlength = min(dlength, max_pdu_len); 2087 hton24(task->hdr->dlength, dlength); 2088 2089 num_pdu = (count + max_pdu_len - 1) / max_pdu_len; 2090 2091 if (iscsi_task_has_unsol_data(task)) 2092 r2t = &task->unsol_r2t; 2093 else 2094 r2t = tcp_task->r2t; 2095 2096 if (r2t) { 2097 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2098 "count %u, tdata->count %u, num_pdu %u," 2099 "task->hdr_len %u, r2t->data_length %u, r2t->sent %u\n", 2100 count, tdata->count, num_pdu, task->hdr_len, 2101 r2t->data_length, r2t->sent); 2102 2103 r2t_dlength = r2t->data_length - r2t->sent; 2104 segment_offset = r2t->sent; 2105 r2t->datasn += num_pdu - 1; 2106 } 2107 2108 if (!r2t || !r2t->sent) 2109 iso_info->flags |= CXGBI_ISO_INFO_FSLICE; 2110 2111 if (task->hdr->flags & ISCSI_FLAG_CMD_FINAL) 2112 iso_info->flags |= CXGBI_ISO_INFO_LSLICE; 2113 2114 task->hdr->flags &= ~ISCSI_FLAG_CMD_FINAL; 2115 2116 iso_info->op = task->hdr->opcode; 2117 iso_info->ahs = task->hdr->hlength; 2118 iso_info->num_pdu = num_pdu; 2119 iso_info->mpdu = max_pdu_len; 2120 iso_info->burst_size = (burst_size + r2t_dlength) >> 2; 2121 iso_info->len = count + task->hdr_len; 2122 iso_info->segment_offset = segment_offset; 2123 2124 cxgbi_skcb_tx_iscsi_hdrlen(skb) = task->hdr_len; 2125 return 0; 2126 } 2127 2128 static inline void tx_skb_setmode(struct sk_buff *skb, int hcrc, int dcrc) 2129 { 2130 if (hcrc || dcrc) { 2131 u8 submode = 0; 2132 2133 if (hcrc) 2134 submode |= 1; 2135 if (dcrc) 2136 submode |= 2; 2137 cxgbi_skcb_tx_ulp_mode(skb) = (ULP2_MODE_ISCSI << 4) | submode; 2138 } else 2139 cxgbi_skcb_tx_ulp_mode(skb) = 0; 2140 } 2141 2142 static struct page *rsvd_page; 2143 2144 int cxgbi_conn_init_pdu(struct iscsi_task *task, unsigned int offset, 2145 unsigned int count) 2146 { 2147 struct iscsi_conn *conn = task->conn; 2148 struct iscsi_tcp_task *tcp_task = task->dd_data; 2149 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2150 struct sk_buff *skb; 2151 struct scsi_cmnd *sc = task->sc; 2152 u32 expected_count, expected_offset; 2153 u32 datalen = count, dlimit = 0; 2154 u32 i, padlen = iscsi_padding(count); 2155 struct page *pg; 2156 int err; 2157 2158 if (!tcp_task || (tcp_task->dd_data != tdata)) { 2159 pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n", 2160 task, task->sc, tcp_task, 2161 tcp_task ? tcp_task->dd_data : NULL, tdata); 2162 return -EINVAL; 2163 } 2164 skb = tdata->skb; 2165 2166 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2167 "task 0x%p,0x%p, skb 0x%p, 0x%x,0x%x,0x%x, %u+%u.\n", 2168 task, task->sc, skb, (*skb->data) & ISCSI_OPCODE_MASK, 2169 be32_to_cpu(task->cmdsn), be32_to_cpu(task->hdr->itt), offset, count); 2170 2171 skb_put(skb, task->hdr_len); 2172 tx_skb_setmode(skb, conn->hdrdgst_en, datalen ? conn->datadgst_en : 0); 2173 if (!count) { 2174 tdata->count = count; 2175 tdata->offset = offset; 2176 tdata->nr_frags = 0; 2177 tdata->total_offset = 0; 2178 tdata->total_count = 0; 2179 if (tdata->max_xmit_dlength) 2180 conn->max_xmit_dlength = tdata->max_xmit_dlength; 2181 cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO); 2182 return 0; 2183 } 2184 2185 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2186 "data->total_count %u, tdata->total_offset %u\n", 2187 tdata->total_count, tdata->total_offset); 2188 2189 expected_count = tdata->total_count; 2190 expected_offset = tdata->total_offset; 2191 2192 if ((count != expected_count) || 2193 (offset != expected_offset)) { 2194 err = cxgbi_task_data_sgl_read(task, offset, count, &dlimit); 2195 if (err < 0) { 2196 pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p " 2197 "dlimit %u, sgl err %d.\n", task, task->sc, 2198 tcp_task, tcp_task ? tcp_task->dd_data : NULL, 2199 tdata, dlimit, err); 2200 return err; 2201 } 2202 } 2203 2204 /* Restore original value of conn->max_xmit_dlength because 2205 * it can get updated to ISO data size. 2206 */ 2207 conn->max_xmit_dlength = tdata->max_xmit_dlength; 2208 2209 if (sc) { 2210 struct page_frag *frag = tdata->frags; 2211 2212 if ((tdata->flags & CXGBI_TASK_SGL_COPY) || 2213 (tdata->nr_frags > MAX_SKB_FRAGS) || 2214 (padlen && (tdata->nr_frags == 2215 MAX_SKB_FRAGS))) { 2216 char *dst = skb->data + task->hdr_len; 2217 2218 /* data fits in the skb's headroom */ 2219 for (i = 0; i < tdata->nr_frags; i++, frag++) { 2220 char *src = kmap_atomic(frag->page); 2221 2222 memcpy(dst, src + frag->offset, frag->size); 2223 dst += frag->size; 2224 kunmap_atomic(src); 2225 } 2226 2227 if (padlen) { 2228 memset(dst, 0, padlen); 2229 padlen = 0; 2230 } 2231 skb_put(skb, count + padlen); 2232 } else { 2233 for (i = 0; i < tdata->nr_frags; i++, frag++) { 2234 get_page(frag->page); 2235 skb_fill_page_desc(skb, i, frag->page, 2236 frag->offset, frag->size); 2237 } 2238 2239 skb->len += count; 2240 skb->data_len += count; 2241 skb->truesize += count; 2242 } 2243 } else { 2244 pg = virt_to_head_page(task->data); 2245 get_page(pg); 2246 skb_fill_page_desc(skb, 0, pg, 2247 task->data - (char *)page_address(pg), 2248 count); 2249 skb->len += count; 2250 skb->data_len += count; 2251 skb->truesize += count; 2252 } 2253 2254 if (padlen) { 2255 get_page(rsvd_page); 2256 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags, 2257 rsvd_page, 0, padlen); 2258 2259 skb->data_len += padlen; 2260 skb->truesize += padlen; 2261 skb->len += padlen; 2262 } 2263 2264 if (likely(count > tdata->max_xmit_dlength)) 2265 cxgbi_prep_iso_info(task, skb, count); 2266 else 2267 cxgbi_skcb_clear_flag(skb, SKCBF_TX_ISO); 2268 2269 return 0; 2270 } 2271 EXPORT_SYMBOL_GPL(cxgbi_conn_init_pdu); 2272 2273 static int cxgbi_sock_tx_queue_up(struct cxgbi_sock *csk, struct sk_buff *skb) 2274 { 2275 struct cxgbi_device *cdev = csk->cdev; 2276 struct cxgbi_iso_info *iso_cpl; 2277 u32 frags = skb_shinfo(skb)->nr_frags; 2278 u32 extra_len, num_pdu, hdr_len; 2279 u32 iso_tx_rsvd = 0; 2280 2281 if (csk->state != CTP_ESTABLISHED) { 2282 log_debug(1 << CXGBI_DBG_PDU_TX, 2283 "csk 0x%p,%u,0x%lx,%u, EAGAIN.\n", 2284 csk, csk->state, csk->flags, csk->tid); 2285 return -EPIPE; 2286 } 2287 2288 if (csk->err) { 2289 log_debug(1 << CXGBI_DBG_PDU_TX, 2290 "csk 0x%p,%u,0x%lx,%u, EPIPE %d.\n", 2291 csk, csk->state, csk->flags, csk->tid, csk->err); 2292 return -EPIPE; 2293 } 2294 2295 if ((cdev->flags & CXGBI_FLAG_DEV_T3) && 2296 before((csk->snd_win + csk->snd_una), csk->write_seq)) { 2297 log_debug(1 << CXGBI_DBG_PDU_TX, 2298 "csk 0x%p,%u,0x%lx,%u, FULL %u-%u >= %u.\n", 2299 csk, csk->state, csk->flags, csk->tid, csk->write_seq, 2300 csk->snd_una, csk->snd_win); 2301 return -ENOBUFS; 2302 } 2303 2304 if (cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO)) 2305 iso_tx_rsvd = cdev->skb_iso_txhdr; 2306 2307 if (unlikely(skb_headroom(skb) < (cdev->skb_tx_rsvd + iso_tx_rsvd))) { 2308 pr_err("csk 0x%p, skb head %u < %u.\n", 2309 csk, skb_headroom(skb), cdev->skb_tx_rsvd); 2310 return -EINVAL; 2311 } 2312 2313 if (skb->len != skb->data_len) 2314 frags++; 2315 2316 if (frags >= SKB_WR_LIST_SIZE) { 2317 pr_err("csk 0x%p, frags %u, %u,%u >%u.\n", 2318 csk, skb_shinfo(skb)->nr_frags, skb->len, 2319 skb->data_len, (unsigned int)SKB_WR_LIST_SIZE); 2320 return -EINVAL; 2321 } 2322 2323 cxgbi_skcb_set_flag(skb, SKCBF_TX_NEED_HDR); 2324 skb_reset_transport_header(skb); 2325 cxgbi_sock_skb_entail(csk, skb); 2326 2327 extra_len = cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb)); 2328 2329 if (likely(cxgbi_skcb_test_flag(skb, SKCBF_TX_ISO))) { 2330 iso_cpl = (struct cxgbi_iso_info *)skb->head; 2331 num_pdu = iso_cpl->num_pdu; 2332 hdr_len = cxgbi_skcb_tx_iscsi_hdrlen(skb); 2333 extra_len = (cxgbi_ulp_extra_len(cxgbi_skcb_tx_ulp_mode(skb)) * 2334 num_pdu) + (hdr_len * (num_pdu - 1)); 2335 } 2336 2337 csk->write_seq += (skb->len + extra_len); 2338 2339 return 0; 2340 } 2341 2342 static int cxgbi_sock_send_skb(struct cxgbi_sock *csk, struct sk_buff *skb) 2343 { 2344 struct cxgbi_device *cdev = csk->cdev; 2345 int len = skb->len; 2346 int err; 2347 2348 spin_lock_bh(&csk->lock); 2349 err = cxgbi_sock_tx_queue_up(csk, skb); 2350 if (err < 0) { 2351 spin_unlock_bh(&csk->lock); 2352 return err; 2353 } 2354 2355 if (likely(skb_queue_len(&csk->write_queue))) 2356 cdev->csk_push_tx_frames(csk, 0); 2357 spin_unlock_bh(&csk->lock); 2358 return len; 2359 } 2360 2361 int cxgbi_conn_xmit_pdu(struct iscsi_task *task) 2362 { 2363 struct iscsi_tcp_conn *tcp_conn = task->conn->dd_data; 2364 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2365 struct iscsi_tcp_task *tcp_task = task->dd_data; 2366 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2367 struct cxgbi_task_tag_info *ttinfo = &tdata->ttinfo; 2368 struct sk_buff *skb; 2369 struct cxgbi_sock *csk = NULL; 2370 u32 pdulen = 0; 2371 u32 datalen; 2372 int err; 2373 2374 if (!tcp_task || (tcp_task->dd_data != tdata)) { 2375 pr_err("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n", 2376 task, task->sc, tcp_task, 2377 tcp_task ? tcp_task->dd_data : NULL, tdata); 2378 return -EINVAL; 2379 } 2380 2381 skb = tdata->skb; 2382 if (!skb) { 2383 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2384 "task 0x%p, skb NULL.\n", task); 2385 return 0; 2386 } 2387 2388 if (cconn && cconn->cep) 2389 csk = cconn->cep->csk; 2390 2391 if (!csk) { 2392 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2393 "task 0x%p, csk gone.\n", task); 2394 return -EPIPE; 2395 } 2396 2397 tdata->skb = NULL; 2398 datalen = skb->data_len; 2399 2400 /* write ppod first if using ofldq to write ppod */ 2401 if (ttinfo->flags & CXGBI_PPOD_INFO_FLAG_VALID) { 2402 struct cxgbi_ppm *ppm = csk->cdev->cdev2ppm(csk->cdev); 2403 2404 ttinfo->flags &= ~CXGBI_PPOD_INFO_FLAG_VALID; 2405 if (csk->cdev->csk_ddp_set_map(ppm, csk, ttinfo) < 0) 2406 pr_err("task 0x%p, ppod writing using ofldq failed.\n", 2407 task); 2408 /* continue. Let fl get the data */ 2409 } 2410 2411 if (!task->sc) 2412 memcpy(skb->data, task->hdr, SKB_TX_ISCSI_PDU_HEADER_MAX); 2413 2414 err = cxgbi_sock_send_skb(csk, skb); 2415 if (err > 0) { 2416 pdulen += err; 2417 2418 log_debug(1 << CXGBI_DBG_PDU_TX, "task 0x%p,0x%p, rv %d.\n", 2419 task, task->sc, err); 2420 2421 if (task->conn->hdrdgst_en) 2422 pdulen += ISCSI_DIGEST_SIZE; 2423 2424 if (datalen && task->conn->datadgst_en) 2425 pdulen += ISCSI_DIGEST_SIZE; 2426 2427 task->conn->txdata_octets += pdulen; 2428 2429 if (unlikely(cxgbi_is_iso_config(csk) && cxgbi_is_iso_disabled(csk))) { 2430 if (time_after(jiffies, csk->prev_iso_ts + HZ)) { 2431 csk->disable_iso = false; 2432 csk->prev_iso_ts = 0; 2433 log_debug(1 << CXGBI_DBG_PDU_TX, 2434 "enable iso: csk 0x%p\n", csk); 2435 } 2436 } 2437 2438 return 0; 2439 } 2440 2441 if (err == -EAGAIN || err == -ENOBUFS) { 2442 log_debug(1 << CXGBI_DBG_PDU_TX, 2443 "task 0x%p, skb 0x%p, len %u/%u, %d EAGAIN.\n", 2444 task, skb, skb->len, skb->data_len, err); 2445 /* reset skb to send when we are called again */ 2446 tdata->skb = skb; 2447 2448 if (cxgbi_is_iso_config(csk) && !cxgbi_is_iso_disabled(csk) && 2449 (csk->no_tx_credits++ >= 2)) { 2450 csk->disable_iso = true; 2451 csk->prev_iso_ts = jiffies; 2452 log_debug(1 << CXGBI_DBG_PDU_TX, 2453 "disable iso:csk 0x%p, ts:%lu\n", 2454 csk, csk->prev_iso_ts); 2455 } 2456 2457 return err; 2458 } 2459 2460 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_PDU_TX, 2461 "itt 0x%x, skb 0x%p, len %u/%u, xmit err %d.\n", 2462 task->itt, skb, skb->len, skb->data_len, err); 2463 __kfree_skb(skb); 2464 iscsi_conn_printk(KERN_ERR, task->conn, "xmit err %d.\n", err); 2465 iscsi_conn_failure(task->conn, ISCSI_ERR_XMIT_FAILED); 2466 return err; 2467 } 2468 EXPORT_SYMBOL_GPL(cxgbi_conn_xmit_pdu); 2469 2470 void cxgbi_cleanup_task(struct iscsi_task *task) 2471 { 2472 struct iscsi_tcp_task *tcp_task = task->dd_data; 2473 struct cxgbi_task_data *tdata = iscsi_task_cxgbi_data(task); 2474 2475 if (!tcp_task || (tcp_task->dd_data != tdata)) { 2476 pr_info("task 0x%p,0x%p, tcp_task 0x%p, tdata 0x%p/0x%p.\n", 2477 task, task->sc, tcp_task, 2478 tcp_task ? tcp_task->dd_data : NULL, tdata); 2479 return; 2480 } 2481 2482 log_debug(1 << CXGBI_DBG_ISCSI, 2483 "task 0x%p, skb 0x%p, itt 0x%x.\n", 2484 task, tdata->skb, task->hdr_itt); 2485 2486 tcp_task->dd_data = NULL; 2487 2488 if (!task->sc) 2489 kfree(task->hdr); 2490 task->hdr = NULL; 2491 2492 /* never reached the xmit task callout */ 2493 if (tdata->skb) { 2494 __kfree_skb(tdata->skb); 2495 tdata->skb = NULL; 2496 } 2497 2498 task_release_itt(task, task->hdr_itt); 2499 memset(tdata, 0, sizeof(*tdata)); 2500 2501 iscsi_tcp_cleanup_task(task); 2502 } 2503 EXPORT_SYMBOL_GPL(cxgbi_cleanup_task); 2504 2505 void cxgbi_get_conn_stats(struct iscsi_cls_conn *cls_conn, 2506 struct iscsi_stats *stats) 2507 { 2508 struct iscsi_conn *conn = cls_conn->dd_data; 2509 2510 stats->txdata_octets = conn->txdata_octets; 2511 stats->rxdata_octets = conn->rxdata_octets; 2512 stats->scsicmd_pdus = conn->scsicmd_pdus_cnt; 2513 stats->dataout_pdus = conn->dataout_pdus_cnt; 2514 stats->scsirsp_pdus = conn->scsirsp_pdus_cnt; 2515 stats->datain_pdus = conn->datain_pdus_cnt; 2516 stats->r2t_pdus = conn->r2t_pdus_cnt; 2517 stats->tmfcmd_pdus = conn->tmfcmd_pdus_cnt; 2518 stats->tmfrsp_pdus = conn->tmfrsp_pdus_cnt; 2519 stats->digest_err = 0; 2520 stats->timeout_err = 0; 2521 stats->custom_length = 1; 2522 strcpy(stats->custom[0].desc, "eh_abort_cnt"); 2523 stats->custom[0].value = conn->eh_abort_cnt; 2524 } 2525 EXPORT_SYMBOL_GPL(cxgbi_get_conn_stats); 2526 2527 static int cxgbi_conn_max_xmit_dlength(struct iscsi_conn *conn) 2528 { 2529 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2530 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2531 struct cxgbi_device *cdev = cconn->chba->cdev; 2532 unsigned int headroom = SKB_MAX_HEAD(cdev->skb_tx_rsvd); 2533 unsigned int max_def = 512 * MAX_SKB_FRAGS; 2534 unsigned int max = max(max_def, headroom); 2535 2536 max = min(cconn->chba->cdev->tx_max_size, max); 2537 if (conn->max_xmit_dlength) 2538 conn->max_xmit_dlength = min(conn->max_xmit_dlength, max); 2539 else 2540 conn->max_xmit_dlength = max; 2541 cxgbi_align_pdu_size(conn->max_xmit_dlength); 2542 2543 return 0; 2544 } 2545 2546 static int cxgbi_conn_max_recv_dlength(struct iscsi_conn *conn) 2547 { 2548 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2549 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2550 unsigned int max = cconn->chba->cdev->rx_max_size; 2551 2552 cxgbi_align_pdu_size(max); 2553 2554 if (conn->max_recv_dlength) { 2555 if (conn->max_recv_dlength > max) { 2556 pr_err("MaxRecvDataSegmentLength %u > %u.\n", 2557 conn->max_recv_dlength, max); 2558 return -EINVAL; 2559 } 2560 conn->max_recv_dlength = min(conn->max_recv_dlength, max); 2561 cxgbi_align_pdu_size(conn->max_recv_dlength); 2562 } else 2563 conn->max_recv_dlength = max; 2564 2565 return 0; 2566 } 2567 2568 int cxgbi_set_conn_param(struct iscsi_cls_conn *cls_conn, 2569 enum iscsi_param param, char *buf, int buflen) 2570 { 2571 struct iscsi_conn *conn = cls_conn->dd_data; 2572 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2573 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2574 struct cxgbi_sock *csk = cconn->cep->csk; 2575 int err; 2576 2577 log_debug(1 << CXGBI_DBG_ISCSI, 2578 "cls_conn 0x%p, param %d, buf(%d) %s.\n", 2579 cls_conn, param, buflen, buf); 2580 2581 switch (param) { 2582 case ISCSI_PARAM_HDRDGST_EN: 2583 err = iscsi_set_param(cls_conn, param, buf, buflen); 2584 if (!err && conn->hdrdgst_en) 2585 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid, 2586 conn->hdrdgst_en, 2587 conn->datadgst_en); 2588 break; 2589 case ISCSI_PARAM_DATADGST_EN: 2590 err = iscsi_set_param(cls_conn, param, buf, buflen); 2591 if (!err && conn->datadgst_en) 2592 err = csk->cdev->csk_ddp_setup_digest(csk, csk->tid, 2593 conn->hdrdgst_en, 2594 conn->datadgst_en); 2595 break; 2596 case ISCSI_PARAM_MAX_R2T: 2597 return iscsi_tcp_set_max_r2t(conn, buf); 2598 case ISCSI_PARAM_MAX_RECV_DLENGTH: 2599 err = iscsi_set_param(cls_conn, param, buf, buflen); 2600 if (!err) 2601 err = cxgbi_conn_max_recv_dlength(conn); 2602 break; 2603 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 2604 err = iscsi_set_param(cls_conn, param, buf, buflen); 2605 if (!err) 2606 err = cxgbi_conn_max_xmit_dlength(conn); 2607 break; 2608 default: 2609 return iscsi_set_param(cls_conn, param, buf, buflen); 2610 } 2611 return err; 2612 } 2613 EXPORT_SYMBOL_GPL(cxgbi_set_conn_param); 2614 2615 int cxgbi_get_ep_param(struct iscsi_endpoint *ep, enum iscsi_param param, 2616 char *buf) 2617 { 2618 struct cxgbi_endpoint *cep = ep->dd_data; 2619 struct cxgbi_sock *csk; 2620 2621 log_debug(1 << CXGBI_DBG_ISCSI, 2622 "cls_conn 0x%p, param %d.\n", ep, param); 2623 2624 switch (param) { 2625 case ISCSI_PARAM_CONN_PORT: 2626 case ISCSI_PARAM_CONN_ADDRESS: 2627 if (!cep) 2628 return -ENOTCONN; 2629 2630 csk = cep->csk; 2631 if (!csk) 2632 return -ENOTCONN; 2633 2634 return iscsi_conn_get_addr_param((struct sockaddr_storage *) 2635 &csk->daddr, param, buf); 2636 default: 2637 break; 2638 } 2639 return -ENOSYS; 2640 } 2641 EXPORT_SYMBOL_GPL(cxgbi_get_ep_param); 2642 2643 struct iscsi_cls_conn * 2644 cxgbi_create_conn(struct iscsi_cls_session *cls_session, u32 cid) 2645 { 2646 struct iscsi_cls_conn *cls_conn; 2647 struct iscsi_conn *conn; 2648 struct iscsi_tcp_conn *tcp_conn; 2649 struct cxgbi_conn *cconn; 2650 2651 cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*cconn), cid); 2652 if (!cls_conn) 2653 return NULL; 2654 2655 conn = cls_conn->dd_data; 2656 tcp_conn = conn->dd_data; 2657 cconn = tcp_conn->dd_data; 2658 cconn->iconn = conn; 2659 2660 log_debug(1 << CXGBI_DBG_ISCSI, 2661 "cid %u(0x%x), cls 0x%p,0x%p, conn 0x%p,0x%p,0x%p.\n", 2662 cid, cid, cls_session, cls_conn, conn, tcp_conn, cconn); 2663 2664 return cls_conn; 2665 } 2666 EXPORT_SYMBOL_GPL(cxgbi_create_conn); 2667 2668 int cxgbi_bind_conn(struct iscsi_cls_session *cls_session, 2669 struct iscsi_cls_conn *cls_conn, 2670 u64 transport_eph, int is_leading) 2671 { 2672 struct iscsi_conn *conn = cls_conn->dd_data; 2673 struct iscsi_tcp_conn *tcp_conn = conn->dd_data; 2674 struct cxgbi_conn *cconn = tcp_conn->dd_data; 2675 struct cxgbi_ppm *ppm; 2676 struct iscsi_endpoint *ep; 2677 struct cxgbi_endpoint *cep; 2678 struct cxgbi_sock *csk; 2679 int err; 2680 2681 ep = iscsi_lookup_endpoint(transport_eph); 2682 if (!ep) 2683 return -EINVAL; 2684 2685 /* setup ddp pagesize */ 2686 cep = ep->dd_data; 2687 csk = cep->csk; 2688 2689 ppm = csk->cdev->cdev2ppm(csk->cdev); 2690 err = csk->cdev->csk_ddp_setup_pgidx(csk, csk->tid, 2691 ppm->tformat.pgsz_idx_dflt); 2692 if (err < 0) 2693 goto put_ep; 2694 2695 err = iscsi_conn_bind(cls_session, cls_conn, is_leading); 2696 if (err) { 2697 err = -EINVAL; 2698 goto put_ep; 2699 } 2700 2701 /* calculate the tag idx bits needed for this conn based on cmds_max */ 2702 cconn->task_idx_bits = (__ilog2_u32(conn->session->cmds_max - 1)) + 1; 2703 2704 write_lock_bh(&csk->callback_lock); 2705 csk->user_data = conn; 2706 cconn->chba = cep->chba; 2707 cconn->cep = cep; 2708 cep->cconn = cconn; 2709 write_unlock_bh(&csk->callback_lock); 2710 2711 cxgbi_conn_max_xmit_dlength(conn); 2712 cxgbi_conn_max_recv_dlength(conn); 2713 2714 log_debug(1 << CXGBI_DBG_ISCSI, 2715 "cls 0x%p,0x%p, ep 0x%p, cconn 0x%p, csk 0x%p.\n", 2716 cls_session, cls_conn, ep, cconn, csk); 2717 /* init recv engine */ 2718 iscsi_tcp_hdr_recv_prep(tcp_conn); 2719 2720 put_ep: 2721 iscsi_put_endpoint(ep); 2722 return err; 2723 } 2724 EXPORT_SYMBOL_GPL(cxgbi_bind_conn); 2725 2726 struct iscsi_cls_session *cxgbi_create_session(struct iscsi_endpoint *ep, 2727 u16 cmds_max, u16 qdepth, 2728 u32 initial_cmdsn) 2729 { 2730 struct cxgbi_endpoint *cep; 2731 struct cxgbi_hba *chba; 2732 struct Scsi_Host *shost; 2733 struct iscsi_cls_session *cls_session; 2734 struct iscsi_session *session; 2735 2736 if (!ep) { 2737 pr_err("missing endpoint.\n"); 2738 return NULL; 2739 } 2740 2741 cep = ep->dd_data; 2742 chba = cep->chba; 2743 shost = chba->shost; 2744 2745 BUG_ON(chba != iscsi_host_priv(shost)); 2746 2747 cls_session = iscsi_session_setup(chba->cdev->itp, shost, 2748 cmds_max, 0, 2749 sizeof(struct iscsi_tcp_task) + 2750 sizeof(struct cxgbi_task_data), 2751 initial_cmdsn, ISCSI_MAX_TARGET); 2752 if (!cls_session) 2753 return NULL; 2754 2755 session = cls_session->dd_data; 2756 if (iscsi_tcp_r2tpool_alloc(session)) 2757 goto remove_session; 2758 2759 log_debug(1 << CXGBI_DBG_ISCSI, 2760 "ep 0x%p, cls sess 0x%p.\n", ep, cls_session); 2761 return cls_session; 2762 2763 remove_session: 2764 iscsi_session_teardown(cls_session); 2765 return NULL; 2766 } 2767 EXPORT_SYMBOL_GPL(cxgbi_create_session); 2768 2769 void cxgbi_destroy_session(struct iscsi_cls_session *cls_session) 2770 { 2771 log_debug(1 << CXGBI_DBG_ISCSI, 2772 "cls sess 0x%p.\n", cls_session); 2773 2774 iscsi_tcp_r2tpool_free(cls_session->dd_data); 2775 iscsi_session_teardown(cls_session); 2776 } 2777 EXPORT_SYMBOL_GPL(cxgbi_destroy_session); 2778 2779 int cxgbi_set_host_param(struct Scsi_Host *shost, enum iscsi_host_param param, 2780 char *buf, int buflen) 2781 { 2782 struct cxgbi_hba *chba = iscsi_host_priv(shost); 2783 2784 if (!chba->ndev) { 2785 shost_printk(KERN_ERR, shost, "Could not get host param. " 2786 "netdev for host not set.\n"); 2787 return -ENODEV; 2788 } 2789 2790 log_debug(1 << CXGBI_DBG_ISCSI, 2791 "shost 0x%p, hba 0x%p,%s, param %d, buf(%d) %s.\n", 2792 shost, chba, chba->ndev->name, param, buflen, buf); 2793 2794 switch (param) { 2795 case ISCSI_HOST_PARAM_IPADDRESS: 2796 { 2797 __be32 addr = in_aton(buf); 2798 log_debug(1 << CXGBI_DBG_ISCSI, 2799 "hba %s, req. ipv4 %pI4.\n", chba->ndev->name, &addr); 2800 cxgbi_set_iscsi_ipv4(chba, addr); 2801 return 0; 2802 } 2803 case ISCSI_HOST_PARAM_HWADDRESS: 2804 case ISCSI_HOST_PARAM_NETDEV_NAME: 2805 return 0; 2806 default: 2807 return iscsi_host_set_param(shost, param, buf, buflen); 2808 } 2809 } 2810 EXPORT_SYMBOL_GPL(cxgbi_set_host_param); 2811 2812 int cxgbi_get_host_param(struct Scsi_Host *shost, enum iscsi_host_param param, 2813 char *buf) 2814 { 2815 struct cxgbi_hba *chba = iscsi_host_priv(shost); 2816 int len = 0; 2817 2818 if (!chba->ndev) { 2819 shost_printk(KERN_ERR, shost, "Could not get host param. " 2820 "netdev for host not set.\n"); 2821 return -ENODEV; 2822 } 2823 2824 log_debug(1 << CXGBI_DBG_ISCSI, 2825 "shost 0x%p, hba 0x%p,%s, param %d.\n", 2826 shost, chba, chba->ndev->name, param); 2827 2828 switch (param) { 2829 case ISCSI_HOST_PARAM_HWADDRESS: 2830 len = sysfs_format_mac(buf, chba->ndev->dev_addr, 6); 2831 break; 2832 case ISCSI_HOST_PARAM_NETDEV_NAME: 2833 len = sprintf(buf, "%s\n", chba->ndev->name); 2834 break; 2835 case ISCSI_HOST_PARAM_IPADDRESS: 2836 { 2837 struct cxgbi_sock *csk = find_sock_on_port(chba->cdev, 2838 chba->port_id); 2839 if (csk) { 2840 len = sprintf(buf, "%pIS", 2841 (struct sockaddr *)&csk->saddr); 2842 } 2843 log_debug(1 << CXGBI_DBG_ISCSI, 2844 "hba %s, addr %s.\n", chba->ndev->name, buf); 2845 break; 2846 } 2847 default: 2848 return iscsi_host_get_param(shost, param, buf); 2849 } 2850 2851 return len; 2852 } 2853 EXPORT_SYMBOL_GPL(cxgbi_get_host_param); 2854 2855 struct iscsi_endpoint *cxgbi_ep_connect(struct Scsi_Host *shost, 2856 struct sockaddr *dst_addr, 2857 int non_blocking) 2858 { 2859 struct iscsi_endpoint *ep; 2860 struct cxgbi_endpoint *cep; 2861 struct cxgbi_hba *hba = NULL; 2862 struct cxgbi_sock *csk; 2863 int ifindex = 0; 2864 int err = -EINVAL; 2865 2866 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2867 "shost 0x%p, non_blocking %d, dst_addr 0x%p.\n", 2868 shost, non_blocking, dst_addr); 2869 2870 if (shost) { 2871 hba = iscsi_host_priv(shost); 2872 if (!hba) { 2873 pr_info("shost 0x%p, priv NULL.\n", shost); 2874 goto err_out; 2875 } 2876 } 2877 2878 check_route: 2879 if (dst_addr->sa_family == AF_INET) { 2880 csk = cxgbi_check_route(dst_addr, ifindex); 2881 #if IS_ENABLED(CONFIG_IPV6) 2882 } else if (dst_addr->sa_family == AF_INET6) { 2883 csk = cxgbi_check_route6(dst_addr, ifindex); 2884 #endif 2885 } else { 2886 pr_info("address family 0x%x NOT supported.\n", 2887 dst_addr->sa_family); 2888 err = -EAFNOSUPPORT; 2889 return (struct iscsi_endpoint *)ERR_PTR(err); 2890 } 2891 2892 if (IS_ERR(csk)) 2893 return (struct iscsi_endpoint *)csk; 2894 cxgbi_sock_get(csk); 2895 2896 if (!hba) 2897 hba = csk->cdev->hbas[csk->port_id]; 2898 else if (hba != csk->cdev->hbas[csk->port_id]) { 2899 if (ifindex != hba->ndev->ifindex) { 2900 cxgbi_sock_put(csk); 2901 cxgbi_sock_closed(csk); 2902 ifindex = hba->ndev->ifindex; 2903 goto check_route; 2904 } 2905 2906 pr_info("Could not connect through requested host %u" 2907 "hba 0x%p != 0x%p (%u).\n", 2908 shost->host_no, hba, 2909 csk->cdev->hbas[csk->port_id], csk->port_id); 2910 err = -ENOSPC; 2911 goto release_conn; 2912 } 2913 2914 err = sock_get_port(csk); 2915 if (err) 2916 goto release_conn; 2917 2918 cxgbi_sock_set_state(csk, CTP_CONNECTING); 2919 err = csk->cdev->csk_init_act_open(csk); 2920 if (err) 2921 goto release_conn; 2922 2923 if (cxgbi_sock_is_closing(csk)) { 2924 err = -ENOSPC; 2925 pr_info("csk 0x%p is closing.\n", csk); 2926 goto release_conn; 2927 } 2928 2929 ep = iscsi_create_endpoint(sizeof(*cep)); 2930 if (!ep) { 2931 err = -ENOMEM; 2932 pr_info("iscsi alloc ep, OOM.\n"); 2933 goto release_conn; 2934 } 2935 2936 cep = ep->dd_data; 2937 cep->csk = csk; 2938 cep->chba = hba; 2939 2940 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2941 "ep 0x%p, cep 0x%p, csk 0x%p, hba 0x%p,%s.\n", 2942 ep, cep, csk, hba, hba->ndev->name); 2943 return ep; 2944 2945 release_conn: 2946 cxgbi_sock_put(csk); 2947 cxgbi_sock_closed(csk); 2948 err_out: 2949 return ERR_PTR(err); 2950 } 2951 EXPORT_SYMBOL_GPL(cxgbi_ep_connect); 2952 2953 int cxgbi_ep_poll(struct iscsi_endpoint *ep, int timeout_ms) 2954 { 2955 struct cxgbi_endpoint *cep = ep->dd_data; 2956 struct cxgbi_sock *csk = cep->csk; 2957 2958 if (!cxgbi_sock_is_established(csk)) 2959 return 0; 2960 return 1; 2961 } 2962 EXPORT_SYMBOL_GPL(cxgbi_ep_poll); 2963 2964 void cxgbi_ep_disconnect(struct iscsi_endpoint *ep) 2965 { 2966 struct cxgbi_endpoint *cep = ep->dd_data; 2967 struct cxgbi_conn *cconn = cep->cconn; 2968 struct cxgbi_sock *csk = cep->csk; 2969 2970 log_debug(1 << CXGBI_DBG_ISCSI | 1 << CXGBI_DBG_SOCK, 2971 "ep 0x%p, cep 0x%p, cconn 0x%p, csk 0x%p,%u,0x%lx.\n", 2972 ep, cep, cconn, csk, csk->state, csk->flags); 2973 2974 if (cconn && cconn->iconn) { 2975 write_lock_bh(&csk->callback_lock); 2976 cep->csk->user_data = NULL; 2977 cconn->cep = NULL; 2978 write_unlock_bh(&csk->callback_lock); 2979 } 2980 iscsi_destroy_endpoint(ep); 2981 2982 if (likely(csk->state >= CTP_ESTABLISHED)) 2983 need_active_close(csk); 2984 else 2985 cxgbi_sock_closed(csk); 2986 2987 cxgbi_sock_put(csk); 2988 } 2989 EXPORT_SYMBOL_GPL(cxgbi_ep_disconnect); 2990 2991 int cxgbi_iscsi_init(struct iscsi_transport *itp, 2992 struct scsi_transport_template **stt) 2993 { 2994 *stt = iscsi_register_transport(itp); 2995 if (*stt == NULL) { 2996 pr_err("unable to register %s transport 0x%p.\n", 2997 itp->name, itp); 2998 return -ENODEV; 2999 } 3000 log_debug(1 << CXGBI_DBG_ISCSI, 3001 "%s, registered iscsi transport 0x%p.\n", 3002 itp->name, stt); 3003 return 0; 3004 } 3005 EXPORT_SYMBOL_GPL(cxgbi_iscsi_init); 3006 3007 void cxgbi_iscsi_cleanup(struct iscsi_transport *itp, 3008 struct scsi_transport_template **stt) 3009 { 3010 if (*stt) { 3011 log_debug(1 << CXGBI_DBG_ISCSI, 3012 "de-register transport 0x%p, %s, stt 0x%p.\n", 3013 itp, itp->name, *stt); 3014 *stt = NULL; 3015 iscsi_unregister_transport(itp); 3016 } 3017 } 3018 EXPORT_SYMBOL_GPL(cxgbi_iscsi_cleanup); 3019 3020 umode_t cxgbi_attr_is_visible(int param_type, int param) 3021 { 3022 switch (param_type) { 3023 case ISCSI_HOST_PARAM: 3024 switch (param) { 3025 case ISCSI_HOST_PARAM_NETDEV_NAME: 3026 case ISCSI_HOST_PARAM_HWADDRESS: 3027 case ISCSI_HOST_PARAM_IPADDRESS: 3028 case ISCSI_HOST_PARAM_INITIATOR_NAME: 3029 return S_IRUGO; 3030 default: 3031 return 0; 3032 } 3033 case ISCSI_PARAM: 3034 switch (param) { 3035 case ISCSI_PARAM_MAX_RECV_DLENGTH: 3036 case ISCSI_PARAM_MAX_XMIT_DLENGTH: 3037 case ISCSI_PARAM_HDRDGST_EN: 3038 case ISCSI_PARAM_DATADGST_EN: 3039 case ISCSI_PARAM_CONN_ADDRESS: 3040 case ISCSI_PARAM_CONN_PORT: 3041 case ISCSI_PARAM_EXP_STATSN: 3042 case ISCSI_PARAM_PERSISTENT_ADDRESS: 3043 case ISCSI_PARAM_PERSISTENT_PORT: 3044 case ISCSI_PARAM_PING_TMO: 3045 case ISCSI_PARAM_RECV_TMO: 3046 case ISCSI_PARAM_INITIAL_R2T_EN: 3047 case ISCSI_PARAM_MAX_R2T: 3048 case ISCSI_PARAM_IMM_DATA_EN: 3049 case ISCSI_PARAM_FIRST_BURST: 3050 case ISCSI_PARAM_MAX_BURST: 3051 case ISCSI_PARAM_PDU_INORDER_EN: 3052 case ISCSI_PARAM_DATASEQ_INORDER_EN: 3053 case ISCSI_PARAM_ERL: 3054 case ISCSI_PARAM_TARGET_NAME: 3055 case ISCSI_PARAM_TPGT: 3056 case ISCSI_PARAM_USERNAME: 3057 case ISCSI_PARAM_PASSWORD: 3058 case ISCSI_PARAM_USERNAME_IN: 3059 case ISCSI_PARAM_PASSWORD_IN: 3060 case ISCSI_PARAM_FAST_ABORT: 3061 case ISCSI_PARAM_ABORT_TMO: 3062 case ISCSI_PARAM_LU_RESET_TMO: 3063 case ISCSI_PARAM_TGT_RESET_TMO: 3064 case ISCSI_PARAM_IFACE_NAME: 3065 case ISCSI_PARAM_INITIATOR_NAME: 3066 return S_IRUGO; 3067 default: 3068 return 0; 3069 } 3070 } 3071 3072 return 0; 3073 } 3074 EXPORT_SYMBOL_GPL(cxgbi_attr_is_visible); 3075 3076 static int __init libcxgbi_init_module(void) 3077 { 3078 pr_info("%s", version); 3079 3080 BUILD_BUG_ON(sizeof_field(struct sk_buff, cb) < 3081 sizeof(struct cxgbi_skb_cb)); 3082 rsvd_page = alloc_page(GFP_KERNEL | __GFP_ZERO); 3083 if (!rsvd_page) 3084 return -ENOMEM; 3085 3086 return 0; 3087 } 3088 3089 static void __exit libcxgbi_exit_module(void) 3090 { 3091 cxgbi_device_unregister_all(0xFF); 3092 put_page(rsvd_page); 3093 return; 3094 } 3095 3096 module_init(libcxgbi_init_module); 3097 module_exit(libcxgbi_exit_module); 3098